Optimized storage and updated readme

db.py

@@ -97,7 +97,7 @@ def clean_url(url):
 
 _pool = []
 _pool_lock = __import__("threading").Lock()
-_POOL_SIZE = 4
+_POOL_SIZE = 16
 
 
 def get_db():
@@ -271,8 +271,15 @@ def init_db():
     )
     db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_page ON chunks(page_id)")
     db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_remote ON chunks(remote_page_id)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_page_idx ON chunks(page_id, chunk_index)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_pages_url ON pages(url)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_pages_modified ON pages(last_modified)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_page ON page_tags(page_id)")
+    db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_tag ON page_tags(tag_id)")
 
     db.execute("PRAGMA journal_mode=WAL")
+    db.execute("PRAGMA synchronous=NORMAL")
+    db.execute("PRAGMA cache_size=-64000")
     db.commit()
     db.close()
 
@@ -286,6 +293,16 @@ def get_setting(key, default=""):
         return_db(db)
 
 
+def vacuum_db():
+    """Run VACUUM and WAL checkpoint to reclaim space after deletions."""
+    db = get_db()
+    try:
+        db.execute("PRAGMA wal_checkpoint(TRUNCATE)")
+        db.execute("VACUUM")
+    finally:
+        return_db(db)
+
+
 def set_setting(key, value):
     db = get_db()
     try: