enabled WAL mode, pooling, pagination

WAL + pooling:
- Enable WAL journal mode for concurrent read/write support
- Add connection pool (size 4) with return_db() to reuse connections
  instead of opening/closing on every request

Pagination:
- Search results, /pages, and /tags/<name> now paginate at 50 per page
- Prev/next navigation links appear when results exceed one page

Delta sync:
- Pages table gains last_modified timestamp, set on insert/update
- /api/sites accepts ?since= param to return only changed pages
- Subscription sync uses last_sync timestamp for incremental fetches
- Remote pages upserted instead of delete-all/re-insert
- Full sync includes all_urls list for detecting remote deletions
This commit is contained in:
lichenblankie 2026-03-26 12:00:43 -07:00
parent b574c4b7f5
commit 67084bbaed
3 changed files with 193 additions and 69 deletions

69
db.py
View file

@ -77,13 +77,35 @@ def clean_url(url):
return urlunparse((scheme, netloc, path, "", new_query, ""))
_pool = []
_pool_lock = __import__("threading").Lock()
_POOL_SIZE = 4
def get_db():
db = sqlite3.connect(DATABASE)
with _pool_lock:
if _pool:
db = _pool.pop()
try:
db.execute("SELECT 1")
return db
except Exception:
pass
db = sqlite3.connect(DATABASE, timeout=10)
db.execute("PRAGMA journal_mode=WAL")
db.execute("PRAGMA foreign_keys = ON")
db.row_factory = sqlite3.Row
return db
def return_db(db):
with _pool_lock:
if len(_pool) < _POOL_SIZE:
_pool.append(db)
else:
db.close()
def init_db():
db = sqlite3.connect(DATABASE)
db.execute(
@ -92,7 +114,8 @@ def init_db():
" url TEXT UNIQUE NOT NULL,"
" title TEXT,"
" body TEXT,"
" note TEXT DEFAULT ''"
" note TEXT DEFAULT '',"
" last_modified TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now'))"
")"
)
db.execute(
@ -196,26 +219,38 @@ def init_db():
db.execute("ALTER TABLE remote_pages ADD COLUMN tags TEXT DEFAULT ''")
db.commit()
# Migrate pages: add last_modified column if missing
page_cols = [row[1] for row in db.execute("PRAGMA table_info(pages)").fetchall()]
if "last_modified" not in page_cols:
db.execute("ALTER TABLE pages ADD COLUMN last_modified TEXT DEFAULT ''")
db.execute("UPDATE pages SET last_modified = strftime('%Y-%m-%dT%H:%M:%S','now') WHERE last_modified = ''")
db.commit()
db.execute("PRAGMA journal_mode=WAL")
db.commit()
db.close()
def get_setting(key, default=""):
db = get_db()
row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
db.close()
return row["value"] if row else default
try:
row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
return row["value"] if row else default
finally:
return_db(db)
def set_setting(key, value):
db = get_db()
db.execute(
"INSERT INTO settings (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
(key, value),
)
db.commit()
db.close()
try:
db.execute(
"INSERT INTO settings (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
(key, value),
)
db.commit()
finally:
return_db(db)
def get_site_name():
@ -273,10 +308,12 @@ def index_url(url, note=""):
title, body, links = fetch_page(url)
db = get_db()
try:
now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
db.execute(
"INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) "
"ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note",
(url, title, body, note),
"INSERT INTO pages (url, title, body, note, last_modified) VALUES (?, ?, ?, ?, ?) "
"ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, "
"note=excluded.note, last_modified=excluded.last_modified",
(url, title, body, note, now),
)
page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0]
db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
@ -287,5 +324,5 @@ def index_url(url, note=""):
)
db.commit()
finally:
db.close()
return_db(db)
return title