Add WAL mode, connection pooling, pagination, and delta sync
WAL + pooling: - Enable WAL journal mode for concurrent read/write support - Add connection pool (size 4) with return_db() to reuse connections instead of opening/closing on every request Pagination: - Search results, /pages, and /tags/<name> now paginate at 50 per page - Prev/next navigation links appear when results exceed one page Delta sync: - Pages table gains last_modified timestamp, set on insert/update - /api/sites accepts ?since= param to return only changed pages - Subscription sync uses last_sync timestamp for incremental fetches - Remote pages upserted instead of delete-all/re-insert - Full sync includes all_urls list for detecting remote deletions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
6981d39ddd
commit
f2e8dd042a
3 changed files with 193 additions and 69 deletions
69
db.py
69
db.py
|
|
@ -77,13 +77,35 @@ def clean_url(url):
|
|||
return urlunparse((scheme, netloc, path, "", new_query, ""))
|
||||
|
||||
|
||||
_pool = []
|
||||
_pool_lock = __import__("threading").Lock()
|
||||
_POOL_SIZE = 4
|
||||
|
||||
|
||||
def get_db():
|
||||
db = sqlite3.connect(DATABASE)
|
||||
with _pool_lock:
|
||||
if _pool:
|
||||
db = _pool.pop()
|
||||
try:
|
||||
db.execute("SELECT 1")
|
||||
return db
|
||||
except Exception:
|
||||
pass
|
||||
db = sqlite3.connect(DATABASE, timeout=10)
|
||||
db.execute("PRAGMA journal_mode=WAL")
|
||||
db.execute("PRAGMA foreign_keys = ON")
|
||||
db.row_factory = sqlite3.Row
|
||||
return db
|
||||
|
||||
|
||||
def return_db(db):
|
||||
with _pool_lock:
|
||||
if len(_pool) < _POOL_SIZE:
|
||||
_pool.append(db)
|
||||
else:
|
||||
db.close()
|
||||
|
||||
|
||||
def init_db():
|
||||
db = sqlite3.connect(DATABASE)
|
||||
db.execute(
|
||||
|
|
@ -92,7 +114,8 @@ def init_db():
|
|||
" url TEXT UNIQUE NOT NULL,"
|
||||
" title TEXT,"
|
||||
" body TEXT,"
|
||||
" note TEXT DEFAULT ''"
|
||||
" note TEXT DEFAULT '',"
|
||||
" last_modified TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now'))"
|
||||
")"
|
||||
)
|
||||
db.execute(
|
||||
|
|
@ -196,26 +219,38 @@ def init_db():
|
|||
db.execute("ALTER TABLE remote_pages ADD COLUMN tags TEXT DEFAULT ''")
|
||||
db.commit()
|
||||
|
||||
# Migrate pages: add last_modified column if missing
|
||||
page_cols = [row[1] for row in db.execute("PRAGMA table_info(pages)").fetchall()]
|
||||
if "last_modified" not in page_cols:
|
||||
db.execute("ALTER TABLE pages ADD COLUMN last_modified TEXT DEFAULT ''")
|
||||
db.execute("UPDATE pages SET last_modified = strftime('%Y-%m-%dT%H:%M:%S','now') WHERE last_modified = ''")
|
||||
db.commit()
|
||||
|
||||
db.execute("PRAGMA journal_mode=WAL")
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
def get_setting(key, default=""):
|
||||
db = get_db()
|
||||
row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
|
||||
db.close()
|
||||
return row["value"] if row else default
|
||||
try:
|
||||
row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone()
|
||||
return row["value"] if row else default
|
||||
finally:
|
||||
return_db(db)
|
||||
|
||||
|
||||
def set_setting(key, value):
|
||||
db = get_db()
|
||||
db.execute(
|
||||
"INSERT INTO settings (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
||||
(key, value),
|
||||
)
|
||||
db.commit()
|
||||
db.close()
|
||||
try:
|
||||
db.execute(
|
||||
"INSERT INTO settings (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
||||
(key, value),
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
return_db(db)
|
||||
|
||||
|
||||
def get_site_name():
|
||||
|
|
@ -273,10 +308,12 @@ def index_url(url, note=""):
|
|||
title, body, links = fetch_page(url)
|
||||
db = get_db()
|
||||
try:
|
||||
now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
||||
db.execute(
|
||||
"INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) "
|
||||
"ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note",
|
||||
(url, title, body, note),
|
||||
"INSERT INTO pages (url, title, body, note, last_modified) VALUES (?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, "
|
||||
"note=excluded.note, last_modified=excluded.last_modified",
|
||||
(url, title, body, note, now),
|
||||
)
|
||||
page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0]
|
||||
db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
|
||||
|
|
@ -287,5 +324,5 @@ def index_url(url, note=""):
|
|||
)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
return_db(db)
|
||||
return title
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue