1406 lines
54 KiB
Python
1406 lines
54 KiB
Python
import json
|
|
import re
|
|
import secrets
|
|
import threading
|
|
from datetime import datetime
|
|
from urllib.parse import unquote
|
|
|
|
from db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url
|
|
from templates import esc, wrap_page, DEFAULT_TEMPLATE
|
|
from rns_client import fetch_remote_sites
|
|
|
|
_request_local = threading.local()
|
|
|
|
|
|
def _get_csrf_token():
|
|
return getattr(_request_local, 'csrf_token', '')
|
|
|
|
|
|
def _csrf_field():
|
|
return f'<input type="hidden" name="_csrf" value="{_get_csrf_token()}">'
|
|
|
|
|
|
def _check_csrf(body):
|
|
token = body.get("_csrf", [""])[0]
|
|
expected = _get_csrf_token()
|
|
if not expected or not token:
|
|
return False
|
|
return secrets.compare_digest(token, expected)
|
|
|
|
|
|
_STOPWORDS = frozenset({
|
|
"a", "an", "the", "and", "or", "but", "is", "are", "was", "were",
|
|
"in", "on", "at", "to", "for", "of", "with", "by", "from", "as",
|
|
"into", "about", "how", "what", "which", "who", "where", "when",
|
|
"do", "does", "did", "be", "been", "being", "have", "has", "had",
|
|
"it", "its", "this", "that", "not", "no", "so", "if", "can", "will",
|
|
"my", "your", "i", "me", "we", "you", "he", "she", "they",
|
|
})
|
|
|
|
|
|
def _sanitize_fts_query(query):
|
|
"""Escape user input for safe use in FTS5 MATCH.
|
|
|
|
Splits into individual quoted tokens joined by implicit AND,
|
|
so all words must appear but in any order. Appends * to the
|
|
last token for prefix matching. Stopwords are dropped to avoid
|
|
overly strict matching.
|
|
"""
|
|
words = query.split()
|
|
if not words:
|
|
return '""'
|
|
tokens = []
|
|
for i, w in enumerate(words):
|
|
# Strip FTS5 special characters to prevent injection
|
|
cleaned = re.sub(r'["\'\(\)\*\+\-\^~]', '', w).strip()
|
|
if not cleaned:
|
|
continue
|
|
if cleaned.lower() in _STOPWORDS:
|
|
continue
|
|
if i == len(words) - 1:
|
|
# Prefix match on the last token for partial word matching
|
|
tokens.append(f"{cleaned}*")
|
|
else:
|
|
tokens.append(f'"{cleaned}"')
|
|
return " ".join(tokens) if tokens else '""'
|
|
|
|
|
|
def _get_bookmark_token():
|
|
token = get_setting("bookmark_token")
|
|
if not token:
|
|
token = secrets.token_hex(16)
|
|
set_setting("bookmark_token", token)
|
|
return token
|
|
|
|
|
|
def _respond(body_html, status=200, use_default=False):
|
|
return {
|
|
"status": status,
|
|
"content_type": "text/html; charset=utf-8",
|
|
"body": wrap_page(body_html, use_default=use_default),
|
|
"headers": {},
|
|
}
|
|
|
|
|
|
def _redirect(location):
|
|
if not location.startswith("/") or location.startswith("//"):
|
|
location = "/"
|
|
return {
|
|
"status": 302,
|
|
"content_type": "text/html; charset=utf-8",
|
|
"body": "",
|
|
"headers": {"Location": location},
|
|
}
|
|
|
|
|
|
def _json_response(data, status=200, headers=None):
|
|
return {
|
|
"status": status,
|
|
"content_type": "application/json",
|
|
"body": json.dumps(data, indent=2),
|
|
"headers": headers or {},
|
|
}
|
|
|
|
|
|
def _text_response(text, status=200, headers=None):
|
|
return {
|
|
"status": status,
|
|
"content_type": "text/plain",
|
|
"body": text,
|
|
"headers": headers or {},
|
|
}
|
|
|
|
|
|
def _error(status):
|
|
return _respond(f"<h1>{status}</h1>", status)
|
|
|
|
|
|
PER_PAGE = 10
|
|
BROWSE_PER_PAGE = 50
|
|
|
|
|
|
def _paginate(query, key="p"):
|
|
try:
|
|
page = int(query.get(key, ["1"])[0])
|
|
except (ValueError, IndexError):
|
|
page = 1
|
|
return max(1, page)
|
|
|
|
|
|
def _page_nav(page, total, base_url, per_page=None):
|
|
per_page = per_page or PER_PAGE
|
|
if total <= per_page:
|
|
return ""
|
|
total_pages = (total + per_page - 1) // per_page
|
|
sep = "&" if "?" in base_url else "?"
|
|
parts = []
|
|
if page > 1:
|
|
parts.append(f'<a href="{base_url}{sep}p={page - 1}">« prev</a>')
|
|
parts.append(f"page {page} of {total_pages}")
|
|
if page < total_pages:
|
|
parts.append(f'<a href="{base_url}{sep}p={page + 1}">next »</a>')
|
|
return f'<p class="pagination">{" | ".join(parts)}</p>'
|
|
|
|
|
|
# --- Tag helpers ---
|
|
|
|
|
|
def _get_page_tags(page_id, db=None):
|
|
close = False
|
|
if db is None:
|
|
db = get_db()
|
|
close = True
|
|
rows = db.execute(
|
|
"SELECT t.name FROM tags t JOIN page_tags pt ON t.id = pt.tag_id "
|
|
"WHERE pt.page_id = ? ORDER BY t.name", (page_id,)
|
|
).fetchall()
|
|
if close:
|
|
return_db(db)
|
|
return [r["name"] for r in rows]
|
|
|
|
|
|
def _set_page_tags(page_id, tag_string, db=None):
|
|
close = False
|
|
if db is None:
|
|
db = get_db()
|
|
close = True
|
|
db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,))
|
|
for name in (t.strip().lower() for t in tag_string.split(",") if t.strip()):
|
|
db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,))
|
|
tag_id = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()["id"]
|
|
db.execute("INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", (page_id, tag_id))
|
|
if close:
|
|
db.commit()
|
|
return_db(db)
|
|
|
|
|
|
# --- Route handlers ---
|
|
|
|
|
|
def handle_search(query):
|
|
q = query.get("q", [""])[0].strip()
|
|
page = _paginate(query)
|
|
offset = (page - 1) * PER_PAGE
|
|
db = get_db()
|
|
try:
|
|
count = db.execute("SELECT count(*) FROM pages").fetchone()[0]
|
|
name = get_site_name()
|
|
|
|
result_html = ""
|
|
trusted_html = ""
|
|
if q:
|
|
# BM25 keyword search with column weights: title=10, body=1, url=5, note=3
|
|
try:
|
|
fts_q = _sanitize_fts_query(q)
|
|
bm25_rows = db.execute(
|
|
"SELECT p.id, p.url, p.title, p.body, p.note "
|
|
"FROM pages_fts f JOIN pages p ON f.rowid = p.id "
|
|
"WHERE pages_fts MATCH ? "
|
|
"ORDER BY bm25(pages_fts, 10.0, 1.0, 5.0, 3.0) LIMIT 100",
|
|
(fts_q,),
|
|
).fetchall()
|
|
except Exception:
|
|
bm25_rows = []
|
|
|
|
# Hybrid search: merge BM25 + semantic via RRF
|
|
bm25_ids = [r["id"] for r in bm25_rows]
|
|
chunk_snippets = {} # page_id -> best chunk text
|
|
if get_setting("semantic_search", "1") == "1":
|
|
try:
|
|
from embeddings import hybrid_search
|
|
use_reranker = get_setting("use_reranker", "1") == "1"
|
|
fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker)
|
|
fused_ids = [pid for pid, _ in fused]
|
|
chunk_snippets = {pid: text for pid, text in fused if text}
|
|
except Exception:
|
|
fused_ids = bm25_ids
|
|
else:
|
|
fused_ids = bm25_ids
|
|
|
|
total_results = len(fused_ids)
|
|
page_ids = fused_ids[offset:offset + PER_PAGE]
|
|
|
|
if page_ids:
|
|
# Fetch rows in fused order
|
|
placeholders = ",".join("?" * len(page_ids))
|
|
all_rows = db.execute(
|
|
f"SELECT id, url, title, body, note, summary FROM pages WHERE id IN ({placeholders})",
|
|
page_ids,
|
|
).fetchall()
|
|
row_map = {r["id"]: r for r in all_rows}
|
|
rows = [row_map[pid] for pid in page_ids if pid in row_map]
|
|
else:
|
|
rows = []
|
|
|
|
if rows:
|
|
for r in rows:
|
|
note_html = ""
|
|
if r["note"]:
|
|
note_html = f'<div class="note"><em>{esc(r["note"])}</em></div>'
|
|
tags = _get_page_tags(r["id"], db)
|
|
tags_html = ""
|
|
if tags:
|
|
tag_links = " ".join(f'<a href="/tags/{esc(t)}" class="tag">[{esc(t)}]</a>' for t in tags)
|
|
tags_html = f'<div class="tags">{tag_links}</div>'
|
|
snip_html = f'<br>{esc(r["summary"])}' if r["summary"] else ""
|
|
result_html += (
|
|
f'<div class="result">'
|
|
f'<a href="{esc(r["url"])}">{esc(r["title"])}</a><br>'
|
|
f'<small>{esc(r["url"])}</small>'
|
|
f'{snip_html}'
|
|
f'{note_html}{tags_html}'
|
|
f'</div>'
|
|
)
|
|
else:
|
|
result_html = "<p>No results in your index.</p>"
|
|
|
|
# search all linked pages from trusted sites
|
|
words = q.lower().split()
|
|
all_links = db.execute(
|
|
"SELECT l.url, l.label, p.title AS source_title "
|
|
"FROM links l JOIN pages p ON l.page_id = p.id",
|
|
).fetchall()
|
|
indexed_urls = set(r["url"] for r in rows) if rows else set()
|
|
seen = set()
|
|
trusted = []
|
|
for l in all_links:
|
|
if l["url"] in indexed_urls or l["url"] in seen:
|
|
continue
|
|
if any(w in l["label"].lower() for w in words):
|
|
seen.add(l["url"])
|
|
trusted.append(l)
|
|
if len(trusted) >= 20:
|
|
break
|
|
|
|
if trusted:
|
|
items = ""
|
|
for l in trusted:
|
|
items += (
|
|
f'<li><a href="{esc(l["url"])}">{esc(l["label"])}</a> '
|
|
f'<small>— from {esc(l["source_title"])}</small></li>'
|
|
)
|
|
trusted_html = (
|
|
f'<details class="trusted">'
|
|
f'<summary>from your trusted sites ({len(trusted)})</summary>'
|
|
f'<ul>{items}</ul>'
|
|
f'</details>'
|
|
)
|
|
|
|
# search synced pages from subscriptions
|
|
try:
|
|
remote_rows = db.execute(
|
|
"SELECT rp.url, rp.title, rp.note, s.name AS source_name "
|
|
"FROM remote_pages_fts rpf "
|
|
"JOIN remote_pages rp ON rpf.rowid = rp.id "
|
|
"JOIN subscriptions s ON rp.subscription_id = s.id "
|
|
"WHERE remote_pages_fts MATCH ? ORDER BY rank LIMIT 50",
|
|
(_sanitize_fts_query(q),),
|
|
).fetchall()
|
|
except Exception:
|
|
remote_rows = []
|
|
|
|
remote_html = ""
|
|
if q and remote_rows:
|
|
# group by source
|
|
by_source = {}
|
|
for r in remote_rows:
|
|
source = r["source_name"] or "unknown"
|
|
by_source.setdefault(source, []).append(r)
|
|
for source, items in by_source.items():
|
|
source_items = ""
|
|
for r in items:
|
|
note_html = f' — <em>{esc(r["note"])}</em>' if r["note"] else ""
|
|
source_items += (
|
|
f'<li><a href="{esc(r["url"])}">{esc(r["title"])}</a>'
|
|
f'{note_html} <small>({esc(r["url"])})</small></li>'
|
|
)
|
|
remote_html += (
|
|
f'<details class="remote" open>'
|
|
f'<summary>from {esc(source)} ({len(items)})</summary>'
|
|
f'<ul>{source_items}</ul>'
|
|
f'</details>'
|
|
)
|
|
finally:
|
|
return_db(db)
|
|
sub_count = ""
|
|
if q and remote_rows:
|
|
sub_count = f" + {len(remote_rows)} from subscriptions"
|
|
return _respond(
|
|
f'<form method="get" action="/">'
|
|
f'<input name="q" value="{esc(q)}" placeholder="search your index" size="40">'
|
|
f' <button type="submit">search</button>'
|
|
f'</form>'
|
|
f'<p class="meta">{count} pages indexed'
|
|
f' · <a href="/add">+ add url</a></p>'
|
|
f'{result_html}'
|
|
f'{_page_nav(page, total_results, f"/?q={esc(q)}") if q else ""}'
|
|
f'{trusted_html}{remote_html}'
|
|
)
|
|
|
|
|
|
def handle_add_form(msg="", action_type="index"):
|
|
if action_type == "subscribe":
|
|
return _respond(
|
|
f"<h1>subscribe</h1>"
|
|
f"<p>Subscribe to a friend's TinyWeb instance to sync their index</p>"
|
|
f'<form method="post" action="/subscriptions/add">'
|
|
f'{_csrf_field()}'
|
|
f'<input name="dest_hash" placeholder="destination hash (32 hex chars)" size="50"><br><br>'
|
|
f'<button type="submit">subscribe</button>'
|
|
f"</form>"
|
|
f"<p><small>or <a href=\"/add\">add a single site</a></small></p>"
|
|
f"<p>{msg}</p>"
|
|
f'<a href="/">back</a>'
|
|
)
|
|
return _respond(
|
|
f"<h1>add url</h1>"
|
|
f"<p>Add a site to your index</p>"
|
|
f'<form method="post" action="/add">'
|
|
f'{_csrf_field()}'
|
|
f'<input type="radio" name="input_type" value="url" id="type_url" checked onchange="toggleInput()">'
|
|
f'<label for="type_url">URL</label>'
|
|
f'<input type="radio" name="input_type" value="hash" id="type_hash" onchange="toggleInput()">'
|
|
f'<label for="type_hash">Reticulum Hash</label>'
|
|
f'<br><br>'
|
|
f'<input name="url" id="url_input" placeholder="https://example.com" size="50"><br>'
|
|
f'<input name="reticulum_dest" id="hash_input" placeholder="reticulum destination hash (32 hex chars)" size="50" style="display:none"><br>'
|
|
f'<input name="note" placeholder="why are you saving this? (optional)" size="50" style="margin-top:8px"><br>'
|
|
f'<input name="tags" placeholder="tags (comma-separated, e.g. solarpunk, mesh)" size="50" style="margin-top:8px"><br><br>'
|
|
f'<button type="submit">index</button>'
|
|
f"</form>"
|
|
f"<p>{msg}</p>"
|
|
f'<a href="/">back</a>'
|
|
f'<script>'
|
|
f'function toggleInput() {{'
|
|
f' var isUrl = document.getElementById("type_url").checked;'
|
|
f' document.getElementById("url_input").style.display = isUrl ? "block" : "none";'
|
|
f' document.getElementById("hash_input").style.display = isUrl ? "none" : "block";'
|
|
f'}}'
|
|
f'</script>'
|
|
)
|
|
|
|
|
|
def handle_add_submit(body):
|
|
input_type = body.get("input_type", ["url"])[0]
|
|
url = body.get("url", [""])[0].strip()
|
|
reticulum_dest = body.get("reticulum_dest", [""])[0].strip().replace("<", "").replace(">", "")
|
|
note = body.get("note", [""])[0].strip()
|
|
tags = body.get("tags", [""])[0].strip()
|
|
|
|
if input_type == "url":
|
|
if not url:
|
|
return handle_add_form("URL is required.")
|
|
url = clean_url(url)
|
|
if not url.startswith(("http://", "https://")):
|
|
return handle_add_form("URL must start with http:// or https://")
|
|
else:
|
|
if not reticulum_dest:
|
|
return handle_add_form("Reticulum destination hash is required.")
|
|
if len(reticulum_dest) != 32 or not all(c in "0123456789abcdefABCDEF" for c in reticulum_dest):
|
|
return handle_add_form("Invalid reticulum destination hash. Must be 32 hex characters.")
|
|
url = f"reticulum:{reticulum_dest}"
|
|
|
|
try:
|
|
title = index_url(url, note, reticulum_dest if reticulum_dest else "")
|
|
if tags:
|
|
db = get_db()
|
|
try:
|
|
row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()
|
|
if row:
|
|
_set_page_tags(row["id"], tags, db)
|
|
db.commit()
|
|
finally:
|
|
return_db(db)
|
|
|
|
return handle_add_form(f'Indexed: {esc(url)}')
|
|
|
|
except ValueError as e:
|
|
return handle_add_form(f"Error: {esc(str(e))}")
|
|
|
|
except Exception as e:
|
|
error_msg = str(e).lower()
|
|
# Check if it's a block response
|
|
if "block" in error_msg or "cloudflare" in error_msg or "403" in error_msg:
|
|
# Show manual entry form for blocked sites
|
|
return _respond(
|
|
f"<h1>add url (manual entry)</h1>"
|
|
f"<p><strong>{esc(url)}</strong> blocks automated access. "
|
|
f"You can still save it manually:</p>"
|
|
f'<form method="post" action="/add/manual">'
|
|
f'{_csrf_field()}'
|
|
f'<input type="hidden" name="url" value="{esc(url)}">'
|
|
f'<input type="hidden" name="note" value="{esc(note)}">'
|
|
f'<input type="hidden" name="tags" value="{esc(tags)}">'
|
|
f'<label>Title:</label><br>'
|
|
f'<input name="manual_title" size="50" placeholder="page title" required><br><br>'
|
|
f'<label>Description:</label><br>'
|
|
f'<textarea name="manual_description" rows="4" cols="50" placeholder="what is this site about?" required></textarea><br><br>'
|
|
f'<button type="submit">save manually</button>'
|
|
f"</form>"
|
|
f'<a href="/">back</a>'
|
|
)
|
|
return handle_add_form(f"Error: could not fetch or index that URL. {esc(str(e)[:100])}")
|
|
|
|
|
|
def handle_add_manual_submit(body):
|
|
url = clean_url(body.get("url", [""])[0].strip())
|
|
note = body.get("note", [""])[0].strip()
|
|
tags = body.get("tags", [""])[0].strip()
|
|
manual_title = body.get("manual_title", [""])[0].strip()
|
|
manual_desc = body.get("manual_description", [""])[0].strip()
|
|
|
|
if not url:
|
|
return handle_add_form("URL is required.")
|
|
|
|
if not manual_title or not manual_desc:
|
|
return handle_add_form("Title and description are required for manual entry.")
|
|
|
|
db = get_db()
|
|
try:
|
|
now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
|
|
|
db.execute(
|
|
"INSERT INTO pages (url, title, body, note, last_modified, summary) VALUES (?, ?, ?, ?, ?, ?) "
|
|
"ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, "
|
|
"note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary",
|
|
(url, manual_title, manual_desc, note, now, manual_desc[:200]),
|
|
)
|
|
|
|
# Get the page ID
|
|
page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0]
|
|
|
|
# Add tags if provided
|
|
if tags:
|
|
_set_page_tags(page_id, tags, db)
|
|
|
|
db.commit()
|
|
|
|
# Generate embeddings for this page (if semantic search is enabled)
|
|
if get_setting("semantic_search", "1") == "1":
|
|
try:
|
|
from embeddings import store_embeddings
|
|
# Pass the page_id, title, description, and db connection
|
|
store_embeddings(page_id, manual_title, manual_desc, db)
|
|
db.commit()
|
|
except Exception as e:
|
|
# Log error but don't fail the whole operation
|
|
print(f"Error generating embeddings: {e}")
|
|
|
|
return handle_add_form(f'Added manually: <a href="{esc(url)}">{esc(manual_title)}</a>')
|
|
finally:
|
|
return_db(db)
|
|
|
|
|
|
def handle_pages(query=None):
|
|
msg = query.get("msg", [""])[0] if query else ""
|
|
msg_html = f'<p class="success">{esc(msg)}</p>' if msg else ""
|
|
page = _paginate(query or {})
|
|
offset = (page - 1) * BROWSE_PER_PAGE
|
|
db = get_db()
|
|
try:
|
|
total = db.execute("SELECT count(*) FROM pages").fetchone()[0]
|
|
rows = db.execute(
|
|
"SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?",
|
|
(BROWSE_PER_PAGE, offset),
|
|
).fetchall()
|
|
items = ""
|
|
for r in rows:
|
|
note_html = f' — <em>{esc(r["note"])}</em>' if r["note"] else ""
|
|
tags = _get_page_tags(r["id"], db)
|
|
tags_html = ""
|
|
if tags:
|
|
tag_links = " ".join(f'<a href="/tags/{esc(t)}">[{esc(t)}]</a>' for t in tags)
|
|
tags_html = f' {tag_links}'
|
|
items += (
|
|
f'<li>{esc(r["title"])}{note_html}{tags_html} '
|
|
f'<small>(<a href="{esc(r["url"])}">{esc(r["url"])}</a>)</small> '
|
|
f'<a href="/edit/{r["id"]}">edit</a> '
|
|
f'<a href="/delete/{r["id"]}">remove</a></li>'
|
|
)
|
|
finally:
|
|
return_db(db)
|
|
return _respond(
|
|
f"<h1>indexed pages ({total})</h1>"
|
|
f"{msg_html}"
|
|
f"<ul>{items}</ul>"
|
|
f'{_page_nav(page, total, "/pages", BROWSE_PER_PAGE)}'
|
|
f'<p><a href="/export">export</a> | <a href="/import">import</a></p>'
|
|
f'<a href="/">back</a>'
|
|
)
|
|
|
|
|
|
def handle_edit_form(page_id, msg=""):
|
|
db = get_db()
|
|
try:
|
|
row = db.execute("SELECT id, url, title, body, note, summary FROM pages WHERE id = ?", (page_id,)).fetchone()
|
|
if not row:
|
|
return _error(404)
|
|
tags = ", ".join(_get_page_tags(page_id, db))
|
|
finally:
|
|
return_db(db)
|
|
|
|
return _respond(
|
|
f"<h1>edit page</h1>"
|
|
f"<p><b>{esc(row['title'])}</b><br>"
|
|
f"<small>{esc(row['url'])}</small></p>"
|
|
f'<form method="post" action="/edit/{row["id"]}">'
|
|
f'{_csrf_field()}'
|
|
f'<label>Title:</label><br>'
|
|
f'<input name="title" value="{esc(row["title"])}" size="60"><br><br>'
|
|
f'<label>Summary (shown in search results):</label><br>'
|
|
f'<textarea name="summary" rows="3" cols="60">{esc(row["summary"] or "")}</textarea><br><br>'
|
|
f'<label>Note (why you saved this):</label><br>'
|
|
f'<input name="note" value="{esc(row["note"])}" size="50"><br><br>'
|
|
f'<label>Tags (comma-separated):</label><br>'
|
|
f'<input name="tags" value="{esc(tags)}" size="50"><br><br>'
|
|
f'<button type="submit">save</button>'
|
|
f"</form>"
|
|
f"<p>{msg}</p>"
|
|
f'<a href="/pages">back</a>'
|
|
)
|
|
|
|
|
|
def handle_edit_submit(page_id, body):
|
|
title = body.get("title", [""])[0].strip()
|
|
summary = body.get("summary", [""])[0].strip()
|
|
note = body.get("note", [""])[0].strip()
|
|
tags = body.get("tags", [""])[0].strip()
|
|
|
|
db = get_db()
|
|
try:
|
|
db.execute(
|
|
"UPDATE pages SET title = ?, summary = ?, note = ? WHERE id = ?",
|
|
(title, summary, note, page_id)
|
|
)
|
|
|
|
_set_page_tags(page_id, tags, db)
|
|
|
|
db.commit()
|
|
|
|
finally:
|
|
return_db(db)
|
|
|
|
return _redirect("/pages")
|
|
|
|
|
|
def handle_delete_confirm(page_id):
|
|
db = get_db()
|
|
try:
|
|
row = db.execute("SELECT id, url, title FROM pages WHERE id = ?", (page_id,)).fetchone()
|
|
finally:
|
|
return_db(db)
|
|
if not row:
|
|
return _error(404)
|
|
return _respond(
|
|
f"<h1>confirm delete</h1>"
|
|
f"<p>Remove <b>{esc(row['title'])}</b><br>"
|
|
f"<small>{esc(row['url'])}</small></p>"
|
|
f'<form method="post" action="/delete/{row["id"]}">'
|
|
f'{_csrf_field()}'
|
|
f'<button type="submit">yes, delete</button>'
|
|
f"</form>"
|
|
f' <a href="/pages">cancel</a>'
|
|
)
|
|
|
|
|
|
def handle_delete(page_id):
|
|
db = get_db()
|
|
try:
|
|
db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,))
|
|
db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
|
|
db.execute("DELETE FROM pages WHERE id = ?", (page_id,))
|
|
db.commit()
|
|
finally:
|
|
return_db(db)
|
|
return _redirect("/pages")
|
|
|
|
|
|
def handle_bookmark(query):
|
|
token = query.get("token", [""])[0]
|
|
expected = _get_bookmark_token()
|
|
if not token or not secrets.compare_digest(token, expected):
|
|
return _text_response("error: invalid or missing token", status=403, headers={"Access-Control-Allow-Origin": "*"})
|
|
url = clean_url(query.get("url", [""])[0].strip())
|
|
if not url or not url.startswith(("http://", "https://")):
|
|
return _text_response("error: invalid url", headers={"Access-Control-Allow-Origin": "*"})
|
|
try:
|
|
title = index_url(url)
|
|
msg = f"ok: {title}"
|
|
except Exception as e:
|
|
msg = f"error: {e}"
|
|
return _text_response(msg, headers={"Access-Control-Allow-Origin": "*"})
|
|
|
|
|
|
def handle_export():
|
|
db = get_db()
|
|
try:
|
|
rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall()
|
|
finally:
|
|
return_db(db)
|
|
data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows]
|
|
return _json_response(data, headers={"Content-Disposition": "attachment; filename=tinyweb-export.json"})
|
|
|
|
|
|
def handle_import_form(msg=""):
|
|
return _respond(
|
|
f"<h1>import</h1>"
|
|
f"<p>Paste the contents of a tinyweb export file (JSON).</p>"
|
|
f'<form method="post" action="/import">'
|
|
f'{_csrf_field()}'
|
|
f'<textarea name="data" rows="12" cols="60" placeholder=\'[{{"url": "...", "note": "..."}}]\'></textarea><br><br>'
|
|
f'<button type="submit">import</button>'
|
|
f"</form>"
|
|
f"<p>{msg}</p>"
|
|
f'<a href="/pages">back</a>'
|
|
)
|
|
|
|
|
|
def handle_import_submit(body):
|
|
raw = body.get("data", [""])[0].strip()
|
|
if not raw:
|
|
return handle_import_form("Paste JSON data.")
|
|
try:
|
|
data = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
return handle_import_form("Invalid JSON.")
|
|
if not isinstance(data, list):
|
|
return handle_import_form("Expected a JSON array.")
|
|
|
|
MAX_IMPORT = 100
|
|
if len(data) > MAX_IMPORT:
|
|
return handle_import_form(f"Too many entries. Maximum is {MAX_IMPORT}.")
|
|
|
|
imported = 0
|
|
errors = 0
|
|
for entry in data:
|
|
url = entry.get("url", "").strip()
|
|
note = entry.get("note", "").strip()
|
|
if not url:
|
|
continue
|
|
try:
|
|
index_url(url, note)
|
|
imported += 1
|
|
except Exception:
|
|
errors += 1
|
|
|
|
return handle_import_form(f"Imported {imported} page(s). {errors} error(s).")
|
|
|
|
|
|
def handle_style_form(msg=""):
|
|
template = get_setting("custom_template") or DEFAULT_TEMPLATE
|
|
name = get_site_name()
|
|
sharing = get_setting("sharing_enabled", "0")
|
|
checked = " checked" if sharing == "1" else ""
|
|
semantic = get_setting("semantic_search", "1")
|
|
semantic_checked = " checked" if semantic == "1" else ""
|
|
reranker = get_setting("use_reranker", "1")
|
|
reranker_checked = " checked" if reranker == "1" else ""
|
|
disabled = "" if semantic == "1" else " disabled"
|
|
dimmed = ' style="opacity:0.4"' if semantic != "1" else ""
|
|
return _respond(
|
|
f"<h1>customize</h1>"
|
|
f"<h2>name your search engine</h2>"
|
|
f'<form method="post" action="/style">'
|
|
f'{_csrf_field()}'
|
|
f'<input name="site_name" value="{esc(name)}" placeholder="tinyweb" size="30"><br><br>'
|
|
f"<h2>sharing</h2>"
|
|
f'<label><input type="checkbox" name="sharing_enabled" value="1"{checked}>'
|
|
f" share your site list publicly at /api/sites</label><br><br>"
|
|
f"<h2>search</h2>"
|
|
f"<h3>ai</h3>"
|
|
f'<label><input type="checkbox" name="semantic_search" value="1"{semantic_checked} '
|
|
f'onchange="var d=!this.checked;document.getElementById(\'reranker\').disabled=d;'
|
|
f'document.getElementById(\'ai-extras\').style.opacity=d?\'0.4\':\'1\'">'
|
|
f" semantic search (similarity matching)</label><br>"
|
|
f"<small>Requires onnxruntime, tokenizers, hnswlib. Downloads ~30MB of models on first use.</small><br><br>"
|
|
f'<div id="ai-extras"{dimmed}>'
|
|
f'<label><input type="checkbox" id="reranker" name="use_reranker" value="1"{reranker_checked}{disabled}>'
|
|
f" cross-encoder reranking (more accurate)</label><br>"
|
|
f"<small>Uses a 22MB model. Adds ~50ms per search. Disable for faster results.</small><br><br>"
|
|
f'<a href="/reindex">manage semantic index</a><br><br>'
|
|
f"</div>"
|
|
f"<h2>custom html</h2>"
|
|
f"<p>Edit the full page template. Use <code>{esc('{{content}}')}</code> "
|
|
f"where page content should appear.</p>"
|
|
f'<textarea name="template" rows="20" cols="60">{esc(template)}</textarea><br><br>'
|
|
f'<button type="submit">save</button>'
|
|
f"</form>"
|
|
f"<h2>bookmarklet</h2>"
|
|
f"<p>Drag this link to your bookmarks bar. Click it on any page to index it instantly.</p>"
|
|
f'<p><a href="javascript:void(fetch(\'http://localhost:8080/bookmark?url=\'+encodeURIComponent(location.href)+\'&token={_get_bookmark_token()}\').then(r=>r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}</a></p>'
|
|
f"<h2>reset</h2>"
|
|
f'<form method="post" action="/style/reset">'
|
|
f'{_csrf_field()}'
|
|
f'<button type="submit">reset template to default</button>'
|
|
f"</form>"
|
|
f"<p>{msg}</p>"
|
|
f'<a href="/">back</a>',
|
|
use_default=True,
|
|
)
|
|
|
|
|
|
def handle_style_submit(body):
|
|
template = body.get("template", [""])[0].replace("\r\n", "\n").replace("\r", "\n")
|
|
name = body.get("site_name", ["tinyweb"])[0].strip()
|
|
sharing = "1" if body.get("sharing_enabled") else "0"
|
|
semantic = "1" if body.get("semantic_search") else "0"
|
|
reranker = "1" if body.get("use_reranker") else "0"
|
|
set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "")
|
|
set_setting("site_name", name or "tinyweb")
|
|
set_setting("sharing_enabled", sharing)
|
|
set_setting("semantic_search", semantic)
|
|
set_setting("use_reranker", reranker)
|
|
return handle_style_form("Saved.")
|
|
|
|
|
|
def handle_about():
|
|
name = get_site_name()
|
|
dest_hash = get_setting("dest_hash")
|
|
sharing = get_setting("sharing_enabled", "0") == "1"
|
|
db = get_db()
|
|
try:
|
|
page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0]
|
|
tag_count = db.execute("SELECT count(DISTINCT tag_id) FROM page_tags").fetchone()[0]
|
|
sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0]
|
|
finally:
|
|
return_db(db)
|
|
|
|
sharing_html = (
|
|
'<p>This instance shares its index publicly. Subscribe to join the network.</p>'
|
|
if sharing else
|
|
'<p>This instance is private.</p>'
|
|
)
|
|
|
|
hash_html = ""
|
|
if dest_hash:
|
|
hash_html = (
|
|
f'<h2>subscribe</h2>'
|
|
f'<p>To subscribe to this instance, add this destination hash in your TinyWeb:</p>'
|
|
f'<pre>{esc(dest_hash)}</pre>'
|
|
)
|
|
|
|
return _respond(
|
|
f'<h1>{esc(name)}</h1>'
|
|
f'<p>A personal search engine, built for the slow web.</p>'
|
|
f'<p>TinyWeb is about taking back the internet. No algorithms, no ads, no tracking. '
|
|
f'Just human-curated pages shared freely across a mesh network.</p>'
|
|
f'<ul>'
|
|
f'<li><b>{page_count}</b> page(s) indexed</li>'
|
|
f'<li><b>{tag_count}</b> tag(s)</li>'
|
|
f'<li><b>{sub_count}</b> subscription(s)</li>'
|
|
f'</ul>'
|
|
f'{sharing_html}'
|
|
f'{hash_html}'
|
|
f'<h2>what is the slow web?</h2>'
|
|
f'<p>The slow web is a movement for intentionality over speed, '
|
|
f'human curation over algorithmic feeds, privacy over surveillance, '
|
|
f'and community over corporations. Every page in this index was saved by a person '
|
|
f'because they found it valuable — not because an algorithm told them to click.</p>'
|
|
f'<h2>how it works</h2>'
|
|
f'<ul>'
|
|
f'<li>Save pages you find valuable with the bookmarklet or /add</li>'
|
|
f'<li>Search your personal index — queries never leave your machine</li>'
|
|
f'<li>Subscribe to friends over Reticulum — encrypted, decentralized, works without the internet</li>'
|
|
f'<li>Tag and organize your collection into curated lists</li>'
|
|
f'</ul>'
|
|
f'<p><a href="/">search</a> | <a href="/pages">browse</a> | <a href="/tags">tags</a></p>'
|
|
)
|
|
|
|
|
|
def handle_tags():
|
|
db = get_db()
|
|
try:
|
|
rows = db.execute(
|
|
"SELECT t.name, COUNT(pt.page_id) AS cnt FROM tags t "
|
|
"JOIN page_tags pt ON t.id = pt.tag_id "
|
|
"GROUP BY t.id ORDER BY t.name"
|
|
).fetchall()
|
|
finally:
|
|
return_db(db)
|
|
items = ""
|
|
for r in rows:
|
|
items += f'<li><a href="/tags/{esc(r["name"])}">{esc(r["name"])}</a> ({r["cnt"]})</li>'
|
|
return _respond(
|
|
f"<h1>tags</h1>"
|
|
f"<ul>{items}</ul>" if items else "<p>No tags yet. Add tags when saving or editing pages.</p>"
|
|
f'<a href="/">back</a>'
|
|
)
|
|
|
|
|
|
def handle_tag_browse(tag_name, query=None):
|
|
page = _paginate(query or {})
|
|
offset = (page - 1) * BROWSE_PER_PAGE
|
|
db = get_db()
|
|
try:
|
|
total = db.execute(
|
|
"SELECT count(*) FROM page_tags pt JOIN tags t ON t.id = pt.tag_id WHERE t.name = ?",
|
|
(tag_name,),
|
|
).fetchone()[0]
|
|
rows = db.execute(
|
|
"SELECT p.id, p.url, p.title, p.note FROM pages p "
|
|
"JOIN page_tags pt ON p.id = pt.page_id "
|
|
"JOIN tags t ON t.id = pt.tag_id "
|
|
"WHERE t.name = ? ORDER BY p.id DESC LIMIT ? OFFSET ?",
|
|
(tag_name, BROWSE_PER_PAGE, offset),
|
|
).fetchall()
|
|
items = ""
|
|
for r in rows:
|
|
note_html = f' — <em>{esc(r["note"])}</em>' if r["note"] else ""
|
|
tags = _get_page_tags(r["id"], db)
|
|
tag_links = " ".join(f'<a href="/tags/{esc(t)}">[{esc(t)}]</a>' for t in tags)
|
|
items += (
|
|
f'<li>{esc(r["title"])}{note_html} {tag_links} '
|
|
f'<small>(<a href="{esc(r["url"])}">{esc(r["url"])}</a>)</small></li>'
|
|
)
|
|
finally:
|
|
return_db(db)
|
|
return _respond(
|
|
f'<h1>tag: {esc(tag_name)}</h1>'
|
|
f'<p>{total} page(s)</p>'
|
|
f'<ul>{items}</ul>'
|
|
f'{_page_nav(page, total, f"/tags/{esc(tag_name)}", BROWSE_PER_PAGE)}'
|
|
f'<a href="/tags">all tags</a> | <a href="/">back</a>'
|
|
)
|
|
|
|
|
|
def handle_api_sites(query=None):
|
|
if get_setting("sharing_enabled", "0") != "1":
|
|
return _json_response(
|
|
{"error": "sharing disabled"},
|
|
status=403,
|
|
headers={"Access-Control-Allow-Origin": "*"},
|
|
)
|
|
since = (query or {}).get("since", [""])[0].strip()
|
|
db = get_db()
|
|
try:
|
|
if since:
|
|
rows = db.execute(
|
|
"SELECT id, url, title, note, last_modified FROM pages "
|
|
"WHERE last_modified > ? ORDER BY id DESC",
|
|
(since,),
|
|
).fetchall()
|
|
else:
|
|
rows = db.execute("SELECT id, url, title, note, last_modified FROM pages ORDER BY id DESC").fetchall()
|
|
sites = []
|
|
for r in rows:
|
|
tags = _get_page_tags(r["id"], db)
|
|
sites.append({
|
|
"url": r["url"], "title": r["title"], "note": r["note"],
|
|
"tags": tags, "last_modified": r["last_modified"] or "",
|
|
})
|
|
# Include list of all current URLs so subscriber can detect deletions
|
|
all_urls = [r["url"] for r in db.execute("SELECT url FROM pages").fetchall()] if not since else None
|
|
finally:
|
|
return_db(db)
|
|
data = {"name": get_site_name(), "sites": sites}
|
|
if all_urls is not None:
|
|
data["all_urls"] = all_urls
|
|
return _json_response(data, headers={"Access-Control-Allow-Origin": "*"})
|
|
|
|
|
|
def handle_subscriptions(msg=""):
|
|
db = get_db()
|
|
try:
|
|
subs = db.execute("SELECT * FROM subscriptions ORDER BY id DESC").fetchall()
|
|
finally:
|
|
return_db(db)
|
|
cards = ""
|
|
for s in subs:
|
|
auto_label = "on" if s["auto_sync"] else "off"
|
|
last = s["last_sync"] or "never"
|
|
cards += (
|
|
f'<div style="border:1px solid #ddd;border-radius:4px;padding:0.9rem 1rem;margin-bottom:0.75rem">'
|
|
f'<div style="margin-bottom:0.4rem"><b>{esc(s["name"] or "unknown")}</b></div>'
|
|
f'<div><small>{esc(s["dest_hash"])}</small></div>'
|
|
f'<div style="margin-top:0.4rem;font-size:0.85rem;color:#606060">last sync: {esc(last)}</div>'
|
|
f'<div style="display:flex;gap:0.5rem;align-items:center;flex-wrap:wrap;margin-top:0.7rem">'
|
|
f'<a href="/subscriptions/browse/{s["id"]}">browse</a>'
|
|
f'<form method="post" action="/subscriptions/sync/{s["id"]}" style="display:inline">'
|
|
f'{_csrf_field()}<button>sync now</button></form>'
|
|
f'<form method="post" action="/subscriptions/autosync/{s["id"]}" style="display:inline">'
|
|
f'{_csrf_field()}<button>auto-sync: {auto_label}</button></form>'
|
|
f'<form method="post" action="/subscriptions/delete/{s["id"]}" style="display:inline">'
|
|
f'{_csrf_field()}<button>remove</button></form>'
|
|
f'</div>'
|
|
f'</div>'
|
|
)
|
|
listing = ""
|
|
if subs:
|
|
listing = (
|
|
f'{cards}'
|
|
f'<form method="post" action="/subscriptions/syncall">'
|
|
f'{_csrf_field()}<button>sync all</button></form>'
|
|
)
|
|
return _respond(
|
|
f"<h1>subscriptions</h1>"
|
|
f'<form method="post" action="/subscriptions/add">'
|
|
f'{_csrf_field()}'
|
|
f'<input name="dest_hash" placeholder="destination hash" size="40"> '
|
|
f'<button>subscribe</button>'
|
|
f'</form>'
|
|
f'<p><small>or <a href="/add?type=subscribe">subscribe to an instance</a></small></p>'
|
|
f'<p>{msg}</p>'
|
|
f'<hr>{listing}'
|
|
f'<br><a href="/">back</a>'
|
|
)
|
|
|
|
|
|
def handle_subscription_add(body):
|
|
dest_hash = body.get("dest_hash", [""])[0].strip().replace("<", "").replace(">", "")
|
|
if not dest_hash or len(dest_hash) != 32:
|
|
return handle_subscriptions("Enter a valid 32-character destination hash.")
|
|
try:
|
|
int(dest_hash, 16)
|
|
except ValueError:
|
|
return handle_subscriptions("Invalid destination hash (must be hex).")
|
|
try:
|
|
data = fetch_remote_sites(dest_hash)
|
|
name = data.get("name", "")
|
|
except PermissionError:
|
|
return handle_subscriptions("That instance has sharing disabled.")
|
|
except Exception:
|
|
return handle_subscriptions("Could not reach that instance.")
|
|
db = get_db()
|
|
try:
|
|
db.execute(
|
|
"INSERT INTO subscriptions (dest_hash, name) VALUES (?, ?) "
|
|
"ON CONFLICT(dest_hash) DO UPDATE SET name=excluded.name",
|
|
(dest_hash, name),
|
|
)
|
|
db.commit()
|
|
finally:
|
|
return_db(db)
|
|
return handle_subscriptions(f"Subscribed to {esc(name or dest_hash)}.")
|
|
|
|
|
|
def handle_subscription_browse(sub_id):
|
|
db = get_db()
|
|
try:
|
|
sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone()
|
|
if not sub:
|
|
return _error(404)
|
|
local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall())
|
|
|
|
# Use locally synced data if available, otherwise fetch live
|
|
remote_rows = db.execute(
|
|
"SELECT url, title, note, tags FROM remote_pages WHERE subscription_id = ?",
|
|
(sub_id,),
|
|
).fetchall()
|
|
finally:
|
|
return_db(db)
|
|
|
|
if remote_rows:
|
|
sites = []
|
|
for r in remote_rows:
|
|
tags = [t for t in r["tags"].split(",") if t] if r["tags"] else []
|
|
sites.append({"url": r["url"], "title": r["title"], "note": r["note"], "tags": tags})
|
|
else:
|
|
try:
|
|
data = fetch_remote_sites(sub["dest_hash"])
|
|
sites = data.get("sites", [])
|
|
except PermissionError:
|
|
return handle_subscriptions("That instance has sharing disabled.")
|
|
except Exception:
|
|
return handle_subscriptions("Could not fetch sites from that instance.")
|
|
|
|
new_items = ""
|
|
existing_items = ""
|
|
new_count = 0
|
|
for s in sites:
|
|
if s["url"] in local_urls:
|
|
existing_items += (
|
|
f'<li style="opacity:0.5">{esc(s["title"])} '
|
|
f'<small>({esc(s["url"])})</small> — already indexed</li>'
|
|
)
|
|
else:
|
|
new_count += 1
|
|
note_html = f' — <em>{esc(s["note"])}</em>' if s.get("note") else ""
|
|
tags_html = ""
|
|
if s.get("tags"):
|
|
tags_html = " " + " ".join(f'[{esc(t)}]' for t in s["tags"])
|
|
new_items += (
|
|
f'<li><label><input type="checkbox" name="urls" value="{esc(s["url"])}">'
|
|
f' {esc(s["title"])}{note_html}{tags_html}'
|
|
f' <small>({esc(s["url"])})</small></label></li>'
|
|
)
|
|
|
|
buttons = ""
|
|
if new_count:
|
|
buttons = '<button>import selected</button> <button name="import_all" value="1">import all new</button>'
|
|
return _respond(
|
|
f'<h1>browsing: {esc(sub["name"] or sub["dest_hash"])}</h1>'
|
|
f'<p>{len(sites)} site(s) available, {new_count} new</p>'
|
|
f'<form method="post" action="/subscriptions/pick">'
|
|
f'{_csrf_field()}'
|
|
f'<input type="hidden" name="sub_id" value="{sub_id}">'
|
|
f'<ul>{new_items}</ul>'
|
|
f'{buttons}'
|
|
f'</form>'
|
|
f'<h3>already indexed</h3><ul>{existing_items}</ul>'
|
|
f'<a href="/subscriptions">back</a>'
|
|
)
|
|
|
|
|
|
def handle_subscription_pick(body):
|
|
sub_id = body.get("sub_id", [""])[0]
|
|
import_all = body.get("import_all", [""])[0]
|
|
|
|
# Build a url->tags map from remote_pages for this subscription
|
|
db = get_db()
|
|
try:
|
|
remote_rows = db.execute(
|
|
"SELECT url, tags FROM remote_pages WHERE subscription_id = ?", (sub_id,)
|
|
).fetchall()
|
|
remote_tags = {r["url"]: r["tags"] for r in remote_rows}
|
|
|
|
if import_all:
|
|
local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall())
|
|
urls = [r["url"] for r in remote_rows if r["url"] not in local_urls]
|
|
else:
|
|
urls = body.get("urls", [])
|
|
finally:
|
|
return_db(db)
|
|
|
|
if not urls:
|
|
return handle_subscriptions("No sites selected.")
|
|
|
|
imported = 0
|
|
errors = 0
|
|
for url in urls:
|
|
try:
|
|
index_url(url)
|
|
# Import tags from the remote page
|
|
tags_str = remote_tags.get(url, "")
|
|
if tags_str:
|
|
db = get_db()
|
|
try:
|
|
row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()
|
|
if row:
|
|
_set_page_tags(row["id"], tags_str, db)
|
|
db.commit()
|
|
finally:
|
|
return_db(db)
|
|
imported += 1
|
|
except Exception:
|
|
errors += 1
|
|
return handle_subscriptions(f"Imported {imported} page(s). {errors} error(s).")
|
|
|
|
|
|
def handle_subscription_sync(sub_id):
|
|
db = get_db()
|
|
try:
|
|
sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone()
|
|
if not sub:
|
|
return handle_subscriptions("Subscription not found.")
|
|
# Use last_sync for delta sync if available
|
|
since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else ""
|
|
try:
|
|
data = fetch_remote_sites(sub["dest_hash"], since=since)
|
|
sites = data.get("sites", [])
|
|
all_urls = data.get("all_urls")
|
|
remote_name = data.get("name", sub["name"])
|
|
except PermissionError:
|
|
return handle_subscriptions("That instance has sharing disabled.")
|
|
except Exception:
|
|
return handle_subscriptions("Could not sync with that instance.")
|
|
|
|
# If full sync (all_urls provided), remove pages no longer on remote
|
|
if all_urls is not None:
|
|
existing = db.execute(
|
|
"SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub_id,)
|
|
).fetchall()
|
|
remote_url_set = set(all_urls)
|
|
for row in existing:
|
|
if row["url"] not in remote_url_set:
|
|
db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],))
|
|
|
|
# Upsert changed/new pages
|
|
synced = 0
|
|
for s in sites:
|
|
try:
|
|
tags_str = ",".join(s.get("tags", []))
|
|
db.execute(
|
|
"INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?) "
|
|
"ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags",
|
|
(sub_id, s["url"], s["title"], s.get("note", ""), tags_str),
|
|
)
|
|
# Embed remote page for semantic search
|
|
if get_setting("semantic_search", "1") == "1":
|
|
try:
|
|
from embeddings import store_remote_embeddings
|
|
rp_id = db.execute(
|
|
"SELECT id FROM remote_pages WHERE subscription_id = ? AND url = ?",
|
|
(sub_id, s["url"]),
|
|
).fetchone()["id"]
|
|
store_remote_embeddings(rp_id, s["title"], s.get("note", ""), db)
|
|
except Exception:
|
|
pass
|
|
synced += 1
|
|
except Exception:
|
|
pass
|
|
now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
|
db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id))
|
|
db.commit()
|
|
finally:
|
|
return_db(db)
|
|
return handle_subscriptions(f"Synced {synced} site(s) from {esc(remote_name)}.")
|
|
|
|
|
|
def handle_subscription_autosync(sub_id):
|
|
db = get_db()
|
|
try:
|
|
db.execute("UPDATE subscriptions SET auto_sync = 1 - auto_sync WHERE id = ?", (sub_id,))
|
|
db.commit()
|
|
finally:
|
|
return_db(db)
|
|
return _redirect("/subscriptions")
|
|
|
|
|
|
def handle_subscription_delete(sub_id):
|
|
db = get_db()
|
|
try:
|
|
db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub_id,))
|
|
db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,))
|
|
db.commit()
|
|
finally:
|
|
return_db(db)
|
|
return _redirect("/subscriptions")
|
|
|
|
|
|
def handle_subscription_syncall():
|
|
db = get_db()
|
|
try:
|
|
subs = db.execute("SELECT * FROM subscriptions WHERE auto_sync = 1").fetchall()
|
|
finally:
|
|
return_db(db)
|
|
if not subs:
|
|
return handle_subscriptions("No subscriptions have auto-sync enabled.")
|
|
total = 0
|
|
for sub in subs:
|
|
try:
|
|
since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else ""
|
|
data = fetch_remote_sites(sub["dest_hash"], since=since)
|
|
sites = data.get("sites", [])
|
|
all_urls = data.get("all_urls")
|
|
remote_name = data.get("name", sub["name"])
|
|
db = get_db()
|
|
try:
|
|
if all_urls is not None:
|
|
existing = db.execute(
|
|
"SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub["id"],)
|
|
).fetchall()
|
|
remote_url_set = set(all_urls)
|
|
for row in existing:
|
|
if row["url"] not in remote_url_set:
|
|
db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],))
|
|
for s in sites:
|
|
try:
|
|
tags_str = ",".join(s.get("tags", []))
|
|
db.execute(
|
|
"INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?) "
|
|
"ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags",
|
|
(sub["id"], s["url"], s["title"], s.get("note", ""), tags_str),
|
|
)
|
|
if get_setting("semantic_search", "1") == "1":
|
|
try:
|
|
from embeddings import store_remote_embeddings
|
|
rp_id = db.execute(
|
|
"SELECT id FROM remote_pages WHERE subscription_id = ? AND url = ?",
|
|
(sub["id"], s["url"]),
|
|
).fetchone()["id"]
|
|
store_remote_embeddings(rp_id, s["title"], s.get("note", ""), db)
|
|
except Exception:
|
|
pass
|
|
except Exception:
|
|
pass
|
|
now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
|
|
db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub["id"]))
|
|
db.commit()
|
|
finally:
|
|
return_db(db)
|
|
total += 1
|
|
except Exception:
|
|
pass
|
|
return handle_subscriptions(f"Synced {total} subscription(s).")
|
|
|
|
|
|
# --- Reindex (semantic search) ---
|
|
|
|
|
|
_reindex_thread = None
|
|
|
|
|
|
def handle_reindex_form():
|
|
if get_setting("semantic_search", "1") != "1":
|
|
return _respond(
|
|
f"<h2>semantic search index</h2>"
|
|
f"<p>Semantic search is disabled. Enable it in <a href=\"/style\">settings</a> to use embeddings.</p>"
|
|
f'<p><a href="/">back to search</a></p>'
|
|
)
|
|
db = get_db()
|
|
try:
|
|
total_pages = db.execute("SELECT count(*) FROM pages").fetchone()[0]
|
|
pages_with_chunks = db.execute(
|
|
"SELECT count(DISTINCT page_id) FROM chunks WHERE page_id IS NOT NULL"
|
|
).fetchone()[0]
|
|
finally:
|
|
return_db(db)
|
|
progress = get_setting("reindex_progress", "")
|
|
status_html = ""
|
|
if progress:
|
|
status_html = f'<p class="meta">Reindex in progress: {esc(progress)}</p>'
|
|
elif _reindex_thread and _reindex_thread.is_alive():
|
|
status_html = '<p class="meta">Reindex running...</p>'
|
|
return _respond(
|
|
f"<h2>semantic search index</h2>"
|
|
f"<p>{pages_with_chunks} of {total_pages} pages have embeddings.</p>"
|
|
f'{status_html}'
|
|
f'<form method="post" action="/reindex">'
|
|
f'{_csrf_field()}'
|
|
f'<button type="submit">reindex all pages</button>'
|
|
f'</form>'
|
|
f'<p><a href="/">back to search</a></p>'
|
|
)
|
|
|
|
|
|
def handle_reindex_submit(body):
|
|
global _reindex_thread
|
|
if _reindex_thread and _reindex_thread.is_alive():
|
|
return handle_reindex_form()
|
|
|
|
def _run():
|
|
try:
|
|
from embeddings import reindex_all
|
|
def progress(current, total):
|
|
set_setting("reindex_progress", f"{current}/{total}")
|
|
reindex_all(progress_callback=progress)
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
set_setting("reindex_progress", "")
|
|
|
|
_reindex_thread = threading.Thread(target=_run, daemon=True)
|
|
_reindex_thread.start()
|
|
return _redirect("/reindex")
|
|
|
|
|
|
# --- Dispatcher ---
|
|
|
|
|
|
def _dispatch_inner(data):
|
|
method = data.get("method", "GET")
|
|
path = data.get("path", "/")
|
|
query = data.get("query", {})
|
|
body = data.get("body", {})
|
|
gateway_host = data.get("gateway_host", "")
|
|
|
|
def extract_id(prefix):
|
|
try:
|
|
return int(path[len(prefix):])
|
|
except (ValueError, IndexError):
|
|
return None
|
|
|
|
if method == "GET":
|
|
if path == "/":
|
|
return handle_search(query)
|
|
elif path == "/add":
|
|
action_type = query.get("type", ["index"])[0]
|
|
return handle_add_form(action_type=action_type if action_type == "subscribe" else "index")
|
|
elif path == "/pages":
|
|
return handle_pages(query)
|
|
elif path.startswith("/edit/"):
|
|
pid = extract_id("/edit/")
|
|
return handle_edit_form(pid) if pid is not None else _error(400)
|
|
elif path.startswith("/delete/"):
|
|
pid = extract_id("/delete/")
|
|
return handle_delete_confirm(pid) if pid is not None else _error(400)
|
|
elif path == "/bookmark":
|
|
return handle_bookmark(query)
|
|
elif path == "/style":
|
|
return handle_style_form()
|
|
elif path == "/about":
|
|
return handle_about()
|
|
elif path == "/export":
|
|
return handle_export()
|
|
elif path == "/import":
|
|
return handle_import_form()
|
|
elif path == "/tags":
|
|
return handle_tags()
|
|
elif path.startswith("/tags/"):
|
|
tag_name = unquote(path[len("/tags/"):])
|
|
return handle_tag_browse(tag_name, query) if tag_name else _error(400)
|
|
elif path == "/reindex":
|
|
return handle_reindex_form()
|
|
elif path == "/api/sites":
|
|
return handle_api_sites(query)
|
|
elif path == "/subscriptions":
|
|
return handle_subscriptions()
|
|
elif path.startswith("/subscriptions/browse/"):
|
|
sid = extract_id("/subscriptions/browse/")
|
|
return handle_subscription_browse(sid) if sid is not None else _error(400)
|
|
elif method == "POST":
|
|
if not _check_csrf(body):
|
|
return _respond("<h1>403 Forbidden</h1><p>Invalid or missing CSRF token.</p>", status=403)
|
|
if path == "/add":
|
|
return handle_add_submit(body)
|
|
elif path == "/add/manual":
|
|
return handle_add_manual_submit(body)
|
|
elif path.startswith("/edit/"):
|
|
pid = extract_id("/edit/")
|
|
return handle_edit_submit(pid, body) if pid is not None else _error(400)
|
|
elif path.startswith("/delete/"):
|
|
pid = extract_id("/delete/")
|
|
return handle_delete(pid) if pid is not None else _error(400)
|
|
elif path == "/style":
|
|
return handle_style_submit(body)
|
|
elif path == "/style/reset":
|
|
set_setting("custom_template", "")
|
|
return handle_style_form("Template reset to default.")
|
|
elif path == "/import":
|
|
return handle_import_submit(body)
|
|
elif path == "/reindex":
|
|
return handle_reindex_submit(body)
|
|
elif path == "/subscriptions/add":
|
|
return handle_subscription_add(body)
|
|
elif path == "/subscriptions/pick":
|
|
return handle_subscription_pick(body)
|
|
elif path.startswith("/subscriptions/sync/"):
|
|
sid = extract_id("/subscriptions/sync/")
|
|
return handle_subscription_sync(sid) if sid is not None else _error(400)
|
|
elif path.startswith("/subscriptions/autosync/"):
|
|
sid = extract_id("/subscriptions/autosync/")
|
|
return handle_subscription_autosync(sid) if sid is not None else _error(400)
|
|
elif path.startswith("/subscriptions/delete/"):
|
|
sid = extract_id("/subscriptions/delete/")
|
|
return handle_subscription_delete(sid) if sid is not None else _error(400)
|
|
elif path == "/subscriptions/syncall":
|
|
return handle_subscription_syncall()
|
|
|
|
return _error(404)
|
|
|
|
|
|
def dispatch_request(data):
|
|
cookies = data.get("cookies", {})
|
|
csrf_token = cookies.get("_csrf", "")
|
|
if not csrf_token:
|
|
csrf_token = secrets.token_hex(32)
|
|
_request_local.csrf_token = csrf_token
|
|
|
|
resp = _dispatch_inner(data)
|
|
|
|
resp.setdefault("headers", {})
|
|
resp["headers"]["Set-Cookie"] = f"_csrf={csrf_token}; SameSite=Strict; HttpOnly; Path=/"
|
|
resp["headers"]["X-Frame-Options"] = "DENY"
|
|
resp["headers"]["X-Content-Type-Options"] = "nosniff"
|
|
if resp.get("content_type", "").startswith("text/html"):
|
|
resp["headers"]["Content-Security-Policy"] = (
|
|
"default-src 'self'; "
|
|
"script-src 'self' 'unsafe-inline'; "
|
|
"style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; "
|
|
"font-src 'self' https://fonts.gstatic.com; "
|
|
"img-src * data:; "
|
|
"frame-ancestors 'none'; "
|
|
"form-action 'self'; "
|
|
"base-uri 'self'"
|
|
)
|
|
return resp
|