tinyweb/handlers.py

import json
import re
import secrets
import threading
from datetime import datetime
from urllib.parse import unquote

from db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url
from templates import esc, wrap_page, DEFAULT_TEMPLATE
from rns_client import fetch_remote_sites

_request_local = threading.local()


def _get_csrf_token():
    return getattr(_request_local, 'csrf_token', '')


def _csrf_field():
    return f'<input type="hidden" name="_csrf" value="{_get_csrf_token()}">'


def _check_csrf(body):
    token = body.get("_csrf", [""])[0]
    expected = _get_csrf_token()
    if not expected or not token:
        return False
    return secrets.compare_digest(token, expected)


_STOPWORDS = frozenset({
    "a", "an", "the", "and", "or", "but", "is", "are", "was", "were",
    "in", "on", "at", "to", "for", "of", "with", "by", "from", "as",
    "into", "about", "how", "what", "which", "who", "where", "when",
    "do", "does", "did", "be", "been", "being", "have", "has", "had",
    "it", "its", "this", "that", "not", "no", "so", "if", "can", "will",
    "my", "your", "i", "me", "we", "you", "he", "she", "they",
})


def _sanitize_fts_query(query):
    """Escape user input for safe use in FTS5 MATCH.

    Splits into individual quoted tokens joined by implicit AND,
    so all words must appear but in any order. Appends * to the
    last token for prefix matching. Stopwords are dropped to avoid
    overly strict matching.
    """
    words = query.split()
    if not words:
        return '""'
    tokens = []
    last_idx = len(words) - 1
    for i, w in enumerate(words):
        # Strip FTS5 special characters (operators, column filter colon) to prevent injection
        cleaned = re.sub(r'["\'\(\)\*\+\-\^~:]', '', w).strip()
        if not cleaned:
            continue
        if cleaned.lower() in _STOPWORDS:
            continue
        # Drop FTS5 operator words so they aren't parsed as operators on the unquoted last token
        if cleaned.upper() in ("AND", "OR", "NOT", "NEAR"):
            continue
        if i == last_idx:
            # Prefix match on the last token for partial word matching
            tokens.append(f"{cleaned}*")
        else:
            tokens.append(f'"{cleaned}"')
    return " ".join(tokens) if tokens else '""'


def _get_bookmark_token():
    token = get_setting("bookmark_token")
    if not token:
        token = secrets.token_hex(16)
        set_setting("bookmark_token", token)
    return token


def _respond(body_html, status=200, use_default=False):
    return {
        "status": status,
        "content_type": "text/html; charset=utf-8",
        "body": wrap_page(body_html, use_default=use_default),
        "headers": {},
    }


def _redirect(location):
    if not location.startswith("/") or location.startswith("//"):
        location = "/"
    return {
        "status": 302,
        "content_type": "text/html; charset=utf-8",
        "body": "",
        "headers": {"Location": location},
    }


def _json_response(data, status=200, headers=None):
    return {
        "status": status,
        "content_type": "application/json",
        "body": json.dumps(data, indent=2),
        "headers": headers or {},
    }


def _text_response(text, status=200, headers=None):
    return {
        "status": status,
        "content_type": "text/plain",
        "body": text,
        "headers": headers or {},
    }


def _error(status):
    return _respond(f"<h1>{status}</h1>", status)


PER_PAGE = 10
BROWSE_PER_PAGE = 50


def _paginate(query, key="p"):
    try:
        page = int(query.get(key, ["1"])[0])
    except (ValueError, IndexError):
        page = 1
    return max(1, page)


def _page_nav(page, total, base_url, per_page=None):
    per_page = per_page or PER_PAGE
    if total <= per_page:
        return ""
    total_pages = (total + per_page - 1) // per_page
    sep = "&" if "?" in base_url else "?"
    parts = []
    if page > 1:
        parts.append(f'<a href="{base_url}{sep}p={page - 1}">&laquo; prev</a>')
    parts.append(f"page {page} of {total_pages}")
    if page < total_pages:
        parts.append(f'<a href="{base_url}{sep}p={page + 1}">next &raquo;</a>')
    return f'<p class="pagination">{" | ".join(parts)}</p>'


# --- Tag helpers ---


def _get_page_tags(page_id, db=None):
    close = False
    if db is None:
        db = get_db()
        close = True
    rows = db.execute(
        "SELECT t.name FROM tags t JOIN page_tags pt ON t.id = pt.tag_id "
        "WHERE pt.page_id = ? ORDER BY t.name", (page_id,)
    ).fetchall()
    if close:
        return_db(db)
    return [r["name"] for r in rows]


def _set_page_tags(page_id, tag_string, db=None):
    close = False
    if db is None:
        db = get_db()
        close = True
    db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,))
    for name in (t.strip().lower() for t in tag_string.split(",") if t.strip()):
        db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,))
        tag_id = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()["id"]
        db.execute("INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", (page_id, tag_id))
    if close:
        db.commit()
        return_db(db)


def _cleanup_orphaned_tags(db):
    """Delete tags that have no page associations."""
    db.execute("DELETE FROM tags WHERE id NOT IN (SELECT DISTINCT tag_id FROM page_tags)")


# --- Route handlers ---


def handle_search(query):
    q = query.get("q", [""])[0].strip()
    page = _paginate(query)
    offset = (page - 1) * PER_PAGE
    db = get_db()
    try:
        count = db.execute("SELECT count(*) FROM pages").fetchone()[0]
        name = get_site_name()

        result_html = ""
        trusted_html = ""
        if q:
            # BM25 keyword search with column weights: title=10, body=1, url=5, note=3
            try:
                fts_q = _sanitize_fts_query(q)
                bm25_rows = db.execute(
                    "SELECT p.id, p.url, p.title, p.body, p.note "
                    "FROM pages_fts f JOIN pages p ON f.rowid = p.id "
                    "WHERE pages_fts MATCH ? "
                    "ORDER BY bm25(pages_fts, 10.0, 1.0, 5.0, 3.0) LIMIT 100",
                    (fts_q,),
                ).fetchall()
            except Exception:
                bm25_rows = []

            # Hybrid search: merge BM25 + semantic via RRF
            bm25_ids = [r["id"] for r in bm25_rows]
            chunk_snippets = {}  # page_id -> best chunk text
            if get_setting("semantic_search", "0") == "1":
                try:
                    from embeddings import hybrid_search
                    use_reranker = get_setting("use_reranker", "1") == "1"
                    fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker)
                    fused_ids = [pid for pid, _ in fused]
                    chunk_snippets = {pid: text for pid, text in fused if text}
                except Exception:
                    fused_ids = bm25_ids
            else:
                fused_ids = bm25_ids

            total_results = len(fused_ids)
            page_ids = fused_ids[offset:offset + PER_PAGE]

            if page_ids:
                # Fetch rows in fused order
                placeholders = ",".join("?" * len(page_ids))
                all_rows = db.execute(
                    f"SELECT id, url, title, body, note, summary FROM pages WHERE id IN ({placeholders})",
                    page_ids,
                ).fetchall()
                row_map = {r["id"]: r for r in all_rows}
                rows = [row_map[pid] for pid in page_ids if pid in row_map]
            else:
                rows = []

            if rows:
                for r in rows:
                    note_html = ""
                    if r["note"]:
                        note_html = f'<div class="note"><em>{esc(r["note"])}</em></div>'
                    tags = _get_page_tags(r["id"], db)
                    tags_html = ""
                    if tags:
                        tag_links = " ".join(f'<a href="/tags/{esc(t)}" class="tag">[{esc(t)}]</a>' for t in tags)
                        tags_html = f'<div class="tags">{tag_links}</div>'
                    snip_html = f'<br>{esc(r["summary"])}' if r["summary"] else ""
                    result_html += (
                        f'<div class="result">'
                        f'<a href="{esc(r["url"])}" rel="noreferrer noopener">{esc(r["title"])}</a><br>'
                        f'<small>{esc(r["url"])}</small>'
                        f'{snip_html}'
                        f'{note_html}{tags_html}'
                        f'</div>'
                    )
            else:
                result_html = "<p>No results in your index.</p>"

            # search all linked pages from trusted sites
            words = q.lower().split()
            all_links = db.execute(
                "SELECT l.url, l.label, p.title AS source_title "
                "FROM links l JOIN pages p ON l.page_id = p.id",
            ).fetchall()
            indexed_urls = set(r["url"] for r in rows) if rows else set()
            seen = set()
            trusted = []
            for l in all_links:
                if l["url"] in indexed_urls or l["url"] in seen:
                    continue
                if any(w in l["label"].lower() for w in words):
                    seen.add(l["url"])
                    trusted.append(l)
                if len(trusted) >= 20:
                    break

            if trusted:
                items = ""
                for l in trusted:
                    items += (
                        f'<li><a href="{esc(clean_url(l["url"]))}" rel="noreferrer noopener">{esc(l["label"])}</a> '
                        f'<small>— from {esc(l["source_title"])}</small></li>'
                    )
                trusted_html = (
                    f'<details class="trusted">'
                    f'<summary>from your trusted sites ({len(trusted)})</summary>'
                    f'<ul>{items}</ul>'
                    f'</details>'
                )

            # search synced pages from subscriptions
            try:
                remote_rows = db.execute(
                    "SELECT rp.url, rp.title, rp.note, s.name AS source_name "
                    "FROM remote_pages_fts rpf "
                    "JOIN remote_pages rp ON rpf.rowid = rp.id "
                    "JOIN subscriptions s ON rp.subscription_id = s.id "
                    "WHERE remote_pages_fts MATCH ? ORDER BY rank LIMIT 50",
                    (_sanitize_fts_query(q),),
                ).fetchall()
            except Exception:
                remote_rows = []

        remote_html = ""
        if q and remote_rows:
            # group by source
            by_source = {}
            for r in remote_rows:
                source = r["source_name"] or "unknown"
                by_source.setdefault(source, []).append(r)
            for source, items in by_source.items():
                source_items = ""
                for r in items:
                    note_html = f' — <em>{esc(r["note"])}</em>' if r["note"] else ""
                    source_items += (
                        f'<li><a href="{esc(clean_url(r["url"]))}" rel="noreferrer noopener">{esc(r["title"])}</a>'
                        f'{note_html} <small>({esc(clean_url(r["url"]))})</small></li>'
                    )
                remote_html += (
                    f'<details class="remote" open>'
                    f'<summary>from {esc(source)} ({len(items)})</summary>'
                    f'<ul>{source_items}</ul>'
                    f'</details>'
                )
    finally:
        return_db(db)
    sub_count = ""
    if q and remote_rows:
        sub_count = f" + {len(remote_rows)} from subscriptions"
    return _respond(
        f'<form method="get" action="/">'
        f'<input name="q" value="{esc(q)}" placeholder="search your index" size="40">'
        f' <button type="submit">search</button>'
        f'</form>'
        f'<p class="meta">{count} pages indexed'
        f' · <a href="/add">+ add url</a></p>'
        f'{result_html}'
        f'{_page_nav(page, total_results, f"/?q={esc(q)}") if q else ""}'
        f'{trusted_html}{remote_html}'
    )


def handle_add_form(msg="", action_type="index"):
    if action_type == "subscribe":
        return _respond(
            f"<h1>subscribe</h1>"
            f"<p>Subscribe to a friend's TinyWeb instance to sync their index</p>"
            f'<form method="post" action="/subscriptions/add">'
            f'{_csrf_field()}'
            f'<input name="dest_hash" placeholder="destination hash (32 hex chars)" size="50"><br><br>'
            f'<button type="submit">subscribe</button>'
            f"</form>"
            f"<p><small>or <a href=\"/add\">add a single site</a></small></p>"
            f"<p>{msg}</p>"
            f'<a href="/">back</a>'
        )
    return _respond(
        f"<h1>add url</h1>"
        f"<p>Add a site to your index</p>"
        f'<form method="post" action="/add">'
        f'{_csrf_field()}'
        f'<input name="url" placeholder="https://example.com" size="50"><br><br>'
        f'<input name="note" placeholder="why are you saving this? (optional)" size="50"><br><br>'
        f'<input name="tags" placeholder="tags (comma-separated, e.g. solarpunk, mesh)" size="50"><br>'
        f'<small>tag: private to exclude from sharing</small><br><br>'
        f'<button type="submit">index</button>'
        f"</form>"
        f"<p>{msg}</p>"
        f'<a href="/">back</a>'
    )


def handle_add_submit(body):
    input_type = body.get("input_type", ["url"])[0]
    url = body.get("url", [""])[0].strip()
    reticulum_dest = body.get("reticulum_dest", [""])[0].strip().replace("<", "").replace(">", "")
    note = body.get("note", [""])[0].strip()
    tags = body.get("tags", [""])[0].strip()
    
    if input_type == "url":
        if not url:
            return handle_add_form("URL is required.")
        url = clean_url(url)
        if not url.startswith(("http://", "https://")):
            return handle_add_form("URL must start with http:// or https://")
    else:
        if not reticulum_dest:
            return handle_add_form("Reticulum destination hash is required.")
        if len(reticulum_dest) != 32 or not all(c in "0123456789abcdefABCDEF" for c in reticulum_dest):
            return handle_add_form("Invalid reticulum destination hash. Must be 32 hex characters.")
        url = f"reticulum:{reticulum_dest}"
    
    try:
        title = index_url(url, note, reticulum_dest if reticulum_dest else "")
        if tags:
            db = get_db()
            try:
                row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()
                if row:
                    _set_page_tags(row["id"], tags, db)
                    db.commit()
            finally:
                return_db(db)
        
        return handle_add_form(f'Indexed: {esc(url)}')
        
    except ValueError as e:
        return handle_add_form(f"Error: {esc(str(e))}")
        
    except Exception as e:
        error_msg = str(e).lower()
        # Check if it's a block response
        if "block" in error_msg or "cloudflare" in error_msg or "403" in error_msg:
            # Show manual entry form for blocked sites
            return _respond(
                f"<h1>add url (manual entry)</h1>"
                f"<p><strong>{esc(url)}</strong> blocks automated access. "
                f"You can still save it manually:</p>"
                f'<form method="post" action="/add/manual">'
                f'{_csrf_field()}'
                f'<input type="hidden" name="url" value="{esc(url)}">'
                f'<input type="hidden" name="note" value="{esc(note)}">'
                f'<input type="hidden" name="tags" value="{esc(tags)}">'
                f'<label>Title:</label><br>'
                f'<input name="manual_title" size="50" placeholder="page title" required><br><br>'
                f'<label>Description:</label><br>'
                f'<textarea name="manual_description" rows="4" cols="50" placeholder="what is this site about? (optional)"></textarea><br><br>'
                f'<button type="submit">save manually</button>'
                f"</form>"
                f'<a href="/">back</a>'
            )
        return handle_add_form(f"Error: could not fetch or index that URL. {esc(str(e)[:100])}")


def handle_add_manual_submit(body):
    url = clean_url(body.get("url", [""])[0].strip())
    note = body.get("note", [""])[0].strip()
    tags = body.get("tags", [""])[0].strip()
    manual_title = body.get("manual_title", [""])[0].strip()
    manual_desc = body.get("manual_description", [""])[0].strip()

    if not url:
        return handle_add_form("URL is required.")
    
    if not manual_title:
        return handle_add_form("Title is required for manual entry.")

    db = get_db()
    try:
        now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

        db.execute(
            "INSERT INTO pages (url, title, body, note, last_modified, summary) VALUES (?, ?, ?, ?, ?, ?) "
            "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, "
            "note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary",
            (url, manual_title, manual_desc, note, now, manual_desc[:200]),
        )

        # Get the page ID
        page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0]

        # Add tags if provided
        if tags:
            _set_page_tags(page_id, tags, db)

        db.commit()

        # Generate embeddings for this page (if semantic search is enabled)
        if get_setting("semantic_search", "0") == "1":
            try:
                from embeddings import store_embeddings
                # Pass the page_id, title, description, and db connection
                store_embeddings(page_id, manual_title, manual_desc, db)
                db.commit()
            except Exception as e:
                # Log error but don't fail the whole operation
                print(f"Error generating embeddings: {e}")

        return handle_add_form(f'Added manually: <a href="{esc(url)}" rel="noreferrer noopener">{esc(manual_title)}</a>')
    finally:
        return_db(db)


def handle_pages(query=None):
    msg = query.get("msg", [""])[0] if query else ""
    msg_html = f'<p class="success">{esc(msg)}</p>' if msg else ""
    page = _paginate(query or {})
    offset = (page - 1) * BROWSE_PER_PAGE
    db = get_db()
    try:
        total = db.execute("SELECT count(*) FROM pages").fetchone()[0]
        rows = db.execute(
            "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?",
            (BROWSE_PER_PAGE, offset),
        ).fetchall()
        items = ""
        for r in rows:
            note_html = f' — <em>{esc(r["note"])}</em>' if r["note"] else ""
            tags = _get_page_tags(r["id"], db)
            tags_html = ""
            if tags:
                tag_links = " ".join(f'<a href="/tags/{esc(t)}">[{esc(t)}]</a>' for t in tags)
                tags_html = f' {tag_links}'
            items += (
                f'<li><label><input type="checkbox" name="ids" value="{r["id"]}"> '
                f'{esc(r["title"])}</label>{note_html}{tags_html} '
                f'<small>(<a href="{esc(r["url"])}" rel="noreferrer noopener">{esc(r["url"])}</a>)</small> '
                f'<a href="/edit/{r["id"]}">edit</a> '
                f'<a href="/delete/{r["id"]}">remove</a></li>'
            )
    finally:
        return_db(db)
    return _respond(
        f"<h1>indexed pages ({total})</h1>"
        f"{msg_html}"
        f'<form method="post" action="/pages/bulk">'
        f'{_csrf_field()}'
        f'<p><label><input type="checkbox" id="select-all"> select all</label></p>'
        f"<ul>{items}</ul>"
        f'{_page_nav(page, total, "/pages", BROWSE_PER_PAGE)}'
        f'<details><summary>bulk actions</summary>'
        f'<p><button type="submit" name="action" value="delete" id="bulk-delete">delete selected</button></p>'
        f'<p><input name="bulk_tags" placeholder="tags (comma-separated)" size="40"> '
        f'<select name="tag_mode"><option value="add">add tags</option><option value="replace">replace tags</option></select> '
        f'<button type="submit" name="action" value="retag">retag selected</button></p>'
        f'</details>'
        f'</form>'
        f'<script>'
        f'document.getElementById("select-all").addEventListener("change",function(){{'
        f'document.querySelectorAll("input[name=ids]").forEach(function(c){{c.checked=this.checked}}.bind(this))'
        f'}});'
        f'document.getElementById("bulk-delete").addEventListener("click",function(e){{'
        f'var n=document.querySelectorAll("input[name=ids]:checked").length;'
        f'if(!n){{e.preventDefault();return}}'
        f'if(!confirm("Delete "+n+" selected page"+(n===1?"":"s")+"?"))e.preventDefault()'
        f'}});'
        f'</script>'
        f'<p><a href="/export">export</a> | <a href="/import">import</a></p>'
        f'<a href="/">back</a>'
    )


def handle_bulk_action(body):
    ids = body.get("ids", [])
    action = body.get("action", [""])[0]
    if not ids:
        return _redirect("/pages")
    # Validate all ids are integers
    try:
        page_ids = [int(i) for i in ids]
    except ValueError:
        return _error(400)
    db = get_db()
    try:
        if action == "delete":
            for pid in page_ids:
                db.execute("DELETE FROM page_tags WHERE page_id = ?", (pid,))
                db.execute("DELETE FROM links WHERE page_id = ?", (pid,))
                db.execute("DELETE FROM pages WHERE id = ?", (pid,))
            _cleanup_orphaned_tags(db)
            db.commit()
        elif action == "retag":
            bulk_tags = body.get("bulk_tags", [""])[0].strip()
            tag_mode = body.get("tag_mode", ["add"])[0]
            if bulk_tags:
                for pid in page_ids:
                    if tag_mode == "add":
                        existing = _get_page_tags(pid, db)
                        new_tags = [t.strip().lower() for t in bulk_tags.split(",") if t.strip()]
                        merged = ", ".join(sorted(set(existing + new_tags)))
                        _set_page_tags(pid, merged, db)
                    else:
                        _set_page_tags(pid, bulk_tags, db)
                _cleanup_orphaned_tags(db)
                db.commit()
    finally:
        return_db(db)
    return _redirect("/pages")


def handle_edit_form(page_id, msg=""):
    db = get_db()
    try:
        row = db.execute("SELECT id, url, title, body, note, summary FROM pages WHERE id = ?", (page_id,)).fetchone()
        if not row:
            return _error(404)
        tags = ", ".join(_get_page_tags(page_id, db))
    finally:
        return_db(db)

    return _respond(
        f"<h1>edit page</h1>"
        f"<p><b>{esc(row['title'])}</b><br>"
        f"<small>{esc(row['url'])}</small></p>"
        f'<form method="post" action="/edit/{row["id"]}">'
        f'{_csrf_field()}'
        f'<label>Title:</label><br>'
        f'<input name="title" value="{esc(row["title"])}" size="60"><br><br>'
        f'<label>Summary (shown in search results):</label><br>'
        f'<textarea name="summary" rows="3" cols="60">{esc(row["summary"] or "")}</textarea><br><br>'
        f'<label>Note (why you saved this):</label><br>'
        f'<input name="note" value="{esc(row["note"])}" size="50"><br><br>'
        f'<label>Tags (comma-separated):</label><br>'
        f'<input name="tags" value="{esc(tags)}" size="50"> '
        f'<small>(tag: private to keep private)</small><br><br>'
        f'<button type="submit">save</button>'
        f"</form>"
        f"<p>{msg}</p>"
        f'<a href="/pages">back</a>'
    )


def handle_edit_submit(page_id, body):
    title = body.get("title", [""])[0].strip()
    summary = body.get("summary", [""])[0].strip()
    note = body.get("note", [""])[0].strip()
    tags = body.get("tags", [""])[0].strip()

    db = get_db()
    try:
        db.execute(
            "UPDATE pages SET title = ?, summary = ?, note = ? WHERE id = ?",
            (title, summary, note, page_id)
        )

        _set_page_tags(page_id, tags, db)
        _cleanup_orphaned_tags(db)

        db.commit()

    finally:
        return_db(db)

    return _redirect("/pages")


def handle_delete_confirm(page_id):
    db = get_db()
    try:
        row = db.execute("SELECT id, url, title FROM pages WHERE id = ?", (page_id,)).fetchone()
    finally:
        return_db(db)
    if not row:
        return _error(404)
    return _respond(
        f"<h1>confirm delete</h1>"
        f"<p>Remove <b>{esc(row['title'])}</b><br>"
        f"<small>{esc(row['url'])}</small></p>"
        f'<form method="post" action="/delete/{row["id"]}">'
        f'{_csrf_field()}'
        f'<button type="submit">yes, delete</button>'
        f"</form>"
        f' <a href="/pages">cancel</a>'
    )


def handle_delete(page_id):
    db = get_db()
    try:
        db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,))
        db.execute("DELETE FROM links WHERE page_id = ?", (page_id,))
        db.execute("DELETE FROM pages WHERE id = ?", (page_id,))
        _cleanup_orphaned_tags(db)
        db.commit()
    finally:
        return_db(db)
    return _redirect("/pages")


def handle_bookmark(query):
    token = query.get("token", [""])[0]
    expected = _get_bookmark_token()
    if not token or not secrets.compare_digest(token, expected):
        return _text_response("error: invalid or missing token", status=403, headers={"Access-Control-Allow-Origin": "*"})
    url = clean_url(query.get("url", [""])[0].strip())
    if not url or not url.startswith(("http://", "https://")):
        return _text_response("error: invalid url", headers={"Access-Control-Allow-Origin": "*"})
    try:
        title = index_url(url)
        msg = f"ok: {title}"
    except Exception as e:
        msg = f"error: {e}"
    return _text_response(msg, headers={"Access-Control-Allow-Origin": "*"})


MAX_EXPORT = 10000

def handle_export(query=None):
    try:
        batch = int((query or {}).get("batch", ["0"])[0])
    except (TypeError, ValueError):
        batch = 0
    db = get_db()
    try:
        rows = db.execute(
            "SELECT url, title, note FROM pages ORDER BY id LIMIT ? OFFSET ?",
            (MAX_EXPORT, batch * MAX_EXPORT),
        ).fetchall()
    finally:
        return_db(db)
    data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows]
    return _json_response(data, headers={"Content-Disposition": "attachment; filename=tinyweb-export.json"})


def handle_import_form(msg=""):
    return _respond(
        f"<h1>import</h1>"
        f"<p>Paste the contents of a tinyweb export file (JSON).</p>"
        f'<form method="post" action="/import">'
        f'{_csrf_field()}'
        f'<textarea name="data" rows="12" cols="60" placeholder=\'[{{"url": "...", "note": "..."}}]\'></textarea><br><br>'
        f'<button type="submit">import</button>'
        f"</form>"
        f"<p>{msg}</p>"
        f'<a href="/pages">back</a>'
    )


def handle_import_submit(body):
    raw = body.get("data", [""])[0].strip()
    if not raw:
        return handle_import_form("Paste JSON data.")
    try:
        data = json.loads(raw)
    except json.JSONDecodeError:
        return handle_import_form("Invalid JSON.")
    if not isinstance(data, list):
        return handle_import_form("Expected a JSON array.")

    MAX_IMPORT = 100
    if len(data) > MAX_IMPORT:
        return handle_import_form(f"Too many entries. Maximum is {MAX_IMPORT}.")

    imported = 0
    errors = 0
    for entry in data:
        url = entry.get("url", "").strip()
        note = entry.get("note", "").strip()
        if not url:
            continue
        try:
            index_url(url, note)
            imported += 1
        except Exception:
            errors += 1

    return handle_import_form(f"Imported {imported} page(s). {errors} error(s).")


def handle_style_form(msg=""):
    template = get_setting("custom_template") or DEFAULT_TEMPLATE
    name = get_site_name()
    sharing = get_setting("sharing_enabled", "0")
    checked = " checked" if sharing == "1" else ""
    sharing_mode = get_setting("sharing_mode", "exclude_private")
    exclude_checked = " checked" if sharing_mode != "require_public" else ""
    require_checked = " checked" if sharing_mode == "require_public" else ""
    shared_count = _count_shared_pages()
    semantic = get_setting("semantic_search", "0")
    semantic_checked = " checked" if semantic == "1" else ""
    reranker = get_setting("use_reranker", "0")
    reranker_checked = " checked" if reranker == "1" else ""
    disabled = "" if semantic == "1" else " disabled"
    dimmed = ' style="opacity:0.4"' if semantic != "1" else ""
    tcp_enabled = get_setting("tcp_enabled", "1")
    tcp_checked = " checked" if tcp_enabled == "1" else ""
    tcp_disabled = "" if tcp_enabled == "1" else " disabled"
    transport_host = get_setting("transport_host", "reticulum.derickphan.com")
    transport_port = get_setting("transport_port", "4242")
    compress = get_setting("compress_embeddings", "0")
    compress_checked = " checked" if compress == "1" else ""
    lora_enabled = get_setting("lora_enabled", "0")
    lora_checked = " checked" if lora_enabled == "1" else ""
    lora_disabled = "" if lora_enabled == "1" else " disabled"
    lora_dimmed = ' style="opacity:0.4"' if lora_enabled != "1" else ""
    lora_port = get_setting("lora_port", "")
    lora_frequency = get_setting("lora_frequency", "867200000")
    lora_bandwidth = get_setting("lora_bandwidth", "125000")
    lora_txpower = get_setting("lora_txpower", "7")
    lora_sf = get_setting("lora_sf", "8")
    lora_cr = get_setting("lora_cr", "5")
    return _respond(
        f"<h1>customize</h1>"
        f"<h2>name your search engine</h2>"
        f'<form method="post" action="/style">'
        f'{_csrf_field()}'
        f'<input name="site_name" value="{esc(name)}" placeholder="tinyweb" size="30"><br><br>'
        f"<h2>sharing</h2>"
        f'<label><input type="checkbox" name="sharing_enabled" value="1"{checked}>'
        f" share your site list publicly at /api/sites</label><br>"
        f'<div style="margin-top:0.6rem">'
        f"<small>What to share:</small><br>"
        f'<label><input type="radio" name="sharing_mode" value="exclude_private"{exclude_checked}>'
        f' share all pages except those tagged <code>private</code></label><br>'
        f'<label><input type="radio" name="sharing_mode" value="require_public"{require_checked}>'
        f' share only pages tagged <code>public</code></label><br>'
        f'<small>The <code>private</code> tag always excludes a page, even in public-only mode.</small>'
        f'</div>'
        f'<p style="margin-top:0.6rem">'
        f'Currently sharing <b>{shared_count}</b> page(s). '
        f'<a href="/share/preview">preview what subscribers would see</a>'
        f'</p>'
        f"<h2>mesh network</h2>"
        f"<p>Choose how to connect to the mesh. You can enable both for maximum reach.</p>"
        f"<h3>internet</h3>"
        f'<label><input type="checkbox" name="tcp_enabled" value="1"{tcp_checked} '
        f'onchange="var d=!this.checked;'
        f'for(var e of document.querySelectorAll(\'#tcp-fields input\'))e.disabled=d;'
        f'document.getElementById(\'tcp-fields\').style.opacity=d?\'0.4\':\'1\'">'
        f" connect via internet transport node</label><br>"
        f"<small>Reach peers anywhere online.</small><br>"
        f'<div id="tcp-fields" style="margin-top:0.5rem{";opacity:0.4" if tcp_enabled != "1" else ""}">'
        f"<small>Default: reticulum.derickphan.com:4242</small><br>"
        f'<input name="transport_host" value="{esc(transport_host)}" placeholder="hostname" size="30"{tcp_disabled}>'
        f' <input name="transport_port" value="{esc(transport_port)}" placeholder="port" size="6"{tcp_disabled}><br>'
        f'<p><a href="https://rmap.world/" target="_blank" rel="noreferrer noopener">discover more nodes</a></p>'
        f'</div><br>'
        f"<h3>LoRa</h3>"
        f'<label><input type="checkbox" name="lora_enabled" value="1"{lora_checked} '
        f'onchange="var d=!this.checked;document.getElementById(\'lora-port\').disabled=d;'
        f'document.getElementById(\'lora-extras\').style.opacity=d?\'0.4\':\'1\';'
        f'for(var e of document.querySelectorAll(\'#lora-extras input\'))e.disabled=d">'
        f" connect via LoRa radio</label><br>"
        f"<small>Reach nearby peers off-grid with an <a href=\"https://unsigned.io/rnode/\" target=\"_blank\" rel=\"noreferrer noopener\">RNode</a>.</small><br><br>"
        f'<div id="lora-fields" style="{";opacity:0.4" if lora_enabled != "1" else ""}">'
        f'<label>Serial port: <input id="lora-port" name="lora_port" value="{esc(lora_port)}" '
        f'placeholder="/dev/ttyUSB0" size="20"{lora_disabled}></label><br><br>'
        f'<details><summary>advanced radio settings</summary>'
        f'<div id="lora-extras" style="margin-top:0.5rem">'
        f'<label>Frequency (Hz): <input name="lora_frequency" value="{esc(lora_frequency)}" size="12"{lora_disabled}></label><br>'
        f"<small>ISM band frequency. Default: 867200000 (868 MHz EU). US: 915000000.</small><br><br>"
        f'<label>Bandwidth (Hz): <input name="lora_bandwidth" value="{esc(lora_bandwidth)}" size="8"{lora_disabled}></label><br>'
        f"<small>Default: 125000</small><br><br>"
        f'<label>TX Power (dBm): <input name="lora_txpower" value="{esc(lora_txpower)}" size="4"{lora_disabled}></label><br>'
        f"<small>0-17 typical. Check local regulations.</small><br><br>"
        f'<label>Spreading Factor: <input name="lora_sf" value="{esc(lora_sf)}" size="4"{lora_disabled}></label><br>'
        f"<small>5-12. Higher = longer range, slower speed.</small><br><br>"
        f'<label>Coding Rate: <input name="lora_cr" value="{esc(lora_cr)}" size="4"{lora_disabled}></label><br>'
        f"<small>5-8. Higher = more error correction.</small><br>"
        f'</div></details></div><br>'
        f"<h2>search</h2>"
        f"<h3>ai</h3>"
        f'<label><input type="checkbox" name="semantic_search" value="1"{semantic_checked} '
        f'onchange="var d=!this.checked;document.getElementById(\'reranker\').disabled=d;'
        f'document.getElementById(\'ai-extras\').style.opacity=d?\'0.4\':\'1\'">'
        f" semantic search (similarity matching)</label><br>"
        f"<small>Requires onnxruntime, tokenizers, hnswlib. Downloads ~30MB of models on first use.</small><br><br>"
        f'<div id="ai-extras"{dimmed}>'
        f'<label><input type="checkbox" id="reranker" name="use_reranker" value="1"{reranker_checked}{disabled}>'
        f" cross-encoder reranking (more accurate)</label><br>"
        f"<small>Uses a 22MB model. Adds ~50ms per search. Disable for faster results.</small><br><br>"
        f'<label><input type="checkbox" name="compress_embeddings" value="1"{compress_checked}{disabled}>'
        f" compress embeddings (50% storage savings)</label><br>"
        f"<small>Saves ~50% on storage for embeddings. Slight quality reduction at large scale.</small><br><br>"
        f'<a href="/reindex">manage semantic index</a><br><br>'
        f"</div>"
        f"<h2>custom html</h2>"
        f"<p>Edit the full page template. Use <code>{esc('{{content}}')}</code> "
        f"where page content should appear.</p>"
        f'<textarea name="template" rows="20" cols="60">{esc(template)}</textarea><br><br>'
        f'<button type="submit">save</button>'
        f"</form>"
        f"<h2>bookmarklet</h2>"
        f"<p>Drag this link to your bookmarks bar. Click it on any page to index it instantly.</p>"
        f'<p><a href="javascript:void(fetch(\'http://localhost:8080/bookmark?url=\'+encodeURIComponent(location.href)+\'&token={_get_bookmark_token()}\').then(r=>r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}</a></p>'
        f"<h2>reset</h2>"
        f'<form method="post" action="/style/reset">'
        f'{_csrf_field()}'
        f'<button type="submit">reset template to default</button>'
        f"</form>"
        f"<h2>maintenance</h2>"
        f'<form method="post" action="/style/vacuum">'
        f'{_csrf_field()}'
        f'<button type="submit">vacuum database</button>'
        f"</form>"
        f"<p>{msg}</p>"
        f'<a href="/">back</a>',
        use_default=True,
    )


def handle_style_submit(body):
    template = body.get("template", [""])[0].replace("\r\n", "\n").replace("\r", "\n")
    name = body.get("site_name", ["tinyweb"])[0].strip()
    sharing = "1" if body.get("sharing_enabled") else "0"
    sharing_mode = body.get("sharing_mode", ["exclude_private"])[0]
    if sharing_mode not in ("exclude_private", "require_public"):
        sharing_mode = "exclude_private"
    set_setting("sharing_mode", sharing_mode)
    semantic = "1" if body.get("semantic_search") else "0"
    reranker = "1" if body.get("use_reranker") else "0"
    compress = "1" if body.get("compress_embeddings") else "0"
    tcp_enabled = "1" if body.get("tcp_enabled") else "0"
    transport_host = body.get("transport_host", [""])[0].strip()
    transport_port = body.get("transport_port", [""])[0].strip()
    set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "")
    set_setting("site_name", name or "tinyweb")
    set_setting("sharing_enabled", sharing)
    set_setting("semantic_search", semantic)
    set_setting("use_reranker", reranker)
    set_setting("compress_embeddings", compress)
    set_setting("tcp_enabled", tcp_enabled)
    if transport_host:
        set_setting("transport_host", transport_host)
    if transport_port:
        set_setting("transport_port", transport_port)
    lora_enabled = "1" if body.get("lora_enabled") else "0"
    set_setting("lora_enabled", lora_enabled)
    set_setting("lora_port", body.get("lora_port", [""])[0].strip())
    set_setting("lora_frequency", body.get("lora_frequency", ["867200000"])[0].strip())
    set_setting("lora_bandwidth", body.get("lora_bandwidth", ["125000"])[0].strip())
    set_setting("lora_txpower", body.get("lora_txpower", ["7"])[0].strip())
    set_setting("lora_sf", body.get("lora_sf", ["8"])[0].strip())
    set_setting("lora_cr", body.get("lora_cr", ["5"])[0].strip())
    return handle_style_form("Saved. Restart TinyWeb for mesh network changes to take effect.")


def handle_about():
    name = get_site_name()
    dest_hash = get_setting("dest_hash")
    sharing = get_setting("sharing_enabled", "0") == "1"
    db = get_db()
    try:
        page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0]
        tag_count = db.execute("SELECT count(DISTINCT tag_id) FROM page_tags").fetchone()[0]
        sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0]
    finally:
        return_db(db)

    sharing_html = (
        '<p>This instance shares its index publicly. Subscribe to join the network.</p>'
        if sharing else
        '<p>This instance is private.</p>'
    )

    hash_html = ""
    if dest_hash:
        hash_html = (
            f'<h2>subscribe</h2>'
            f'<p>To subscribe to this instance, add this destination hash in your TinyWeb:</p>'
            f'<pre>{esc(dest_hash)}</pre>'
        )

    return _respond(
        f'<h1>{esc(name)}</h1>'
        f'<p>A personal search engine, built for the slow web.</p>'
        f'<p>TinyWeb is about taking back the internet. No algorithms, no ads, no tracking. '
        f'Just human-curated pages shared freely across a mesh network.</p>'
        f'<ul>'
        f'<li><b>{page_count}</b> page(s) indexed</li>'
        f'<li><b>{tag_count}</b> tag(s)</li>'
        f'<li><b>{sub_count}</b> subscription(s)</li>'
        f'</ul>'
        f'{sharing_html}'
        f'{hash_html}'
        f'<h2>what is the slow web?</h2>'
        f'<p>The slow web is a movement for intentionality over speed, '
        f'human curation over algorithmic feeds, privacy over surveillance, '
        f'and community over corporations. Every page in this index was saved by a person '
        f'because they found it valuable — not because an algorithm told them to click.</p>'
        f'<h2>how it works</h2>'
        f'<ul>'
        f'<li>Save pages you find valuable with the bookmarklet or /add</li>'
        f'<li>Search your personal index — queries never leave your machine</li>'
        f'<li>Subscribe to friends over Reticulum — encrypted, decentralized, works without the internet</li>'
        f'<li>Tag and organize your collection into curated lists</li>'
        f'</ul>'
        f'<p><a href="/">search</a> | <a href="/pages">browse</a> | <a href="/tags">tags</a></p>'
    )


def handle_tags():
    db = get_db()
    try:
        rows = db.execute(
            "SELECT t.name, COUNT(pt.page_id) AS cnt FROM tags t "
            "JOIN page_tags pt ON t.id = pt.tag_id "
            "GROUP BY t.id ORDER BY t.name"
        ).fetchall()
    finally:
        return_db(db)
    items = ""
    for r in rows:
        items += f'<li><a href="/tags/{esc(r["name"])}">{esc(r["name"])}</a> ({r["cnt"]})</li>'
    return _respond(
        f"<h1>tags</h1>"
        f"<ul>{items}</ul>" if items else "<p>No tags yet. Add tags when saving or editing pages.</p>"
        f'<a href="/">back</a>'
    )


def handle_tag_browse(tag_name, query=None):
    page = _paginate(query or {})
    offset = (page - 1) * BROWSE_PER_PAGE
    db = get_db()
    try:
        total = db.execute(
            "SELECT count(*) FROM page_tags pt JOIN tags t ON t.id = pt.tag_id WHERE t.name = ?",
            (tag_name,),
        ).fetchone()[0]
        rows = db.execute(
            "SELECT p.id, p.url, p.title, p.note FROM pages p "
            "JOIN page_tags pt ON p.id = pt.page_id "
            "JOIN tags t ON t.id = pt.tag_id "
            "WHERE t.name = ? ORDER BY p.id DESC LIMIT ? OFFSET ?",
            (tag_name, BROWSE_PER_PAGE, offset),
        ).fetchall()
        items = ""
        for r in rows:
            note_html = f' — <em>{esc(r["note"])}</em>' if r["note"] else ""
            tags = _get_page_tags(r["id"], db)
            tag_links = " ".join(f'<a href="/tags/{esc(t)}">[{esc(t)}]</a>' for t in tags)
            items += (
                f'<li>{esc(r["title"])}{note_html} {tag_links} '
                f'<small>(<a href="{esc(r["url"])}" rel="noreferrer noopener">{esc(r["url"])}</a>)</small></li>'
            )
    finally:
        return_db(db)
    return _respond(
        f'<h1>tag: {esc(tag_name)}</h1>'
        f'<p>{total} page(s)</p>'
        f'<ul>{items}</ul>'
        f'{_page_nav(page, total, f"/tags/{esc(tag_name)}", BROWSE_PER_PAGE)}'
        f'<a href="/tags">all tags</a> | <a href="/">back</a>'
    )


MAX_API_SITES = 5000


def _page_is_shared(tags, mode):
    """Decide whether a page with the given tags is shared under the given mode.

    `private` always wins — a page tagged private is never shared, regardless of mode.
    """
    if "private" in tags:
        return False
    if mode == "require_public" and "public" not in tags:
        return False
    return True


def _shared_sites(db, since=""):
    """Return the full site records that a subscriber would receive.

    The caller owns the db connection.
    """
    mode = get_setting("sharing_mode", "exclude_private")
    if since:
        rows = db.execute(
            "SELECT id, url, title, note, last_modified FROM pages "
            "WHERE last_modified > ? ORDER BY id DESC LIMIT ?",
            (since, MAX_API_SITES),
        ).fetchall()
    else:
        rows = db.execute(
            "SELECT id, url, title, note, last_modified FROM pages ORDER BY id DESC LIMIT ?",
            (MAX_API_SITES,),
        ).fetchall()
    sites = []
    for r in rows:
        tags = _get_page_tags(r["id"], db)
        if not _page_is_shared(tags, mode):
            continue
        sites.append({
            "url": r["url"], "title": r["title"], "note": r["note"],
            "tags": tags, "last_modified": r["last_modified"] or "",
        })
    return sites


def _shared_all_urls(db):
    """Return the URL list a subscriber uses to detect deletions."""
    mode = get_setting("sharing_mode", "exclude_private")
    rows = db.execute(
        "SELECT id, url FROM pages ORDER BY id DESC LIMIT ?", (MAX_API_SITES,)
    ).fetchall()
    return [r["url"] for r in rows if _page_is_shared(_get_page_tags(r["id"], db), mode)]


def _count_shared_pages():
    """Cheap page count under the current sharing rule — used by the settings UI."""
    db = get_db()
    try:
        return len(_shared_all_urls(db))
    finally:
        return_db(db)


def handle_share_preview():
    """Show the list of pages a subscriber would currently receive.

    Works regardless of whether sharing is enabled — lets the user see the surface
    before flipping it on.
    """
    mode = get_setting("sharing_mode", "exclude_private")
    mode_label = (
        "only pages tagged <code>public</code>"
        if mode == "require_public"
        else "all pages except those tagged <code>private</code>"
    )
    sharing_on = get_setting("sharing_enabled", "0") == "1"
    status = (
        '<p>Sharing is <b>enabled</b>. Subscribers see the pages listed below.</p>'
        if sharing_on else
        '<p>Sharing is <b>disabled</b>. Nothing is actually being shared right now; '
        'this is the list that would be exposed if you enabled it.</p>'
    )
    db = get_db()
    try:
        sites = _shared_sites(db)
    finally:
        return_db(db)
    if not sites:
        body = (
            "<h1>sharing preview</h1>"
            f"<p>Rule: {mode_label}.</p>"
            f"{status}"
            "<p><em>No pages match the current rule.</em></p>"
            '<p><a href="/style">back to settings</a></p>'
        )
        return _respond(body)
    rows = ""
    for s in sites:
        tags_html = ""
        if s["tags"]:
            tags_html = " " + " ".join(f"[{esc(t)}]" for t in s["tags"])
        note_html = f' — <em>{esc(s["note"])}</em>' if s["note"] else ""
        rows += (
            f'<li>'
            f'<a href="{esc(s["url"])}" rel="noreferrer noopener">{esc(s["title"] or s["url"])}</a>'
            f'{note_html}{tags_html} '
            f'<br><small>{esc(s["url"])}</small>'
            f'</li>'
        )
    body = (
        "<h1>sharing preview</h1>"
        f"<p>Rule: {mode_label}.</p>"
        f"{status}"
        f"<p><b>{len(sites)}</b> page(s) visible to subscribers.</p>"
        f"<ul>{rows}</ul>"
        '<p><a href="/style">back to settings</a></p>'
    )
    return _respond(body)


def handle_api_sites(query=None):
    if get_setting("sharing_enabled", "0") != "1":
        return _json_response(
            {"error": "sharing disabled"},
            status=403,
            headers={"Access-Control-Allow-Origin": "*"},
        )
    since = (query or {}).get("since", [""])[0].strip()
    db = get_db()
    try:
        sites = _shared_sites(db, since=since)
        all_urls = _shared_all_urls(db) if not since else None
    finally:
        return_db(db)
    data = {"name": get_site_name(), "sites": sites}
    if all_urls is not None:
        data["all_urls"] = all_urls
    return _json_response(data, headers={"Access-Control-Allow-Origin": "*"})


_sync_threads = {}


def handle_subscriptions(msg=""):
    db = get_db()
    try:
        subs = db.execute("SELECT * FROM subscriptions ORDER BY id DESC").fetchall()
    finally:
        return_db(db)
    cards = ""
    for s in subs:
        sub_id = s["id"]
        auto_label = "on" if s["auto_sync"] else "off"
        last = s["last_sync"] or "never"
        sync_status = get_setting(f"sync_status_{sub_id}", "")
        is_syncing = sub_id in _sync_threads and _sync_threads[sub_id].is_alive()

        # Status line: show syncing indicator or last result
        if is_syncing:
            status_html = '<div style="margin-top:0.4rem;font-size:0.85rem;color:#2070c0">syncing...</div>'
        elif sync_status.startswith("error:"):
            err_msg = sync_status[6:]
            status_html = f'<div style="margin-top:0.4rem;font-size:0.85rem;color:#c03030">{esc(err_msg)}</div>'
        else:
            status_html = ""

        # Disable sync button while syncing
        if is_syncing:
            sync_btn = '<button disabled>syncing...</button>'
        else:
            sync_btn = (
                f'<form method="post" action="/subscriptions/sync/{sub_id}" style="display:inline">'
                f'{_csrf_field()}<button>sync now</button></form>'
            )

        cards += (
            f'<div style="border:1px solid #ddd;border-radius:4px;padding:0.9rem 1rem;margin-bottom:0.75rem">'
            f'<div style="margin-bottom:0.4rem"><b>{esc(s["name"] or "unknown")}</b></div>'
            f'<div><small>{esc(s["dest_hash"])}</small></div>'
            f'<div style="margin-top:0.4rem;font-size:0.85rem;color:#606060">last sync: {esc(last)}</div>'
            f'{status_html}'
            f'<div style="display:flex;gap:0.5rem;align-items:center;flex-wrap:wrap;margin-top:0.7rem">'
            f'<a href="/subscriptions/browse/{sub_id}">browse</a>'
            f'{sync_btn}'
            f'<form method="post" action="/subscriptions/autosync/{sub_id}" style="display:inline">'
            f'{_csrf_field()}<button>auto-sync: {auto_label}</button></form>'
            f'<form method="post" action="/subscriptions/delete/{sub_id}" style="display:inline">'
            f'{_csrf_field()}<button>remove</button></form>'
            f'</div>'
            f'</div>'
        )
    listing = ""
    if subs:
        any_syncing = any(sid in _sync_threads and _sync_threads[sid].is_alive() for sid in [s["id"] for s in subs])
        syncall_btn = '<button disabled>syncing...</button>' if any_syncing else '<button>sync all</button>'
        listing = (
            f'{cards}'
            f'<form method="post" action="/subscriptions/syncall">'
            f'{_csrf_field()}{syncall_btn}</form>'
        )
    return _respond(
        f"<h1>subscriptions</h1>"
        f'<form method="post" action="/subscriptions/add">'
        f'{_csrf_field()}'
        f'<input name="dest_hash" placeholder="destination hash" size="40"> '
        f'<button>subscribe</button>'
        f'</form>'
        f'<p><small>or <a href="/add?type=subscribe">subscribe to an instance</a></small></p>'
        f'<p>{msg}</p>'
        f'<hr>{listing}'
        f'<br><a href="/">back</a>'
    )


def handle_subscription_add(body):
    dest_hash = body.get("dest_hash", [""])[0].strip().replace("<", "").replace(">", "")
    if not dest_hash or len(dest_hash) != 32:
        return handle_subscriptions("Enter a valid 32-character destination hash.")
    try:
        int(dest_hash, 16)
    except ValueError:
        return handle_subscriptions("Invalid destination hash (must be hex).")
    try:
        data = fetch_remote_sites(dest_hash)
        name = data.get("name", "")
    except PermissionError:
        return handle_subscriptions("That instance has sharing disabled.")
    except Exception:
        return handle_subscriptions("Could not reach that instance.")
    db = get_db()
    try:
        db.execute(
            "INSERT INTO subscriptions (dest_hash, name) VALUES (?, ?) "
            "ON CONFLICT(dest_hash) DO UPDATE SET name=excluded.name",
            (dest_hash, name),
        )
        db.commit()
    finally:
        return_db(db)
    return handle_subscriptions(f"Subscribed to {esc(name or dest_hash)}.")


MAX_BROWSE = 5000

def handle_subscription_browse(sub_id):
    db = get_db()
    try:
        sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone()
        if not sub:
            return _error(404)
        local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages LIMIT ?", (MAX_BROWSE,)).fetchall())

        # Use locally synced data if available, otherwise fetch live
        remote_rows = db.execute(
            "SELECT url, title, note, tags FROM remote_pages WHERE subscription_id = ? LIMIT ?",
            (sub_id, MAX_BROWSE),
        ).fetchall()
    finally:
        return_db(db)

    if remote_rows:
        sites = []
        for r in remote_rows:
            tags = [t for t in r["tags"].split(",") if t] if r["tags"] else []
            sites.append({"url": r["url"], "title": r["title"], "note": r["note"], "tags": tags})
    else:
        try:
            data = fetch_remote_sites(sub["dest_hash"])
            sites = data.get("sites", [])
        except PermissionError:
            return handle_subscriptions("That instance has sharing disabled.")
        except Exception:
            return handle_subscriptions("Could not fetch sites from that instance.")

    new_items = ""
    existing_items = ""
    new_count = 0
    for s in sites:
        if s["url"] in local_urls:
            existing_items += (
                f'<li style="opacity:0.5">{esc(s["title"])} '
                f'<small>({esc(s["url"])})</small> — already indexed</li>'
            )
        else:
            new_count += 1
            note_html = f' — <em>{esc(s["note"])}</em>' if s.get("note") else ""
            tags_html = ""
            if s.get("tags"):
                tags_html = " " + " ".join(f'[{esc(t)}]' for t in s["tags"])
            new_items += (
                f'<li><label><input type="checkbox" name="urls" value="{esc(s["url"])}">'
                f' {esc(s["title"])}{note_html}{tags_html}'
                f' <small>({esc(s["url"])})</small></label></li>'
            )

    buttons = ""
    if new_count:
        buttons = '<button>import selected</button> <button name="import_all" value="1">import all new</button>'
    return _respond(
        f'<h1>browsing: {esc(sub["name"] or sub["dest_hash"])}</h1>'
        f'<p>{len(sites)} site(s) available, {new_count} new</p>'
        f'<form method="post" action="/subscriptions/pick">'
        f'{_csrf_field()}'
        f'<input type="hidden" name="sub_id" value="{sub_id}">'
        f'<ul>{new_items}</ul>'
        f'{buttons}'
        f'</form>'
        f'<h3>already indexed</h3><ul>{existing_items}</ul>'
        f'<a href="/subscriptions">back</a>'
    )


def handle_subscription_pick(body):
    sub_id = body.get("sub_id", [""])[0]
    import_all = body.get("import_all", [""])[0]

    # Build a url->tags map from remote_pages for this subscription
    db = get_db()
    try:
        remote_rows = db.execute(
            "SELECT url, tags FROM remote_pages WHERE subscription_id = ?", (sub_id,)
        ).fetchall()
        remote_tags = {r["url"]: r["tags"] for r in remote_rows}

        if import_all:
            local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages LIMIT ?", (MAX_BROWSE,)).fetchall())
            urls = [r["url"] for r in remote_rows if r["url"] not in local_urls]
        else:
            urls = body.get("urls", [])
    finally:
        return_db(db)

    if not urls:
        return handle_subscriptions("No sites selected.")

    imported = 0
    errors = 0
    for url in urls:
        try:
            index_url(url)
            # Import tags from the remote page
            tags_str = remote_tags.get(url, "")
            if tags_str:
                db = get_db()
                try:
                    row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()
                    if row:
                        _set_page_tags(row["id"], tags_str, db)
                        db.commit()
                finally:
                    return_db(db)
            imported += 1
        except Exception:
            errors += 1
    return handle_subscriptions(f"Imported {imported} page(s). {errors} error(s).")


def _sync_subscription(sub_id):
    """Run a single subscription sync. Designed to run in a background thread."""
    set_setting(f"sync_status_{sub_id}", "syncing")
    db = get_db()
    try:
        sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone()
        if not sub:
            set_setting(f"sync_status_{sub_id}", "error:Subscription not found.")
            return
        since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else ""
        try:
            data = fetch_remote_sites(sub["dest_hash"], since=since)
            sites = data.get("sites", [])
            all_urls = data.get("all_urls")
            remote_name = data.get("name", sub["name"])
        except PermissionError:
            set_setting(f"sync_status_{sub_id}", "error:That instance has sharing disabled.")
            return
        except Exception as e:
            set_setting(f"sync_status_{sub_id}", f"error:Could not sync \u2014 {e}")
            return

        if all_urls is not None:
            existing = db.execute(
                "SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub_id,)
            ).fetchall()
            remote_url_set = set(all_urls)
            for row in existing:
                if row["url"] not in remote_url_set:
                    db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],))

        synced = 0
        for s in sites:
            try:
                tags_str = ",".join(s.get("tags", []))
                db.execute(
                    "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?) "
                    "ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags",
                    (sub_id, s["url"], s["title"], s.get("note", ""), tags_str),
                )
                if get_setting("semantic_search", "0") == "1":
                    try:
                        from embeddings import store_remote_embeddings
                        rp_id = db.execute(
                            "SELECT id FROM remote_pages WHERE subscription_id = ? AND url = ?",
                            (sub_id, s["url"]),
                        ).fetchone()["id"]
                        store_remote_embeddings(rp_id, s["title"], s.get("note", ""), db)
                    except Exception:
                        pass
                synced += 1
            except Exception:
                pass
        now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
        db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id))
        db.commit()
        set_setting(f"sync_status_{sub_id}", f"done:{synced}")
    except Exception as e:
        set_setting(f"sync_status_{sub_id}", f"error:{e}")
    finally:
        return_db(db)


def handle_subscription_sync(sub_id):
    if sub_id in _sync_threads and _sync_threads[sub_id].is_alive():
        return _redirect("/subscriptions")
    # Clear previous status
    set_setting(f"sync_status_{sub_id}", "syncing")
    t = threading.Thread(target=_sync_subscription, args=(sub_id,), daemon=True)
    _sync_threads[sub_id] = t
    t.start()
    return _redirect("/subscriptions")


def handle_subscription_autosync(sub_id):
    db = get_db()
    try:
        db.execute("UPDATE subscriptions SET auto_sync = 1 - auto_sync WHERE id = ?", (sub_id,))
        db.commit()
    finally:
        return_db(db)
    return _redirect("/subscriptions")


def handle_subscription_delete(sub_id):
    db = get_db()
    try:
        db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub_id,))
        db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,))
        db.commit()
    finally:
        return_db(db)
    return _redirect("/subscriptions")


def handle_subscription_syncall():
    db = get_db()
    try:
        subs = db.execute("SELECT * FROM subscriptions WHERE auto_sync = 1").fetchall()
    finally:
        return_db(db)
    if not subs:
        return handle_subscriptions("No subscriptions have auto-sync enabled.")
    for sub in subs:
        sub_id = sub["id"]
        if sub_id in _sync_threads and _sync_threads[sub_id].is_alive():
            continue
        set_setting(f"sync_status_{sub_id}", "syncing")
        t = threading.Thread(target=_sync_subscription, args=(sub_id,), daemon=True)
        _sync_threads[sub_id] = t
        t.start()
    return _redirect("/subscriptions")


# --- Reindex (semantic search) ---


_reindex_thread = None


def handle_reindex_form():
    if get_setting("semantic_search", "0") != "1":
        return _respond(
            f"<h2>semantic search index</h2>"
            f"<p>Semantic search is disabled. Enable it in <a href=\"/style\">settings</a> to use embeddings.</p>"
            f'<p><a href="/">back to search</a></p>'
        )
    db = get_db()
    try:
        total_pages = db.execute("SELECT count(*) FROM pages").fetchone()[0]
        pages_with_chunks = db.execute(
            "SELECT count(DISTINCT page_id) FROM chunks WHERE page_id IS NOT NULL"
        ).fetchone()[0]
    finally:
        return_db(db)
    progress = get_setting("reindex_progress", "")
    status_html = ""
    if progress:
        status_html = f'<p class="meta">Reindex in progress: {esc(progress)}</p>'
    elif _reindex_thread and _reindex_thread.is_alive():
        status_html = '<p class="meta">Reindex running...</p>'
    return _respond(
        f"<h2>semantic search index</h2>"
        f"<p>{pages_with_chunks} of {total_pages} pages have embeddings.</p>"
        f'{status_html}'
        f'<form method="post" action="/reindex">'
        f'{_csrf_field()}'
        f'<button type="submit">reindex all pages</button>'
        f'</form>'
        f'<p><a href="/">back to search</a></p>'
    )


def handle_reindex_submit(body):
    global _reindex_thread
    if _reindex_thread and _reindex_thread.is_alive():
        return handle_reindex_form()

    def _run():
        try:
            from embeddings import reindex_all
            def progress(current, total):
                set_setting("reindex_progress", f"{current}/{total}")
            reindex_all(progress_callback=progress)
        except Exception:
            pass
        finally:
            set_setting("reindex_progress", "")

    _reindex_thread = threading.Thread(target=_run, daemon=True)
    _reindex_thread.start()
    return _redirect("/reindex")


# --- Dispatcher ---


def _dispatch_inner(data):
    method = data.get("method", "GET")
    path = data.get("path", "/")
    query = data.get("query", {})
    body = data.get("body", {})
    gateway_host = data.get("gateway_host", "")

    def extract_id(prefix):
        try:
            return int(path[len(prefix):])
        except (ValueError, IndexError):
            return None

    if method == "GET":
        if path == "/":
            return handle_search(query)
        elif path == "/add":
            action_type = query.get("type", ["index"])[0]
            return handle_add_form(action_type=action_type if action_type == "subscribe" else "index")
        elif path == "/pages":
            return handle_pages(query)
        elif path.startswith("/edit/"):
            pid = extract_id("/edit/")
            return handle_edit_form(pid) if pid is not None else _error(400)
        elif path.startswith("/delete/"):
            pid = extract_id("/delete/")
            return handle_delete_confirm(pid) if pid is not None else _error(400)
        elif path == "/bookmark":
            return handle_bookmark(query)
        elif path == "/style":
            return handle_style_form()
        elif path == "/share/preview":
            return handle_share_preview()
        elif path == "/about":
            return handle_about()
        elif path == "/export":
            return handle_export(query)
        elif path == "/import":
            return handle_import_form()
        elif path == "/tags":
            return handle_tags()
        elif path.startswith("/tags/"):
            tag_name = unquote(path[len("/tags/"):])
            return handle_tag_browse(tag_name, query) if tag_name else _error(400)
        elif path == "/reindex":
            return handle_reindex_form()
        elif path == "/api/sites":
            return handle_api_sites(query)
        elif path == "/subscriptions":
            return handle_subscriptions()
        elif path.startswith("/subscriptions/browse/"):
            sid = extract_id("/subscriptions/browse/")
            return handle_subscription_browse(sid) if sid is not None else _error(400)
    elif method == "POST":
        if not _check_csrf(body):
            return _respond("<h1>403 Forbidden</h1><p>Invalid or missing CSRF token.</p>", status=403)
        if path == "/add":
            return handle_add_submit(body)
        elif path == "/pages/bulk":
            return handle_bulk_action(body)
        elif path == "/add/manual":
            return handle_add_manual_submit(body)
        elif path.startswith("/edit/"):
            pid = extract_id("/edit/")
            return handle_edit_submit(pid, body) if pid is not None else _error(400)
        elif path.startswith("/delete/"):
            pid = extract_id("/delete/")
            return handle_delete(pid) if pid is not None else _error(400)
        elif path == "/style":
            return handle_style_submit(body)
        elif path == "/style/reset":
            set_setting("custom_template", "")
            return handle_style_form("Template reset to default.")
        elif path == "/style/vacuum":
            from db import vacuum_db
            vacuum_db()
            return handle_style_form("Database vacuumed.")
        elif path == "/import":
            return handle_import_submit(body)
        elif path == "/reindex":
            return handle_reindex_submit(body)
        elif path == "/subscriptions/add":
            return handle_subscription_add(body)
        elif path == "/subscriptions/pick":
            return handle_subscription_pick(body)
        elif path.startswith("/subscriptions/sync/"):
            sid = extract_id("/subscriptions/sync/")
            return handle_subscription_sync(sid) if sid is not None else _error(400)
        elif path.startswith("/subscriptions/autosync/"):
            sid = extract_id("/subscriptions/autosync/")
            return handle_subscription_autosync(sid) if sid is not None else _error(400)
        elif path.startswith("/subscriptions/delete/"):
            sid = extract_id("/subscriptions/delete/")
            return handle_subscription_delete(sid) if sid is not None else _error(400)
        elif path == "/subscriptions/syncall":
            return handle_subscription_syncall()

    return _error(404)


def dispatch_request(data):
    cookies = data.get("cookies", {})
    csrf_token = cookies.get("_csrf", "")
    if not csrf_token:
        csrf_token = secrets.token_hex(32)
    _request_local.csrf_token = csrf_token

    resp = _dispatch_inner(data)

    resp.setdefault("headers", {})
    resp["headers"]["Set-Cookie"] = f"_csrf={csrf_token}; SameSite=Strict; HttpOnly; Path=/"
    resp["headers"]["X-Frame-Options"] = "DENY"
    resp["headers"]["X-Content-Type-Options"] = "nosniff"
    if resp.get("content_type", "").startswith("text/html"):
        resp["headers"]["Content-Security-Policy"] = (
            "default-src 'self'; "
            "script-src 'self' 'unsafe-inline'; "
            "style-src 'self' 'unsafe-inline'; "
            "img-src * data:; "
            "frame-ancestors 'none'; "
            "form-action 'self'; "
            "base-uri 'self'"
        )
    return resp