diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..031c6d8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +__pycache__/ +index.db* +tinyweb_identity +.git/ +*.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..98ade52 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,43 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What is TinyWeb + +A personal, decentralized search engine built on the Reticulum mesh network. Users curate and search their own index of web pages, share collections over an encrypted mesh, and subscribe to friends' indexes. No algorithms, no tracking. + +## Running + +```bash +pip install -r requirements.txt +python app.py # Starts RNS server + HTTP gateway on 0.0.0.0:8080 +python gateway.py # Run as HTTP gateway to a remote TinyWeb instance +``` + +There are no tests, linter, or build step. + +## Architecture + +Three entry points form a pipeline: + +- **app.py** — Boots Reticulum, loads/creates identity from `tinyweb_identity`, announces on mesh, starts HTTP gateway as a daemon thread, then loops handling RNS requests. +- **gateway.py** — `BaseHTTPRequestHandler` that translates HTTP GET/POST into a request dict and dispatches it. When `local_dispatch` is set (the default when launched from app.py), it calls handlers directly; otherwise it sends requests over a Reticulum link. +- **handlers.py** — Central router (`handle_request`) that pattern-matches the path and calls the appropriate handler. Every handler returns `{"status", "content_type", "body", "headers"}`. + +## Database (db.py → index.db) + +SQLite with FTS5. Schema is initialized and migrated in `init_db()` on every startup. + +Key tables: `pages` (indexed URLs), `links` (extracted same-domain links), `tags`/`page_tags` (many-to-many tagging), `pages_fts` (full-text search via triggers), `subscriptions` (remote instances), `remote_pages`/`remote_pages_fts` (synced content). + +`get_db()` opens a fresh connection each call — no connection pooling. + +## Patterns to follow + +- All HTML output is built as inline strings in handlers.py; there is no template engine. Use `templates.wrap_page(title, body_html)` to wrap content with boilerplate and custom CSS. +- Use `esc()` (html.escape) for all user-supplied content rendered in HTML. +- Handlers receive `(path_segment, ...)` args extracted by the router and return a response dict. +- Tags are stored in a join table; orphaned rows in `tags` can accumulate — always query through `page_tags` for accurate counts. +- Link extraction (`extract_links`) only follows same-domain URLs and skips binary file extensions and Wikipedia special pages. +- URL cleanup: fragments are stripped, tracking params (utm_*, fbclid, gclid, etc.) are removed before storing. +- Settings are stored as key-value pairs in the `settings` table; access via `get_setting(key, default)`. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9895f65 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +RUN mkdir -p /data \ + && ln -sf /data/index.db index.db \ + && ln -sf /data/tinyweb_identity tinyweb_identity + +ENV PYTHONUNBUFFERED=1 + +EXPOSE 8080 + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/README.md b/README.md index e69de29..ec66704 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,75 @@ +# TinyWeb + +A personal, decentralized search engine built on the [Reticulum](https://reticulum.network/) mesh network. Curate your own index of web pages, search it locally, and share collections with friends over an encrypted mesh. No algorithms, no ads, no tracking. + +## Features + +- **Personal search index** — Save pages you find valuable, search them with full-text search (SQLite FTS5) +- **Tagging** — Organize saved pages with comma-separated tags +- **Bookmarklet** — One-click indexing from any browser tab +- **Subscriptions** — Subscribe to friends' TinyWeb instances over Reticulum and search their indexes alongside yours +- **Custom templates** — Full HTML/CSS/JS template editor to personalize your instance +- **Import/export** — JSON-based backup and restore +- **Mesh-native** — Works over Reticulum without the internet; encrypted and decentralized by default + +## Getting started + +```bash +pip install -r requirements.txt +python app.py +``` + +This starts the Reticulum server and an HTTP gateway on `http://localhost:8080`. Open it in your browser. + +Your destination hash is printed on startup — share it with friends so they can subscribe to your index. + +## Remote gateway + +To browse a remote TinyWeb instance without running your own index: + +```bash +python gateway.py +``` + +This connects over Reticulum and serves the remote instance at `http://localhost:8080`. + +## How it works + +1. **Save pages** — Use the `/add` form or the bookmarklet (found on `/style`) to index any URL +2. **Search** — Full-text search across your saved pages, linked pages from trusted sites, and synced subscriptions +3. **Subscribe** — Add a friend's destination hash on `/subscriptions` to sync their shared index +4. **Customize** — Edit your site name, HTML template, and sharing settings on `/style` + +## Project structure + +``` +app.py — Entry point: boots Reticulum, starts HTTP gateway +gateway.py — HTTP-to-RNS bridge (local or remote dispatch) +handlers.py — Route dispatcher and all request handlers +db.py — SQLite database, FTS5, URL fetching, SSRF protection +templates.py — HTML template rendering and escaping +rns_client.py — Reticulum client for fetching remote site lists +themes/ — Saved HTML templates (e.g. kodama.html) +``` + +## Security + +TinyWeb includes several hardening measures: + +- **CSRF protection** — All POST forms use per-session tokens via double-submit cookies +- **SSRF prevention** — URL fetching validates hostnames against private IP ranges, with redirect re-validation +- **FTS5 injection prevention** — Search queries are sanitized before passing to SQLite MATCH +- **Content Security Policy** — CSP headers on all HTML responses restrict script/style/frame sources +- **XSS escaping** — All user-supplied content is HTML-escaped before rendering +- **Bookmark authentication** — The bookmarklet endpoint requires a secret token +- **Identity file protection** — The Reticulum identity key is restricted to owner-only permissions (0600) + +## Dependencies + +- [requests](https://docs.python-requests.org/) — HTTP fetching +- [beautifulsoup4](https://www.crummy.com/software/BeautifulSoup/) — HTML parsing and link extraction +- [rns](https://reticulum.network/) — Reticulum mesh networking + +## Philosophy + +TinyWeb is built for the slow web — intentionality over speed, human curation over algorithmic feeds, privacy over surveillance, and community over corporations. Every page in your index was saved because you found it valuable, not because an algorithm told you to click. diff --git a/app.py b/app.py index 6f520d2..a0bc4c6 100644 --- a/app.py +++ b/app.py @@ -11,13 +11,20 @@ from gateway import GatewayState, GatewayHandler, GATEWAY_PORT APP_NAME = "tinyweb" ASPECTS = ["server"] IDENTITY_FILE = "tinyweb_identity" +DEFAULT_TRANSPORT_HOST = "reticulum.derickphan.com" +DEFAULT_TRANSPORT_PORT = 4242 def load_or_create_identity(): if os.path.isfile(IDENTITY_FILE): + # Ensure identity file is only readable by owner + current = os.stat(IDENTITY_FILE).st_mode & 0o777 + if current != 0o600: + os.chmod(IDENTITY_FILE, 0o600) return RNS.Identity.from_file(IDENTITY_FILE) identity = RNS.Identity() identity.to_file(IDENTITY_FILE) + os.chmod(IDENTITY_FILE, 0o600) return identity @@ -35,9 +42,41 @@ def start_gateway(reticulum): thread.start() +def ensure_rns_config(config_dir): + """Generate a default Reticulum config with internet transport if none exists.""" + if config_dir is None: + config_dir = os.path.expanduser("~/.reticulum") + config_file = os.path.join(config_dir, "config") + if os.path.exists(config_file): + return + os.makedirs(config_dir, exist_ok=True) + with open(config_file, "w") as f: + f.write(f"""[reticulum] + enable_transport = False + share_instance = No + +[logging] + loglevel = 4 + +[interfaces] + [[Default Interface]] + type = AutoInterface + enabled = Yes + + [[TCP Transport]] + type = TCPClientInterface + enabled = yes + target_host = {DEFAULT_TRANSPORT_HOST} + target_port = {DEFAULT_TRANSPORT_PORT} +""") + print(f"Created Reticulum config at {config_file}") + + def main(): init_db() - reticulum = RNS.Reticulum() + config_dir = os.environ.get("RNS_CONFIG_DIR") + ensure_rns_config(config_dir) + reticulum = RNS.Reticulum(configdir=config_dir) identity = load_or_create_identity() destination = RNS.Destination( @@ -54,6 +93,8 @@ def main(): allow=RNS.Destination.ALLOW_ALL, ) + # Brief delay to ensure all interfaces (especially TCP) are fully ready + time.sleep(2) destination.announce() set_setting("dest_hash", destination.hash.hex()) start_gateway(reticulum) diff --git a/db.py b/db.py index b523c79..0d32b3a 100644 --- a/db.py +++ b/db.py @@ -1,10 +1,42 @@ +import socket +import ipaddress import sqlite3 import requests -from urllib.parse import urlparse, urljoin, parse_qs, urlencode, urlunparse +from urllib.parse import urlparse, urljoin, parse_qs, urlencode, urlunparse, quote from bs4 import BeautifulSoup DATABASE = "index.db" +BLOCKED_NETWORKS = [ + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("169.254.0.0/16"), + ipaddress.ip_network("0.0.0.0/8"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("fc00::/7"), + ipaddress.ip_network("fe80::/10"), +] + + +def _validate_url_target(url): + """Resolve hostname and block private/internal IPs to prevent SSRF.""" + parsed = urlparse(url) + hostname = parsed.hostname + port = parsed.port or (443 if parsed.scheme == "https" else 80) + if not hostname: + raise ValueError(f"No hostname in URL: {url}") + try: + addrs = socket.getaddrinfo(hostname, port, proto=socket.IPPROTO_TCP) + except socket.gaierror: + raise ValueError(f"Cannot resolve hostname: {hostname}") + for family, type_, proto, canonname, sockaddr in addrs: + ip = ipaddress.ip_address(sockaddr[0]) + for network in BLOCKED_NETWORKS: + if ip in network: + raise ValueError(f"URL resolves to blocked address: {ip}") + SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") TRACKING_PARAMS = { @@ -16,18 +48,64 @@ TRACKING_PARAMS = { def clean_url(url): parsed = urlparse(url) + + # Prefer https + scheme = "https" if parsed.scheme in ("http", "https") else parsed.scheme + + # Normalize hostname: lowercase, strip www. + hostname = (parsed.hostname or "").lower() + if hostname.startswith("www."): + hostname = hostname[4:] + + # Preserve explicit non-default ports + port = parsed.port + if port and ((scheme == "https" and port == 443) or (scheme == "http" and port == 80)): + port = None + netloc = f"{hostname}:{port}" if port else hostname + + # Strip trailing slash (keep root "/" as-is) + path = parsed.path.rstrip("/") or "/" + + # Remove tracking params and sort remaining for consistent ordering params = parse_qs(parsed.query) - cleaned = {k: v for k, v in params.items() if k.lower() not in TRACKING_PARAMS} - new_query = urlencode(cleaned, doseq=True) - return urlunparse(parsed._replace(query=new_query)) + cleaned = sorted( + ((k, sorted(v)) for k, v in params.items() if k.lower() not in TRACKING_PARAMS), + key=lambda x: x[0], + ) + new_query = urlencode(cleaned, doseq=True, quote_via=quote) + + return urlunparse((scheme, netloc, path, "", new_query, "")) + + +_pool = [] +_pool_lock = __import__("threading").Lock() +_POOL_SIZE = 4 def get_db(): - db = sqlite3.connect(DATABASE) + with _pool_lock: + if _pool: + db = _pool.pop() + try: + db.execute("SELECT 1") + return db + except Exception: + pass + db = sqlite3.connect(DATABASE, timeout=10) + db.execute("PRAGMA journal_mode=WAL") + db.execute("PRAGMA foreign_keys = ON") db.row_factory = sqlite3.Row return db +def return_db(db): + with _pool_lock: + if len(_pool) < _POOL_SIZE: + _pool.append(db) + else: + db.close() + + def init_db(): db = sqlite3.connect(DATABASE) db.execute( @@ -36,7 +114,8 @@ def init_db(): " url TEXT UNIQUE NOT NULL," " title TEXT," " body TEXT," - " note TEXT DEFAULT ''" + " note TEXT DEFAULT ''," + " last_modified TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now'))" ")" ) db.execute( @@ -140,26 +219,38 @@ def init_db(): db.execute("ALTER TABLE remote_pages ADD COLUMN tags TEXT DEFAULT ''") db.commit() + # Migrate pages: add last_modified column if missing + page_cols = [row[1] for row in db.execute("PRAGMA table_info(pages)").fetchall()] + if "last_modified" not in page_cols: + db.execute("ALTER TABLE pages ADD COLUMN last_modified TEXT DEFAULT ''") + db.execute("UPDATE pages SET last_modified = strftime('%Y-%m-%dT%H:%M:%S','now') WHERE last_modified = ''") + db.commit() + + db.execute("PRAGMA journal_mode=WAL") db.commit() db.close() def get_setting(key, default=""): db = get_db() - row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() - db.close() - return row["value"] if row else default + try: + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + return row["value"] if row else default + finally: + return_db(db) def set_setting(key, value): db = get_db() - db.execute( - "INSERT INTO settings (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value=excluded.value", - (key, value), - ) - db.commit() - db.close() + try: + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + finally: + return_db(db) def get_site_name(): @@ -167,7 +258,19 @@ def get_site_name(): def fetch_page(url): - resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) + _validate_url_target(url) + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False) + # Follow redirects manually, re-validating each target + max_redirects = 5 + while resp.is_redirect and max_redirects > 0: + redirect_url = resp.headers.get("Location") + if not redirect_url: + break + redirect_url = urljoin(url, redirect_url) + _validate_url_target(redirect_url) + url = redirect_url + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False) + max_redirects -= 1 resp.raise_for_status() soup = BeautifulSoup(resp.text, "html.parser") @@ -204,18 +307,22 @@ def index_url(url, note=""): url = clean_url(url) title, body, links = fetch_page(url) db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: + try: + now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), + "INSERT INTO pages (url, title, body, note, last_modified) VALUES (?, ?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, " + "note=excluded.note, last_modified=excluded.last_modified", + (url, title, body, note, now), ) - db.commit() - db.close() + page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0] + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + finally: + return_db(db) return title diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..151a79c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +services: + tinyweb: + build: . + ports: + - "8080:8080" + volumes: + - tinyweb-data:/data + restart: unless-stopped + # Connect to another Reticulum instance over TCP. + # Required on macOS (Docker can't do LAN auto-discovery). + # On Linux, auto-discovery works with network_mode: host. + # environment: + # - RNS_TCP_HOST=10.0.0.100 + # - RNS_TCP_PORT=4242 + +volumes: + tinyweb-data: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..e4a9719 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# Generate Reticulum config with optional TCP peer +# Set RNS_TCP_HOST and RNS_TCP_PORT env vars to connect to a remote instance + +CONFIG_DIR="/data/.reticulum" +CONFIG_FILE="$CONFIG_DIR/config" + +mkdir -p "$CONFIG_DIR" + +if [ ! -f "$CONFIG_FILE" ]; then + cat > "$CONFIG_FILE" <' + + +def _check_csrf(body): + token = body.get("_csrf", [""])[0] + expected = _get_csrf_token() + if not expected or not token: + return False + return secrets.compare_digest(token, expected) + + +def _sanitize_fts_query(query): + """Escape user input for safe use in FTS5 MATCH.""" + escaped = query.replace('"', '""') + return f'"{escaped}"' + + +def _get_bookmark_token(): + token = get_setting("bookmark_token") + if not token: + token = secrets.token_hex(16) + set_setting("bookmark_token", token) + return token + + +def _respond(body_html, status=200, use_default=False): return { "status": status, "content_type": "text/html; charset=utf-8", - "body": wrap_page(body_html), + "body": wrap_page(body_html, use_default=use_default), "headers": {}, } def _redirect(location): + if not location.startswith("/") or location.startswith("//"): + location = "/" return { "status": 302, "content_type": "text/html; charset=utf-8", @@ -46,6 +83,31 @@ def _error(status): return _respond(f"

{status}

", status) +PER_PAGE = 10 + + +def _paginate(query, key="p"): + try: + page = int(query.get(key, ["1"])[0]) + except (ValueError, IndexError): + page = 1 + return max(1, page) + + +def _page_nav(page, total, base_url): + if total <= PER_PAGE: + return "" + total_pages = (total + PER_PAGE - 1) // PER_PAGE + sep = "&" if "?" in base_url else "?" + parts = [] + if page > 1: + parts.append(f'« prev') + parts.append(f"page {page} of {total_pages}") + if page < total_pages: + parts.append(f'next »') + return f'

{" | ".join(parts)}

' + + # --- Tag helpers --- @@ -59,7 +121,7 @@ def _get_page_tags(page_id, db=None): "WHERE pt.page_id = ? ORDER BY t.name", (page_id,) ).fetchall() if close: - db.close() + return_db(db) return [r["name"] for r in rows] @@ -75,7 +137,7 @@ def _set_page_tags(page_id, tag_string, db=None): db.execute("INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", (page_id, tag_id)) if close: db.commit() - db.close() + return_db(db) # --- Route handlers --- @@ -83,122 +145,132 @@ def _set_page_tags(page_id, tag_string, db=None): def handle_search(query): q = query.get("q", [""])[0].strip() + page = _paginate(query) + offset = (page - 1) * PER_PAGE db = get_db() - count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - name = get_site_name() + try: + count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + name = get_site_name() - result_html = "" - trusted_html = "" - if q: - rows = db.execute( - "SELECT p.id, p.url, p.title, p.body, p.note " - "FROM pages_fts f JOIN pages p ON f.rowid = p.id " - "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", - (q,), - ).fetchall() - if rows: - for r in rows: - note_html = "" - if r["note"]: - note_html = f'
{esc(r["note"])}
' - tags = _get_page_tags(r["id"], db) - tags_html = "" - if tags: - tag_links = " ".join(f'[{esc(t)}]' for t in tags) - tags_html = f'
{tag_links}
' - result_html += ( - f'
' - f'{esc(r["title"])}
' - f'{esc(r["url"])}
' - f'{esc(snippet(r["body"], q))}' - f'{note_html}{tags_html}' - f'
' + result_html = "" + trusted_html = "" + if q: + try: + total_results = db.execute( + "SELECT count(*) FROM pages_fts WHERE pages_fts MATCH ?", + (_sanitize_fts_query(q),), + ).fetchone()[0] + rows = db.execute( + "SELECT p.id, p.url, p.title, p.body, p.note " + "FROM pages_fts f JOIN pages p ON f.rowid = p.id " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT ? OFFSET ?", + (_sanitize_fts_query(q), PER_PAGE, offset), + ).fetchall() + except Exception: + rows = [] + total_results = 0 + if rows: + for r in rows: + note_html = "" + if r["note"]: + note_html = f'
{esc(r["note"])}
' + tags = _get_page_tags(r["id"], db) + tags_html = "" + if tags: + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + tags_html = f'
{tag_links}
' + result_html += ( + f'
' + f'{esc(r["title"])}
' + f'{esc(r["url"])}
' + f'{esc(snippet(r["body"], q))}' + f'{note_html}{tags_html}' + f'
' + ) + else: + result_html = "

No results in your index.

" + + # search all linked pages from trusted sites + words = q.lower().split() + all_links = db.execute( + "SELECT l.url, l.label, p.title AS source_title " + "FROM links l JOIN pages p ON l.page_id = p.id", + ).fetchall() + indexed_urls = set(r["url"] for r in rows) if rows else set() + seen = set() + trusted = [] + for l in all_links: + if l["url"] in indexed_urls or l["url"] in seen: + continue + if any(w in l["label"].lower() for w in words): + seen.add(l["url"]) + trusted.append(l) + if len(trusted) >= 20: + break + + if trusted: + items = "" + for l in trusted: + items += ( + f'
  • {esc(l["label"])} ' + f'— from {esc(l["source_title"])}
    • ' + ) + trusted_html = ( + f'
    ' + f'from your trusted sites ({len(trusted)})' + f'
      {items}
    ' + f'
    ' ) - else: - result_html = "

    No results in your index.

    " - # search all linked pages from trusted sites - words = q.lower().split() - all_links = db.execute( - "SELECT l.url, l.label, p.title AS source_title " - "FROM links l JOIN pages p ON l.page_id = p.id", - ).fetchall() - indexed_urls = set(r["url"] for r in rows) if rows else set() - seen = set() - trusted = [] - for l in all_links: - if l["url"] in indexed_urls or l["url"] in seen: - continue - if any(w in l["label"].lower() for w in words): - seen.add(l["url"]) - trusted.append(l) - if len(trusted) >= 20: - break + # search synced pages from subscriptions + try: + remote_rows = db.execute( + "SELECT rp.url, rp.title, rp.note, s.name AS source_name " + "FROM remote_pages_fts rpf " + "JOIN remote_pages rp ON rpf.rowid = rp.id " + "JOIN subscriptions s ON rp.subscription_id = s.id " + "WHERE remote_pages_fts MATCH ? ORDER BY rank LIMIT 50", + (_sanitize_fts_query(q),), + ).fetchall() + except Exception: + remote_rows = [] - if trusted: - items = "" - for l in trusted: - items += ( - f'
  • {esc(l["label"])} ' - f'— from {esc(l["source_title"])}
    • ' + remote_html = "" + if q and remote_rows: + # group by source + by_source = {} + for r in remote_rows: + source = r["source_name"] or "unknown" + by_source.setdefault(source, []).append(r) + for source, items in by_source.items(): + source_items = "" + for r in items: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + source_items += ( + f'
  • {esc(r["title"])}' + f'{note_html} ({esc(r["url"])})
    • ' + ) + remote_html += ( + f'
    ' + f'from {esc(source)} ({len(items)})' + f'
      {source_items}
    ' + f'
    ' ) - trusted_html = ( - f'
    ' - f'from your trusted sites ({len(trusted)})' - f'
      {items}
    ' - f'
    ' - ) - - # search synced pages from subscriptions - remote_rows = db.execute( - "SELECT rp.url, rp.title, rp.note, s.name AS source_name " - "FROM remote_pages_fts rpf " - "JOIN remote_pages rp ON rpf.rowid = rp.id " - "JOIN subscriptions s ON rp.subscription_id = s.id " - "WHERE remote_pages_fts MATCH ? ORDER BY rank LIMIT 50", - (q,), - ).fetchall() - - remote_html = "" - if q and remote_rows: - # group by source - by_source = {} - for r in remote_rows: - source = r["source_name"] or "unknown" - by_source.setdefault(source, []).append(r) - for source, items in by_source.items(): - source_items = "" - for r in items: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - source_items += ( - f'
  • {esc(r["title"])}' - f'{note_html} ({esc(r["url"])})
    • ' - ) - remote_html += ( - f'
    ' - f'from {esc(source)} ({len(items)})' - f'
      {source_items}
    ' - f'
    ' - ) - - db.close() + finally: + return_db(db) sub_count = "" if q and remote_rows: sub_count = f" + {len(remote_rows)} from subscriptions" return _respond( - f'

    {esc(name)}

    ' f'
    ' f'' f' ' f'
    ' - f'

    {count} page(s) indexed.' - f' + add url' - f' | browse' - f' | tags' - f' | subscriptions' - f' | customize' - f' | about

    ' - f'
    {result_html}{trusted_html}{remote_html}' + f'

    {count} pages indexed' + f' · + add url

    ' + f'{result_html}' + f'{_page_nav(page, total_results, f"/?q={esc(q)}") if q else ""}' + f'{trusted_html}{remote_html}' ) @@ -206,6 +278,7 @@ def handle_add_form(msg=""): return _respond( f"

    add url

    " f'
    ' + f'{_csrf_field()}' f'

    ' f'

    ' f'

    ' @@ -228,37 +301,50 @@ def handle_add_submit(body): title = index_url(url, note) if tags: db = get_db() - row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() - if row: - _set_page_tags(row["id"], tags, db) - db.commit() - db.close() + try: + row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() + if row: + _set_page_tags(row["id"], tags, db) + db.commit() + finally: + return_db(db) return handle_add_form(f'Indexed: {esc(title)}') - except Exception as e: + except ValueError as e: return handle_add_form(f"Error: {esc(str(e))}") + except Exception: + return handle_add_form("Error: could not fetch or index that URL.") -def handle_pages(): +def handle_pages(query=None): + page = _paginate(query or {}) + offset = (page - 1) * PER_PAGE db = get_db() - rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() - items = "" - for r in rows: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - tags = _get_page_tags(r["id"], db) - tags_html = "" - if tags: - tag_links = " ".join(f'[{esc(t)}]' for t in tags) - tags_html = f' {tag_links}' - items += ( - f'
  • {esc(r["title"])}{note_html}{tags_html} ' - f'({esc(r["url"])}) ' - f'edit ' - f'remove
    • ' - ) - db.close() + try: + total = db.execute("SELECT count(*) FROM pages").fetchone()[0] + rows = db.execute( + "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?", + (PER_PAGE, offset), + ).fetchall() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tags_html = "" + if tags: + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + tags_html = f' {tag_links}' + items += ( + f'
  • {esc(r["title"])}{note_html}{tags_html} ' + f'({esc(r["url"])}) ' + f'edit ' + f'remove
    • ' + ) + finally: + return_db(db) return _respond( - f"

    indexed pages ({len(rows)})

    " + f"

    indexed pages ({total})

    " f"
      {items}
    " + f'{_page_nav(page, total, "/pages")}' f'

    export | import

    ' f'back' ) @@ -266,17 +352,19 @@ def handle_pages(): def handle_edit_form(page_id, msg=""): db = get_db() - row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() - if not row: - db.close() - return _error(404) - tags = ", ".join(_get_page_tags(page_id, db)) - db.close() + try: + row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() + if not row: + return _error(404) + tags = ", ".join(_get_page_tags(page_id, db)) + finally: + return_db(db) return _respond( f"

    edit page

    " f"

    {esc(row['title'])}
    " f"{esc(row['url'])}

    " f'' + f'{_csrf_field()}' f'

    ' f'

    ' f'' @@ -290,23 +378,52 @@ def handle_edit_submit(page_id, body): note = body.get("note", [""])[0].strip() tags = body.get("tags", [""])[0].strip() db = get_db() - db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) - _set_page_tags(page_id, tags, db) - db.commit() - db.close() + try: + db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) + _set_page_tags(page_id, tags, db) + db.commit() + finally: + return_db(db) return _redirect("/pages") +def handle_delete_confirm(page_id): + db = get_db() + try: + row = db.execute("SELECT id, url, title FROM pages WHERE id = ?", (page_id,)).fetchone() + finally: + return_db(db) + if not row: + return _error(404) + return _respond( + f"

    confirm delete

    " + f"

    Remove {esc(row['title'])}
    " + f"{esc(row['url'])}

    " + f'' + f'{_csrf_field()}' + f'' + f"
    " + f' cancel' + ) + + def handle_delete(page_id): db = get_db() - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) - db.commit() - db.close() + try: + db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,)) + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) + db.commit() + finally: + return_db(db) return _redirect("/pages") def handle_bookmark(query): + token = query.get("token", [""])[0] + expected = _get_bookmark_token() + if not token or not secrets.compare_digest(token, expected): + return _text_response("error: invalid or missing token", status=403, headers={"Access-Control-Allow-Origin": "*"}) url = clean_url(query.get("url", [""])[0].strip()) if not url or not url.startswith(("http://", "https://")): return _text_response("error: invalid url", headers={"Access-Control-Allow-Origin": "*"}) @@ -320,8 +437,10 @@ def handle_bookmark(query): def handle_export(): db = get_db() - rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() - db.close() + try: + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() + finally: + return_db(db) data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] return _json_response(data, headers={"Content-Disposition": "attachment; filename=tinyweb-export.json"}) @@ -331,6 +450,7 @@ def handle_import_form(msg=""): f"

    import

    " f"

    Paste the contents of a tinyweb export file (JSON).

    " f'
    ' + f'{_csrf_field()}' f'

    ' f'' f"
    " @@ -350,6 +470,10 @@ def handle_import_submit(body): if not isinstance(data, list): return handle_import_form("Expected a JSON array.") + MAX_IMPORT = 100 + if len(data) > MAX_IMPORT: + return handle_import_form(f"Too many entries. Maximum is {MAX_IMPORT}.") + imported = 0 errors = 0 for entry in data: @@ -367,7 +491,7 @@ def handle_import_submit(body): def handle_style_form(msg=""): - css = get_setting("custom_css") + template = get_setting("custom_template") or DEFAULT_TEMPLATE name = get_site_name() sharing = get_setting("sharing_enabled", "0") checked = " checked" if sharing == "1" else "" @@ -375,39 +499,36 @@ def handle_style_form(msg=""): f"

    customize

    " f"

    name your search engine

    " f'
    ' + f'{_csrf_field()}' f'

    ' f"

    sharing

    " f'

    " - f"

    custom css

    " - f"

    Some classes you can target:

    " - f"
    "
-        f"body          - page background, font\n"
-        f"h1            - page titles\n"
-        f"input, button - search bar\n"
-        f"a             - links\n"
-        f".result       - each search result\n"
-        f".note         - your notes on results\n"
-        f".trusted      - trusted sites dropdown\n"
-        f"small         - url text\n"
-        f"ul, li        - browse page list"
-        f"
    " - f'

    ' + f"

    custom html

    " + f"

    Edit the full page template. Use {esc('{{content}}')} " + f"where page content should appear.

    " + f'

    ' f'' f"
    " f"

    bookmarklet

    " f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " - f'

    + save to {esc(name)}

    ' + f'

    + save to {esc(name)}

    ' + f"

    reset

    " + f'
    ' + f'{_csrf_field()}' + f'' + f"
    " f"

    {msg}

    " - f'back' + f'back', + use_default=True, ) def handle_style_submit(body): - css = body.get("css", [""])[0] + template = body.get("template", [""])[0].replace("\r\n", "\n").replace("\r", "\n") name = body.get("site_name", ["tinyweb"])[0].strip() sharing = "1" if body.get("sharing_enabled") else "0" - set_setting("custom_css", css) + set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "") set_setting("site_name", name or "tinyweb") set_setting("sharing_enabled", sharing) return handle_style_form("Saved.") @@ -418,10 +539,12 @@ def handle_about(): dest_hash = get_setting("dest_hash") sharing = get_setting("sharing_enabled", "0") == "1" db = get_db() - page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - tag_count = db.execute("SELECT count(DISTINCT tag_id) FROM page_tags").fetchone()[0] - sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0] - db.close() + try: + page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + tag_count = db.execute("SELECT count(DISTINCT tag_id) FROM page_tags").fetchone()[0] + sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0] + finally: + return_db(db) sharing_html = ( '

    This instance shares its index publicly. Subscribe to join the network.

    ' @@ -467,12 +590,14 @@ def handle_about(): def handle_tags(): db = get_db() - rows = db.execute( - "SELECT t.name, COUNT(pt.page_id) AS cnt FROM tags t " - "JOIN page_tags pt ON t.id = pt.tag_id " - "GROUP BY t.id ORDER BY t.name" - ).fetchall() - db.close() + try: + rows = db.execute( + "SELECT t.name, COUNT(pt.page_id) AS cnt FROM tags t " + "JOIN page_tags pt ON t.id = pt.tag_id " + "GROUP BY t.id ORDER BY t.name" + ).fetchall() + finally: + return_db(db) items = "" for r in rows: items += f'
  • {esc(r["name"])} ({r["cnt"]})
    • ' @@ -483,92 +608,119 @@ def handle_tags(): ) -def handle_tag_browse(tag_name): +def handle_tag_browse(tag_name, query=None): + page = _paginate(query or {}) + offset = (page - 1) * PER_PAGE db = get_db() - rows = db.execute( - "SELECT p.id, p.url, p.title, p.note FROM pages p " - "JOIN page_tags pt ON p.id = pt.page_id " - "JOIN tags t ON t.id = pt.tag_id " - "WHERE t.name = ? ORDER BY p.id DESC", - (tag_name,), - ).fetchall() - items = "" - for r in rows: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - tags = _get_page_tags(r["id"], db) - tag_links = " ".join(f'[{esc(t)}]' for t in tags) - items += ( - f'
  • {esc(r["title"])}{note_html} {tag_links} ' - f'({esc(r["url"])})
    • ' - ) - db.close() + try: + total = db.execute( + "SELECT count(*) FROM page_tags pt JOIN tags t ON t.id = pt.tag_id WHERE t.name = ?", + (tag_name,), + ).fetchone()[0] + rows = db.execute( + "SELECT p.id, p.url, p.title, p.note FROM pages p " + "JOIN page_tags pt ON p.id = pt.page_id " + "JOIN tags t ON t.id = pt.tag_id " + "WHERE t.name = ? ORDER BY p.id DESC LIMIT ? OFFSET ?", + (tag_name, PER_PAGE, offset), + ).fetchall() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + items += ( + f'
  • {esc(r["title"])}{note_html} {tag_links} ' + f'({esc(r["url"])})
    • ' + ) + finally: + return_db(db) return _respond( f'

    tag: {esc(tag_name)}

    ' - f'

    {len(rows)} page(s)

    ' + f'

    {total} page(s)

    ' f'
      {items}
    ' + f'{_page_nav(page, total, f"/tags/{esc(tag_name)}")}' f'all tags | back' ) -def handle_api_sites(): +def handle_api_sites(query=None): if get_setting("sharing_enabled", "0") != "1": return _json_response( {"error": "sharing disabled"}, status=403, headers={"Access-Control-Allow-Origin": "*"}, ) + since = (query or {}).get("since", [""])[0].strip() db = get_db() - rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() - sites = [] - for r in rows: - tags = _get_page_tags(r["id"], db) - sites.append({"url": r["url"], "title": r["title"], "note": r["note"], "tags": tags}) - db.close() + try: + if since: + rows = db.execute( + "SELECT id, url, title, note, last_modified FROM pages " + "WHERE last_modified > ? ORDER BY id DESC", + (since,), + ).fetchall() + else: + rows = db.execute("SELECT id, url, title, note, last_modified FROM pages ORDER BY id DESC").fetchall() + sites = [] + for r in rows: + tags = _get_page_tags(r["id"], db) + sites.append({ + "url": r["url"], "title": r["title"], "note": r["note"], + "tags": tags, "last_modified": r["last_modified"] or "", + }) + # Include list of all current URLs so subscriber can detect deletions + all_urls = [r["url"] for r in db.execute("SELECT url FROM pages").fetchall()] if not since else None + finally: + return_db(db) data = {"name": get_site_name(), "sites": sites} + if all_urls is not None: + data["all_urls"] = all_urls return _json_response(data, headers={"Access-Control-Allow-Origin": "*"}) def handle_subscriptions(msg=""): db = get_db() - subs = db.execute("SELECT * FROM subscriptions ORDER BY id DESC").fetchall() - db.close() - items = "" + try: + subs = db.execute("SELECT * FROM subscriptions ORDER BY id DESC").fetchall() + finally: + return_db(db) + cards = "" for s in subs: auto_label = "on" if s["auto_sync"] else "off" last = s["last_sync"] or "never" - items += ( - f'' - f'{esc(s["name"] or "unknown")}
    {esc(s["dest_hash"])}' - f'{esc(last)}' - f'' - f'
    ' - f'
    ' - f'' - f'' - f'browse ' + cards += ( + f'
    ' + f'
    {esc(s["name"] or "unknown")}
    ' + f'
    {esc(s["dest_hash"])}
    ' + f'
    last sync: {esc(last)}
    ' + f'
    ' + f'browse' f'
    ' - f'
    ' + f'{_csrf_field()}' + f'
    ' + f'{_csrf_field()}
    ' f'
    ' - f'
    ' - f'' - f'' + f'{_csrf_field()}' + f'
    ' + f'
    ' ) - table = "" + listing = "" if subs: - table = ( - f'' - f'{items}
    instancelast syncauto-syncactions
    ' + listing = ( + f'{cards}' f'
    ' - f'
    ' + f'{_csrf_field()}' ) return _respond( f"

    subscriptions

    " f'
    ' + f'{_csrf_field()}' f' ' f'' f'
    ' f'

    {msg}

    ' - f'
    {table}' + f'
    {listing}' f'
    back' ) @@ -586,8 +738,8 @@ def handle_subscription_add(body): name = data.get("name", "") except PermissionError: return handle_subscriptions("That instance has sharing disabled.") - except Exception as e: - return handle_subscriptions(f"Could not reach that instance: {esc(str(e))}") + except Exception: + return handle_subscriptions("Could not reach that instance.") db = get_db() try: db.execute( @@ -597,24 +749,25 @@ def handle_subscription_add(body): ) db.commit() finally: - db.close() + return_db(db) return handle_subscriptions(f"Subscribed to {esc(name or dest_hash)}.") def handle_subscription_browse(sub_id): db = get_db() - sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() - if not sub: - db.close() - return _error(404) - local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + try: + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + if not sub: + return _error(404) + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) - # Use locally synced data if available, otherwise fetch live - remote_rows = db.execute( - "SELECT url, title, note, tags FROM remote_pages WHERE subscription_id = ?", - (sub_id,), - ).fetchall() - db.close() + # Use locally synced data if available, otherwise fetch live + remote_rows = db.execute( + "SELECT url, title, note, tags FROM remote_pages WHERE subscription_id = ?", + (sub_id,), + ).fetchall() + finally: + return_db(db) if remote_rows: sites = [] @@ -627,8 +780,8 @@ def handle_subscription_browse(sub_id): sites = data.get("sites", []) except PermissionError: return handle_subscriptions("That instance has sharing disabled.") - except Exception as e: - return handle_subscriptions(f"Could not fetch sites: {esc(str(e))}") + except Exception: + return handle_subscriptions("Could not fetch sites from that instance.") new_items = "" existing_items = "" @@ -658,6 +811,7 @@ def handle_subscription_browse(sub_id): f'

    browsing: {esc(sub["name"] or sub["dest_hash"])}

    ' f'

    {len(sites)} site(s) available, {new_count} new

    ' f'
    ' + f'{_csrf_field()}' f'' f'
      {new_items}
    ' f'{buttons}' @@ -673,17 +827,19 @@ def handle_subscription_pick(body): # Build a url->tags map from remote_pages for this subscription db = get_db() - remote_rows = db.execute( - "SELECT url, tags FROM remote_pages WHERE subscription_id = ?", (sub_id,) - ).fetchall() - remote_tags = {r["url"]: r["tags"] for r in remote_rows} + try: + remote_rows = db.execute( + "SELECT url, tags FROM remote_pages WHERE subscription_id = ?", (sub_id,) + ).fetchall() + remote_tags = {r["url"]: r["tags"] for r in remote_rows} - if import_all: - local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) - urls = [r["url"] for r in remote_rows if r["url"] not in local_urls] - else: - urls = body.get("urls", []) - db.close() + if import_all: + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + urls = [r["url"] for r in remote_rows if r["url"] not in local_urls] + else: + urls = body.get("urls", []) + finally: + return_db(db) if not urls: return handle_subscriptions("No sites selected.") @@ -697,11 +853,13 @@ def handle_subscription_pick(body): tags_str = remote_tags.get(url, "") if tags_str: db = get_db() - row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() - if row: - _set_page_tags(row["id"], tags_str, db) - db.commit() - db.close() + try: + row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() + if row: + _set_page_tags(row["id"], tags_str, db) + db.commit() + finally: + return_db(db) imported += 1 except Exception: errors += 1 @@ -710,84 +868,115 @@ def handle_subscription_pick(body): def handle_subscription_sync(sub_id): db = get_db() - sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() - if not sub: - db.close() - return handle_subscriptions("Subscription not found.") try: - data = fetch_remote_sites(sub["dest_hash"]) - sites = data.get("sites", []) - remote_name = data.get("name", sub["name"]) - except PermissionError: - db.close() - return handle_subscriptions("That instance has sharing disabled.") - except Exception as e: - db.close() - return handle_subscriptions(f"Could not sync: {esc(str(e))}") - - # Clear old remote pages for this subscription and re-insert - db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub_id,)) - synced = 0 - for s in sites: + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + if not sub: + return handle_subscriptions("Subscription not found.") + # Use last_sync for delta sync if available + since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else "" try: - tags_str = ",".join(s.get("tags", [])) - db.execute( - "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?)", - (sub_id, s["url"], s["title"], s.get("note", ""), tags_str), - ) - synced += 1 + data = fetch_remote_sites(sub["dest_hash"], since=since) + sites = data.get("sites", []) + all_urls = data.get("all_urls") + remote_name = data.get("name", sub["name"]) + except PermissionError: + return handle_subscriptions("That instance has sharing disabled.") except Exception: - pass - now = datetime.now().strftime("%Y-%m-%d %H:%M") - db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id)) - db.commit() - db.close() + return handle_subscriptions("Could not sync with that instance.") + + # If full sync (all_urls provided), remove pages no longer on remote + if all_urls is not None: + existing = db.execute( + "SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub_id,) + ).fetchall() + remote_url_set = set(all_urls) + for row in existing: + if row["url"] not in remote_url_set: + db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],)) + + # Upsert changed/new pages + synced = 0 + for s in sites: + try: + tags_str = ",".join(s.get("tags", [])) + db.execute( + "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?) " + "ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags", + (sub_id, s["url"], s["title"], s.get("note", ""), tags_str), + ) + synced += 1 + except Exception: + pass + now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id)) + db.commit() + finally: + return_db(db) return handle_subscriptions(f"Synced {synced} site(s) from {esc(remote_name)}.") def handle_subscription_autosync(sub_id): db = get_db() - db.execute("UPDATE subscriptions SET auto_sync = 1 - auto_sync WHERE id = ?", (sub_id,)) - db.commit() - db.close() + try: + db.execute("UPDATE subscriptions SET auto_sync = 1 - auto_sync WHERE id = ?", (sub_id,)) + db.commit() + finally: + return_db(db) return _redirect("/subscriptions") def handle_subscription_delete(sub_id): db = get_db() - db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,)) - db.commit() - db.close() + try: + db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub_id,)) + db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,)) + db.commit() + finally: + return_db(db) return _redirect("/subscriptions") def handle_subscription_syncall(): db = get_db() - subs = db.execute("SELECT * FROM subscriptions WHERE auto_sync = 1").fetchall() - db.close() + try: + subs = db.execute("SELECT * FROM subscriptions WHERE auto_sync = 1").fetchall() + finally: + return_db(db) if not subs: return handle_subscriptions("No subscriptions have auto-sync enabled.") total = 0 for sub in subs: try: - data = fetch_remote_sites(sub["dest_hash"]) + since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else "" + data = fetch_remote_sites(sub["dest_hash"], since=since) sites = data.get("sites", []) + all_urls = data.get("all_urls") remote_name = data.get("name", sub["name"]) db = get_db() - db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub["id"],)) - for s in sites: - try: - tags_str = ",".join(s.get("tags", [])) - db.execute( - "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?)", - (sub["id"], s["url"], s["title"], s.get("note", ""), tags_str), - ) - except Exception: - pass - now = datetime.now().strftime("%Y-%m-%d %H:%M") - db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub["id"])) - db.commit() - db.close() + try: + if all_urls is not None: + existing = db.execute( + "SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub["id"],) + ).fetchall() + remote_url_set = set(all_urls) + for row in existing: + if row["url"] not in remote_url_set: + db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],)) + for s in sites: + try: + tags_str = ",".join(s.get("tags", [])) + db.execute( + "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?) " + "ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags", + (sub["id"], s["url"], s["title"], s.get("note", ""), tags_str), + ) + except Exception: + pass + now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub["id"])) + db.commit() + finally: + return_db(db) total += 1 except Exception: pass @@ -797,7 +986,7 @@ def handle_subscription_syncall(): # --- Dispatcher --- -def dispatch_request(data): +def _dispatch_inner(data): method = data.get("method", "GET") path = data.get("path", "/") query = data.get("query", {}) @@ -816,13 +1005,13 @@ def dispatch_request(data): elif path == "/add": return handle_add_form() elif path == "/pages": - return handle_pages() + return handle_pages(query) elif path.startswith("/edit/"): pid = extract_id("/edit/") return handle_edit_form(pid) if pid is not None else _error(400) elif path.startswith("/delete/"): pid = extract_id("/delete/") - return handle_delete(pid) if pid is not None else _error(400) + return handle_delete_confirm(pid) if pid is not None else _error(400) elif path == "/bookmark": return handle_bookmark(query) elif path == "/style": @@ -836,23 +1025,31 @@ def dispatch_request(data): elif path == "/tags": return handle_tags() elif path.startswith("/tags/"): - tag_name = path[len("/tags/"):] - return handle_tag_browse(tag_name) if tag_name else _error(400) + tag_name = unquote(path[len("/tags/"):]) + return handle_tag_browse(tag_name, query) if tag_name else _error(400) elif path == "/api/sites": - return handle_api_sites() + return handle_api_sites(query) elif path == "/subscriptions": return handle_subscriptions() elif path.startswith("/subscriptions/browse/"): sid = extract_id("/subscriptions/browse/") return handle_subscription_browse(sid) if sid is not None else _error(400) elif method == "POST": + if not _check_csrf(body): + return _respond("

    403 Forbidden

    Invalid or missing CSRF token.

    ", status=403) if path == "/add": return handle_add_submit(body) elif path.startswith("/edit/"): pid = extract_id("/edit/") return handle_edit_submit(pid, body) if pid is not None else _error(400) + elif path.startswith("/delete/"): + pid = extract_id("/delete/") + return handle_delete(pid) if pid is not None else _error(400) elif path == "/style": return handle_style_submit(body) + elif path == "/style/reset": + set_setting("custom_template", "") + return handle_style_form("Template reset to default.") elif path == "/import": return handle_import_submit(body) elif path == "/subscriptions/add": @@ -872,3 +1069,30 @@ def dispatch_request(data): return handle_subscription_syncall() return _error(404) + + +def dispatch_request(data): + cookies = data.get("cookies", {}) + csrf_token = cookies.get("_csrf", "") + if not csrf_token: + csrf_token = secrets.token_hex(32) + _request_local.csrf_token = csrf_token + + resp = _dispatch_inner(data) + + resp.setdefault("headers", {}) + resp["headers"]["Set-Cookie"] = f"_csrf={csrf_token}; SameSite=Strict; HttpOnly; Path=/" + resp["headers"]["X-Frame-Options"] = "DENY" + resp["headers"]["X-Content-Type-Options"] = "nosniff" + if resp.get("content_type", "").startswith("text/html"): + resp["headers"]["Content-Security-Policy"] = ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline'; " + "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " + "font-src 'self' https://fonts.gstatic.com; " + "img-src * data:; " + "frame-ancestors 'none'; " + "form-action 'self'; " + "base-uri 'self'" + ) + return resp diff --git a/index.db b/index.db deleted file mode 100644 index eaa39d2..0000000 Binary files a/index.db and /dev/null differ diff --git a/rns_client.py b/rns_client.py index 32eeadc..dbc0af5 100644 --- a/rns_client.py +++ b/rns_client.py @@ -6,11 +6,11 @@ ASPECTS = ["server"] REQUEST_TIMEOUT = 30 -def fetch_remote_sites(dest_hash_hex): +def fetch_remote_sites(dest_hash_hex, since=""): """ Connect to a remote TinyWeb instance over Reticulum and fetch its shared sites. Returns the response dict from /api/sites, or raises - an exception on failure. + an exception on failure. Pass `since` as ISO timestamp for delta sync. """ dest_hash = bytes.fromhex(dest_hash_hex) @@ -48,10 +48,11 @@ def fetch_remote_sites(dest_hash_hex): try: # Request /api/sites + query = {"since": [since]} if since else {} request_data = { "method": "GET", "path": "/api/sites", - "query": {}, + "query": query, "body": {}, "gateway_host": "", } diff --git a/templates.py b/templates.py index 735a38e..372e736 100644 --- a/templates.py +++ b/templates.py @@ -15,7 +15,26 @@ def snippet(text, query, ctx=80): return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") -def wrap_page(body_html): - css = get_setting("custom_css") - style = f"" if css else "" - return f"{style}{body_html}" +DEFAULT_TEMPLATE = "\n\n\n\n{{content}}\n\n" + + +def _default_template(): + name = esc(get_setting("site_name", "tinyweb")) + return ( + "\n\n\n\n" + f'

    {name}' + ' | search | browse' + ' | tags | subscriptions' + ' | customize | about

    \n' + "
    \n{{content}}\n\n" + ) + + +def wrap_page(body_html, use_default=False): + if use_default: + template = _default_template() + else: + template = get_setting("custom_template") or _default_template() + if "{{content}}" not in template: + template = _default_template() + return template.replace("{{content}}", body_html) diff --git a/themes/kodama.html b/themes/kodama.html new file mode 100644 index 0000000..d48b577 --- /dev/null +++ b/themes/kodama.html @@ -0,0 +1,746 @@ + + + + + + + + + + + +
    + +
    +
    + {{content}} +
    +
    +
    curated by hand · shared over mesh
    +
    +
    +
    + + + \ No newline at end of file