From ea7cb400e289888883a9d2c32b653338e823f973 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 24 Mar 2026 20:35:10 -0700 Subject: [PATCH 001/194] first commit --- README.md | 0 app.py | 524 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 + 3 files changed, 526 insertions(+) create mode 100644 README.md create mode 100644 app.py create mode 100644 requirements.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/app.py b/app.py new file mode 100644 index 0000000..f6f85d1 --- /dev/null +++ b/app.py @@ -0,0 +1,524 @@ +import json +import sqlite3 +import html +import requests +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import parse_qs, urlparse, urljoin +from bs4 import BeautifulSoup + +DATABASE = "index.db" + + +def get_db(): + db = sqlite3.connect(DATABASE) + db.row_factory = sqlite3.Row + return db + + +def init_db(): + db = sqlite3.connect(DATABASE) + db.execute( + "CREATE TABLE IF NOT EXISTS pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " title TEXT," + " body TEXT," + " note TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " + "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS links (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " page_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " label TEXT," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS settings (" + " key TEXT PRIMARY KEY," + " value TEXT" + ")" + ) + db.executescript(""" + CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + """) + db.commit() + db.close() + + +SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + + +def fetch_page(url): + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) + resp.raise_for_status() + soup = BeautifulSoup(resp.text, "html.parser") + + # extract links before stripping tags + domain = urlparse(url).netloc + seen = set() + links = [] + for a in soup.find_all("a", href=True): + href = urljoin(url, a["href"]).split("#")[0] + parsed = urlparse(href) + if parsed.netloc != domain: + continue + if any(href.lower().endswith(ext) for ext in SKIP_EXT): + continue + if parsed.query or "action=" in href: + continue + path = parsed.path.lower() + if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): + continue + if href in seen or href == url: + continue + seen.add(href) + label = a.get_text(strip=True) or href + links.append((href, label[:200])) + + for tag in soup(["script", "style", "nav", "footer", "header"]): + tag.decompose() + title = soup.title.string.strip() if soup.title and soup.title.string else url + body = soup.get_text(separator=" ", strip=True) + return title, body, links + + +def snippet(text, query, ctx=80): + pos = text.lower().find(query.lower()) + if pos == -1: + return text[:200] + start = max(0, pos - ctx) + end = min(len(text), pos + len(query) + ctx) + return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") + + +def esc(s): + return html.escape(str(s)) + + +def get_setting(key, default=""): + db = get_db() + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + db.close() + return row["value"] if row else default + + +def set_setting(key, value): + db = get_db() + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + db.close() + + +def get_site_name(): + return get_setting("site_name", "tinyweb") + + +def wrap_page(body_html): + css = get_setting("custom_css") + style = f"" if css else "" + return f"{style}{body_html}" + + +class Handler(BaseHTTPRequestHandler): + + def respond(self, body, status=200): + self.send_response(status) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.end_headers() + self.wfile.write(wrap_page(body).encode()) + + def do_GET(self): + parsed = urlparse(self.path) + path = parsed.path + params = parse_qs(parsed.query) + + if path == "/": + self.handle_search(params) + elif path == "/add": + self.handle_add_form() + elif path == "/pages": + self.handle_pages() + elif path.startswith("/delete/"): + self.handle_delete(path) + elif path.startswith("/edit/"): + self.handle_edit_form(path) + elif path == "/style": + self.handle_style_form() + elif path == "/bookmark": + self.handle_bookmark(params) + elif path == "/export": + self.handle_export() + elif path == "/import": + self.handle_import_form() + else: + self.respond("

404

", 404) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length).decode() + params = parse_qs(body) + + if self.path == "/add": + self.handle_add_submit(params) + elif self.path.startswith("/edit/"): + self.handle_edit_submit(self.path, params) + elif self.path == "/style": + self.handle_style_submit(params) + elif self.path == "/import": + self.handle_import_submit(params) + else: + self.respond("

404

", 404) + + def handle_search(self, params): + q = params.get("q", [""])[0].strip() + db = get_db() + count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + name = get_site_name() + + result_html = "" + trusted_html = "" + if q: + rows = db.execute( + "SELECT p.id, p.url, p.title, p.body, p.note " + "FROM pages_fts f JOIN pages p ON f.rowid = p.id " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", + (q,), + ).fetchall() + if rows: + for r in rows: + note_html = "" + if r["note"]: + note_html = f'
{esc(r["note"])}
' + result_html += ( + f'
' + f'{esc(r["title"])}
' + f'{esc(r["url"])}
' + f'{esc(snippet(r["body"], q))}' + f'{note_html}' + f'
' + ) + else: + result_html = "

No results in your index.

" + + # search all linked pages from trusted sites + words = q.lower().split() + all_links = db.execute( + "SELECT l.url, l.label, p.title AS source_title " + "FROM links l JOIN pages p ON l.page_id = p.id", + ).fetchall() + indexed_urls = set(r["url"] for r in rows) if rows else set() + seen = set() + trusted = [] + for l in all_links: + if l["url"] in indexed_urls or l["url"] in seen: + continue + if any(w in l["label"].lower() for w in words): + seen.add(l["url"]) + trusted.append(l) + if len(trusted) >= 20: + break + + if trusted: + items = "" + for l in trusted: + items += ( + f'
  • {esc(l["label"])} ' + f'— from {esc(l["source_title"])}
    • ' + ) + trusted_html = ( + f'
    ' + f'from your trusted sites ({len(trusted)})' + f'' + f'
    ' + ) + + db.close() + self.respond( + f'

    {esc(name)}

    ' + f'
    ' + f'' + f' ' + f'
    ' + f'

    {count} page(s) indexed.' + f' + add url' + f' | browse' + f' | customize

    ' + f'
    {result_html}{trusted_html}' + ) + + def handle_add_form(self, msg=""): + self.respond( + f"

    add url

    " + f'
    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_add_submit(self, params): + url = params.get("url", [""])[0].strip() + note = params.get("note", [""])[0].strip() + if not url: + return self.handle_add_form("URL is required.") + if not url.startswith(("http://", "https://")): + return self.handle_add_form("URL must start with http:// or https://") + try: + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", + (url, title, body, note), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + self.handle_add_form(f'Indexed: {esc(title)}') + except Exception as e: + self.handle_add_form(f"Error: {esc(str(e))}") + + def handle_pages(self): + db = get_db() + rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() + db.close() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + items += ( + f'
  • {esc(r["title"])}{note_html} ' + f'({esc(r["url"])}) ' + f'edit ' + f'remove
    • ' + ) + self.respond( + f"

    indexed pages ({len(rows)})

    " + f"" + f'

    export | import

    ' + f'back' + ) + + def handle_edit_form(self, path, msg=""): + try: + page_id = int(path.split("/")[-1]) + except ValueError: + return self.respond("

    400

    ", 400) + db = get_db() + row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() + db.close() + if not row: + return self.respond("

    404

    ", 404) + self.respond( + f"

    edit note

    " + f"

    {esc(row['title'])}
    " + f"{esc(row['url'])}

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_edit_submit(self, path, params): + try: + page_id = int(path.split("/")[-1]) + except ValueError: + return self.respond("

    400

    ", 400) + note = params.get("note", [""])[0].strip() + db = get_db() + db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) + db.commit() + db.close() + self.send_response(302) + self.send_header("Location", "/pages") + self.end_headers() + + def handle_delete(self, path): + try: + page_id = int(path.split("/")[-1]) + except ValueError: + return self.respond("

    400

    ", 400) + db = get_db() + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) + db.commit() + db.close() + self.send_response(302) + self.send_header("Location", "/pages") + self.end_headers() + + def handle_bookmark(self, params): + url = params.get("url", [""])[0].strip() + if not url or not url.startswith(("http://", "https://")): + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(b"error: invalid url") + return + try: + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, '') " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body", + (url, title, body), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + msg = f"ok: {title}" + except Exception as e: + msg = f"error: {e}" + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(msg.encode()) + + def handle_export(self): + db = get_db() + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() + db.close() + data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Disposition", "attachment; filename=tinyweb-export.json") + self.end_headers() + self.wfile.write(json.dumps(data, indent=2).encode()) + + def handle_import_form(self, msg=""): + self.respond( + f"

    import

    " + f"

    Paste the contents of a tinyweb export file (JSON).

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_import_submit(self, params): + raw = params.get("data", [""])[0].strip() + if not raw: + return self.handle_import_form("Paste JSON data.") + try: + data = json.loads(raw) + except json.JSONDecodeError: + return self.handle_import_form("Invalid JSON.") + if not isinstance(data, list): + return self.handle_import_form("Expected a JSON array.") + + imported = 0 + errors = 0 + for entry in data: + url = entry.get("url", "").strip() + note = entry.get("note", "").strip() + if not url: + continue + try: + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", + (url, title, body, note), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + imported += 1 + except Exception: + errors += 1 + + self.handle_import_form(f"Imported {imported} page(s). {errors} error(s).") + + def handle_style_form(self, msg=""): + css = get_setting("custom_css") + name = get_site_name() + self.respond( + f"

    customize

    " + f"

    name your search engine

    " + f'
    ' + f'

    ' + f"

    custom css

    " + f"

    Some classes you can target:

    " + f"
    "
+            f"body          - page background, font\n"
+            f"h1            - page titles\n"
+            f"input, button - search bar\n"
+            f"a             - links\n"
+            f".result       - each search result\n"
+            f".note         - your notes on results\n"
+            f".trusted      - trusted sites dropdown\n"
+            f"small         - url text\n"
+            f"ul, li        - browse page list"
+            f"
    " + f'

    ' + f'' + f"
    " + f"

    bookmarklet

    " + f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " + f'

    + save to {esc(name)}

    ' + f"

    {msg}

    " + f'back' + ) + + def handle_style_submit(self, params): + css = params.get("css", [""])[0] + name = params.get("site_name", ["tinyweb"])[0].strip() + set_setting("custom_css", css) + set_setting("site_name", name or "tinyweb") + self.handle_style_form("Saved.") + + +if __name__ == "__main__": + init_db() + print("running on http://localhost:5001") + HTTPServer(("localhost", 5001), Handler).serve_forever() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1190bd8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +beautifulsoup4 From 2c69a10f0ecd55fa0124cad0d7e18ecc54e33001 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 24 Mar 2026 20:35:10 -0700 Subject: [PATCH 002/194] first commit --- README.md | 0 app.py | 524 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 + 3 files changed, 526 insertions(+) create mode 100644 README.md create mode 100644 app.py create mode 100644 requirements.txt diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/app.py b/app.py new file mode 100644 index 0000000..f6f85d1 --- /dev/null +++ b/app.py @@ -0,0 +1,524 @@ +import json +import sqlite3 +import html +import requests +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import parse_qs, urlparse, urljoin +from bs4 import BeautifulSoup + +DATABASE = "index.db" + + +def get_db(): + db = sqlite3.connect(DATABASE) + db.row_factory = sqlite3.Row + return db + + +def init_db(): + db = sqlite3.connect(DATABASE) + db.execute( + "CREATE TABLE IF NOT EXISTS pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " title TEXT," + " body TEXT," + " note TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " + "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS links (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " page_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " label TEXT," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS settings (" + " key TEXT PRIMARY KEY," + " value TEXT" + ")" + ) + db.executescript(""" + CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + """) + db.commit() + db.close() + + +SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + + +def fetch_page(url): + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) + resp.raise_for_status() + soup = BeautifulSoup(resp.text, "html.parser") + + # extract links before stripping tags + domain = urlparse(url).netloc + seen = set() + links = [] + for a in soup.find_all("a", href=True): + href = urljoin(url, a["href"]).split("#")[0] + parsed = urlparse(href) + if parsed.netloc != domain: + continue + if any(href.lower().endswith(ext) for ext in SKIP_EXT): + continue + if parsed.query or "action=" in href: + continue + path = parsed.path.lower() + if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): + continue + if href in seen or href == url: + continue + seen.add(href) + label = a.get_text(strip=True) or href + links.append((href, label[:200])) + + for tag in soup(["script", "style", "nav", "footer", "header"]): + tag.decompose() + title = soup.title.string.strip() if soup.title and soup.title.string else url + body = soup.get_text(separator=" ", strip=True) + return title, body, links + + +def snippet(text, query, ctx=80): + pos = text.lower().find(query.lower()) + if pos == -1: + return text[:200] + start = max(0, pos - ctx) + end = min(len(text), pos + len(query) + ctx) + return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") + + +def esc(s): + return html.escape(str(s)) + + +def get_setting(key, default=""): + db = get_db() + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + db.close() + return row["value"] if row else default + + +def set_setting(key, value): + db = get_db() + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + db.close() + + +def get_site_name(): + return get_setting("site_name", "tinyweb") + + +def wrap_page(body_html): + css = get_setting("custom_css") + style = f"" if css else "" + return f"{style}{body_html}" + + +class Handler(BaseHTTPRequestHandler): + + def respond(self, body, status=200): + self.send_response(status) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.end_headers() + self.wfile.write(wrap_page(body).encode()) + + def do_GET(self): + parsed = urlparse(self.path) + path = parsed.path + params = parse_qs(parsed.query) + + if path == "/": + self.handle_search(params) + elif path == "/add": + self.handle_add_form() + elif path == "/pages": + self.handle_pages() + elif path.startswith("/delete/"): + self.handle_delete(path) + elif path.startswith("/edit/"): + self.handle_edit_form(path) + elif path == "/style": + self.handle_style_form() + elif path == "/bookmark": + self.handle_bookmark(params) + elif path == "/export": + self.handle_export() + elif path == "/import": + self.handle_import_form() + else: + self.respond("

    404

    ", 404) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length).decode() + params = parse_qs(body) + + if self.path == "/add": + self.handle_add_submit(params) + elif self.path.startswith("/edit/"): + self.handle_edit_submit(self.path, params) + elif self.path == "/style": + self.handle_style_submit(params) + elif self.path == "/import": + self.handle_import_submit(params) + else: + self.respond("

    404

    ", 404) + + def handle_search(self, params): + q = params.get("q", [""])[0].strip() + db = get_db() + count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + name = get_site_name() + + result_html = "" + trusted_html = "" + if q: + rows = db.execute( + "SELECT p.id, p.url, p.title, p.body, p.note " + "FROM pages_fts f JOIN pages p ON f.rowid = p.id " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", + (q,), + ).fetchall() + if rows: + for r in rows: + note_html = "" + if r["note"]: + note_html = f'
    {esc(r["note"])}
    ' + result_html += ( + f'
    ' + f'{esc(r["title"])}
    ' + f'{esc(r["url"])}
    ' + f'{esc(snippet(r["body"], q))}' + f'{note_html}' + f'
    ' + ) + else: + result_html = "

    No results in your index.

    " + + # search all linked pages from trusted sites + words = q.lower().split() + all_links = db.execute( + "SELECT l.url, l.label, p.title AS source_title " + "FROM links l JOIN pages p ON l.page_id = p.id", + ).fetchall() + indexed_urls = set(r["url"] for r in rows) if rows else set() + seen = set() + trusted = [] + for l in all_links: + if l["url"] in indexed_urls or l["url"] in seen: + continue + if any(w in l["label"].lower() for w in words): + seen.add(l["url"]) + trusted.append(l) + if len(trusted) >= 20: + break + + if trusted: + items = "" + for l in trusted: + items += ( + f'
  • {esc(l["label"])} ' + f'— from {esc(l["source_title"])}
    • ' + ) + trusted_html = ( + f'
    ' + f'from your trusted sites ({len(trusted)})' + f'
      {items}
    ' + f'
    ' + ) + + db.close() + self.respond( + f'

    {esc(name)}

    ' + f'
    ' + f'' + f' ' + f'
    ' + f'

    {count} page(s) indexed.' + f' + add url' + f' | browse' + f' | customize

    ' + f'
    {result_html}{trusted_html}' + ) + + def handle_add_form(self, msg=""): + self.respond( + f"

    add url

    " + f'
    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_add_submit(self, params): + url = params.get("url", [""])[0].strip() + note = params.get("note", [""])[0].strip() + if not url: + return self.handle_add_form("URL is required.") + if not url.startswith(("http://", "https://")): + return self.handle_add_form("URL must start with http:// or https://") + try: + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", + (url, title, body, note), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + self.handle_add_form(f'Indexed: {esc(title)}') + except Exception as e: + self.handle_add_form(f"Error: {esc(str(e))}") + + def handle_pages(self): + db = get_db() + rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() + db.close() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + items += ( + f'
  • {esc(r["title"])}{note_html} ' + f'({esc(r["url"])}) ' + f'edit ' + f'remove
    • ' + ) + self.respond( + f"

    indexed pages ({len(rows)})

    " + f"" + f'

    export | import

    ' + f'back' + ) + + def handle_edit_form(self, path, msg=""): + try: + page_id = int(path.split("/")[-1]) + except ValueError: + return self.respond("

    400

    ", 400) + db = get_db() + row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() + db.close() + if not row: + return self.respond("

    404

    ", 404) + self.respond( + f"

    edit note

    " + f"

    {esc(row['title'])}
    " + f"{esc(row['url'])}

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_edit_submit(self, path, params): + try: + page_id = int(path.split("/")[-1]) + except ValueError: + return self.respond("

    400

    ", 400) + note = params.get("note", [""])[0].strip() + db = get_db() + db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) + db.commit() + db.close() + self.send_response(302) + self.send_header("Location", "/pages") + self.end_headers() + + def handle_delete(self, path): + try: + page_id = int(path.split("/")[-1]) + except ValueError: + return self.respond("

    400

    ", 400) + db = get_db() + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) + db.commit() + db.close() + self.send_response(302) + self.send_header("Location", "/pages") + self.end_headers() + + def handle_bookmark(self, params): + url = params.get("url", [""])[0].strip() + if not url or not url.startswith(("http://", "https://")): + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(b"error: invalid url") + return + try: + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, '') " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body", + (url, title, body), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + msg = f"ok: {title}" + except Exception as e: + msg = f"error: {e}" + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(msg.encode()) + + def handle_export(self): + db = get_db() + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() + db.close() + data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Disposition", "attachment; filename=tinyweb-export.json") + self.end_headers() + self.wfile.write(json.dumps(data, indent=2).encode()) + + def handle_import_form(self, msg=""): + self.respond( + f"

    import

    " + f"

    Paste the contents of a tinyweb export file (JSON).

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_import_submit(self, params): + raw = params.get("data", [""])[0].strip() + if not raw: + return self.handle_import_form("Paste JSON data.") + try: + data = json.loads(raw) + except json.JSONDecodeError: + return self.handle_import_form("Invalid JSON.") + if not isinstance(data, list): + return self.handle_import_form("Expected a JSON array.") + + imported = 0 + errors = 0 + for entry in data: + url = entry.get("url", "").strip() + note = entry.get("note", "").strip() + if not url: + continue + try: + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", + (url, title, body, note), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + imported += 1 + except Exception: + errors += 1 + + self.handle_import_form(f"Imported {imported} page(s). {errors} error(s).") + + def handle_style_form(self, msg=""): + css = get_setting("custom_css") + name = get_site_name() + self.respond( + f"

    customize

    " + f"

    name your search engine

    " + f'
    ' + f'

    ' + f"

    custom css

    " + f"

    Some classes you can target:

    " + f"
    "
+            f"body          - page background, font\n"
+            f"h1            - page titles\n"
+            f"input, button - search bar\n"
+            f"a             - links\n"
+            f".result       - each search result\n"
+            f".note         - your notes on results\n"
+            f".trusted      - trusted sites dropdown\n"
+            f"small         - url text\n"
+            f"ul, li        - browse page list"
+            f"
    " + f'

    ' + f'' + f"
    " + f"

    bookmarklet

    " + f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " + f'

    + save to {esc(name)}

    ' + f"

    {msg}

    " + f'back' + ) + + def handle_style_submit(self, params): + css = params.get("css", [""])[0] + name = params.get("site_name", ["tinyweb"])[0].strip() + set_setting("custom_css", css) + set_setting("site_name", name or "tinyweb") + self.handle_style_form("Saved.") + + +if __name__ == "__main__": + init_db() + print("running on http://localhost:5001") + HTTPServer(("localhost", 5001), Handler).serve_forever() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1190bd8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +beautifulsoup4 From 647b14e99920c2bfbe84a7a3b5d36b41cfd77797 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 24 Mar 2026 20:45:14 -0700 Subject: [PATCH 003/194] bound to 0.0.0.0, dynamic Host header Makes the server accessible from other devices on the network instead of only localhost. The bookmarklet now uses the Host header from the request so it works regardless of how the server is accessed. --- app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index f6f85d1..dc83ae4 100644 --- a/app.py +++ b/app.py @@ -505,7 +505,7 @@ class Handler(BaseHTTPRequestHandler): f"" f"

    bookmarklet

    " f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " - f'

    + save to {esc(name)}

    ' + f'

    r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}

    ' f"

    {msg}

    " f'back' ) @@ -520,5 +520,5 @@ class Handler(BaseHTTPRequestHandler): if __name__ == "__main__": init_db() - print("running on http://localhost:5001") - HTTPServer(("localhost", 5001), Handler).serve_forever() + print("running on http://0.0.0.0:5001") + HTTPServer(("0.0.0.0", 5001), Handler).serve_forever() From ecc48547d11989dca418b7988956a1f515e423f7 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 24 Mar 2026 20:45:14 -0700 Subject: [PATCH 004/194] bound to 0.0.0.0, dynamic Host header Makes the server accessible from other devices on the network instead of only localhost. The bookmarklet now uses the Host header from the request so it works regardless of how the server is accessed. --- app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app.py b/app.py index f6f85d1..dc83ae4 100644 --- a/app.py +++ b/app.py @@ -505,7 +505,7 @@ class Handler(BaseHTTPRequestHandler): f"" f"

    bookmarklet

    " f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " - f'

    + save to {esc(name)}

    ' + f'

    r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}

    ' f"

    {msg}

    " f'back' ) @@ -520,5 +520,5 @@ class Handler(BaseHTTPRequestHandler): if __name__ == "__main__": init_db() - print("running on http://localhost:5001") - HTTPServer(("localhost", 5001), Handler).serve_forever() + print("running on http://0.0.0.0:5001") + HTTPServer(("0.0.0.0", 5001), Handler).serve_forever() From be020d215d2cba38f089e5827462e58f0e16c625 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 22:17:51 -0700 Subject: [PATCH 005/194] ported everything to Reticulum mesh Replace HTTP server with Reticulum-native architecture. The server now speaks only Reticulum, with a client-side gateway providing browser access by translating HTTP to/from RNS requests. - Extract db layer (db.py), templates (templates.py), handlers (handlers.py) - app.py is now the RNS server with persistent identity and destination - gateway.py bridges HTTP on localhost:8080 to RNS link requests - Add rns dependency, add .gitignore --- .gitignore | 3 + app.py | 555 ++++----------------------------------- db.py | 149 +++++++++++ gateway.py | 149 +++++++++++ handlers.py | 660 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + templates.py | 21 ++ 7 files changed, 1027 insertions(+), 511 deletions(-) create mode 100644 .gitignore create mode 100644 db.py create mode 100644 gateway.py create mode 100644 handlers.py create mode 100644 templates.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..799f1c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +tinyweb_identity +index.db diff --git a/app.py b/app.py index dc83ae4..890bf0a 100644 --- a/app.py +++ b/app.py @@ -1,524 +1,57 @@ -import json -import sqlite3 -import html -import requests -from http.server import HTTPServer, BaseHTTPRequestHandler -from urllib.parse import parse_qs, urlparse, urljoin -from bs4 import BeautifulSoup +import os +import time +import RNS -DATABASE = "index.db" +from db import init_db +from handlers import dispatch_request + +APP_NAME = "tinyweb" +ASPECTS = ["server"] +IDENTITY_FILE = "tinyweb_identity" -def get_db(): - db = sqlite3.connect(DATABASE) - db.row_factory = sqlite3.Row - return db +def load_or_create_identity(): + if os.path.isfile(IDENTITY_FILE): + return RNS.Identity.from_file(IDENTITY_FILE) + identity = RNS.Identity() + identity.to_file(IDENTITY_FILE) + return identity -def init_db(): - db = sqlite3.connect(DATABASE) - db.execute( - "CREATE TABLE IF NOT EXISTS pages (" - " id INTEGER PRIMARY KEY AUTOINCREMENT," - " url TEXT UNIQUE NOT NULL," - " title TEXT," - " body TEXT," - " note TEXT DEFAULT ''" - ")" +def rns_request_handler(path, data, request_id, link_id, remote_identity, requested_at): + if data is None: + data = {"method": "GET", "path": "/", "query": {}, "body": {}, "gateway_host": ""} + return dispatch_request(data) + + +def main(): + init_db() + reticulum = RNS.Reticulum() + identity = load_or_create_identity() + + destination = RNS.Destination( + identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, ) - db.execute( - "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " - "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + + destination.register_request_handler( + "/tinyweb", + response_generator=rns_request_handler, + allow=RNS.Destination.ALLOW_ALL, ) - db.execute( - "CREATE TABLE IF NOT EXISTS links (" - " id INTEGER PRIMARY KEY AUTOINCREMENT," - " page_id INTEGER NOT NULL," - " url TEXT NOT NULL," - " label TEXT," - " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" - ")" - ) - db.execute( - "CREATE TABLE IF NOT EXISTS settings (" - " key TEXT PRIMARY KEY," - " value TEXT" - ")" - ) - db.executescript(""" - CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN - INSERT INTO pages_fts(rowid, title, body, url, note) - VALUES (new.id, new.title, new.body, new.url, new.note); - END; - CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN - INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) - VALUES ('delete', old.id, old.title, old.body, old.url, old.note); - END; - CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN - INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) - VALUES ('delete', old.id, old.title, old.body, old.url, old.note); - INSERT INTO pages_fts(rowid, title, body, url, note) - VALUES (new.id, new.title, new.body, new.url, new.note); - END; - """) - db.commit() - db.close() + destination.announce() -SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + print(f"TinyWeb Reticulum server running") + print(f"Destination hash: {RNS.prettyhexrep(destination.hash)}") + print(f"Share this hash with clients to connect via gateway.py") - -def fetch_page(url): - resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) - resp.raise_for_status() - soup = BeautifulSoup(resp.text, "html.parser") - - # extract links before stripping tags - domain = urlparse(url).netloc - seen = set() - links = [] - for a in soup.find_all("a", href=True): - href = urljoin(url, a["href"]).split("#")[0] - parsed = urlparse(href) - if parsed.netloc != domain: - continue - if any(href.lower().endswith(ext) for ext in SKIP_EXT): - continue - if parsed.query or "action=" in href: - continue - path = parsed.path.lower() - if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): - continue - if href in seen or href == url: - continue - seen.add(href) - label = a.get_text(strip=True) or href - links.append((href, label[:200])) - - for tag in soup(["script", "style", "nav", "footer", "header"]): - tag.decompose() - title = soup.title.string.strip() if soup.title and soup.title.string else url - body = soup.get_text(separator=" ", strip=True) - return title, body, links - - -def snippet(text, query, ctx=80): - pos = text.lower().find(query.lower()) - if pos == -1: - return text[:200] - start = max(0, pos - ctx) - end = min(len(text), pos + len(query) + ctx) - return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") - - -def esc(s): - return html.escape(str(s)) - - -def get_setting(key, default=""): - db = get_db() - row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() - db.close() - return row["value"] if row else default - - -def set_setting(key, value): - db = get_db() - db.execute( - "INSERT INTO settings (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value=excluded.value", - (key, value), - ) - db.commit() - db.close() - - -def get_site_name(): - return get_setting("site_name", "tinyweb") - - -def wrap_page(body_html): - css = get_setting("custom_css") - style = f"" if css else "" - return f"{style}{body_html}" - - -class Handler(BaseHTTPRequestHandler): - - def respond(self, body, status=200): - self.send_response(status) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.end_headers() - self.wfile.write(wrap_page(body).encode()) - - def do_GET(self): - parsed = urlparse(self.path) - path = parsed.path - params = parse_qs(parsed.query) - - if path == "/": - self.handle_search(params) - elif path == "/add": - self.handle_add_form() - elif path == "/pages": - self.handle_pages() - elif path.startswith("/delete/"): - self.handle_delete(path) - elif path.startswith("/edit/"): - self.handle_edit_form(path) - elif path == "/style": - self.handle_style_form() - elif path == "/bookmark": - self.handle_bookmark(params) - elif path == "/export": - self.handle_export() - elif path == "/import": - self.handle_import_form() - else: - self.respond("

    404

    ", 404) - - def do_POST(self): - length = int(self.headers.get("Content-Length", 0)) - body = self.rfile.read(length).decode() - params = parse_qs(body) - - if self.path == "/add": - self.handle_add_submit(params) - elif self.path.startswith("/edit/"): - self.handle_edit_submit(self.path, params) - elif self.path == "/style": - self.handle_style_submit(params) - elif self.path == "/import": - self.handle_import_submit(params) - else: - self.respond("

    404

    ", 404) - - def handle_search(self, params): - q = params.get("q", [""])[0].strip() - db = get_db() - count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - name = get_site_name() - - result_html = "" - trusted_html = "" - if q: - rows = db.execute( - "SELECT p.id, p.url, p.title, p.body, p.note " - "FROM pages_fts f JOIN pages p ON f.rowid = p.id " - "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", - (q,), - ).fetchall() - if rows: - for r in rows: - note_html = "" - if r["note"]: - note_html = f'
    {esc(r["note"])}
    ' - result_html += ( - f'
    ' - f'{esc(r["title"])}
    ' - f'{esc(r["url"])}
    ' - f'{esc(snippet(r["body"], q))}' - f'{note_html}' - f'
    ' - ) - else: - result_html = "

    No results in your index.

    " - - # search all linked pages from trusted sites - words = q.lower().split() - all_links = db.execute( - "SELECT l.url, l.label, p.title AS source_title " - "FROM links l JOIN pages p ON l.page_id = p.id", - ).fetchall() - indexed_urls = set(r["url"] for r in rows) if rows else set() - seen = set() - trusted = [] - for l in all_links: - if l["url"] in indexed_urls or l["url"] in seen: - continue - if any(w in l["label"].lower() for w in words): - seen.add(l["url"]) - trusted.append(l) - if len(trusted) >= 20: - break - - if trusted: - items = "" - for l in trusted: - items += ( - f'
  • {esc(l["label"])} ' - f'— from {esc(l["source_title"])}
    • ' - ) - trusted_html = ( - f'
    ' - f'from your trusted sites ({len(trusted)})' - f'
      {items}
    ' - f'
    ' - ) - - db.close() - self.respond( - f'

    {esc(name)}

    ' - f'
    ' - f'' - f' ' - f'
    ' - f'

    {count} page(s) indexed.' - f' + add url' - f' | browse' - f' | customize

    ' - f'
    {result_html}{trusted_html}' - ) - - def handle_add_form(self, msg=""): - self.respond( - f"

    add url

    " - f'
    ' - f'

    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_add_submit(self, params): - url = params.get("url", [""])[0].strip() - note = params.get("note", [""])[0].strip() - if not url: - return self.handle_add_form("URL is required.") - if not url.startswith(("http://", "https://")): - return self.handle_add_form("URL must start with http:// or https://") - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - self.handle_add_form(f'Indexed: {esc(title)}') - except Exception as e: - self.handle_add_form(f"Error: {esc(str(e))}") - - def handle_pages(self): - db = get_db() - rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() - db.close() - items = "" - for r in rows: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - items += ( - f'
  • {esc(r["title"])}{note_html} ' - f'({esc(r["url"])}) ' - f'edit ' - f'remove
    • ' - ) - self.respond( - f"

    indexed pages ({len(rows)})

    " - f"
      {items}
    " - f'

    export | import

    ' - f'back' - ) - - def handle_edit_form(self, path, msg=""): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - db = get_db() - row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() - db.close() - if not row: - return self.respond("

    404

    ", 404) - self.respond( - f"

    edit note

    " - f"

    {esc(row['title'])}
    " - f"{esc(row['url'])}

    " - f'
    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_edit_submit(self, path, params): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - note = params.get("note", [""])[0].strip() - db = get_db() - db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) - db.commit() - db.close() - self.send_response(302) - self.send_header("Location", "/pages") - self.end_headers() - - def handle_delete(self, path): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - db = get_db() - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) - db.commit() - db.close() - self.send_response(302) - self.send_header("Location", "/pages") - self.end_headers() - - def handle_bookmark(self, params): - url = params.get("url", [""])[0].strip() - if not url or not url.startswith(("http://", "https://")): - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Access-Control-Allow-Origin", "*") - self.end_headers() - self.wfile.write(b"error: invalid url") - return - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, '') " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body", - (url, title, body), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - msg = f"ok: {title}" - except Exception as e: - msg = f"error: {e}" - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Access-Control-Allow-Origin", "*") - self.end_headers() - self.wfile.write(msg.encode()) - - def handle_export(self): - db = get_db() - rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() - db.close() - data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Disposition", "attachment; filename=tinyweb-export.json") - self.end_headers() - self.wfile.write(json.dumps(data, indent=2).encode()) - - def handle_import_form(self, msg=""): - self.respond( - f"

    import

    " - f"

    Paste the contents of a tinyweb export file (JSON).

    " - f'
    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_import_submit(self, params): - raw = params.get("data", [""])[0].strip() - if not raw: - return self.handle_import_form("Paste JSON data.") - try: - data = json.loads(raw) - except json.JSONDecodeError: - return self.handle_import_form("Invalid JSON.") - if not isinstance(data, list): - return self.handle_import_form("Expected a JSON array.") - - imported = 0 - errors = 0 - for entry in data: - url = entry.get("url", "").strip() - note = entry.get("note", "").strip() - if not url: - continue - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - imported += 1 - except Exception: - errors += 1 - - self.handle_import_form(f"Imported {imported} page(s). {errors} error(s).") - - def handle_style_form(self, msg=""): - css = get_setting("custom_css") - name = get_site_name() - self.respond( - f"

    customize

    " - f"

    name your search engine

    " - f'
    ' - f'

    ' - f"

    custom css

    " - f"

    Some classes you can target:

    " - f"
    "
-            f"body          - page background, font\n"
-            f"h1            - page titles\n"
-            f"input, button - search bar\n"
-            f"a             - links\n"
-            f".result       - each search result\n"
-            f".note         - your notes on results\n"
-            f".trusted      - trusted sites dropdown\n"
-            f"small         - url text\n"
-            f"ul, li        - browse page list"
-            f"
    " - f'

    ' - f'' - f"
    " - f"

    bookmarklet

    " - f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " - f'

    r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}

    ' - f"

    {msg}

    " - f'back' - ) - - def handle_style_submit(self, params): - css = params.get("css", [""])[0] - name = params.get("site_name", ["tinyweb"])[0].strip() - set_setting("custom_css", css) - set_setting("site_name", name or "tinyweb") - self.handle_style_form("Saved.") + while True: + time.sleep(1) if __name__ == "__main__": - init_db() - print("running on http://0.0.0.0:5001") - HTTPServer(("0.0.0.0", 5001), Handler).serve_forever() + main() diff --git a/db.py b/db.py new file mode 100644 index 0000000..903f824 --- /dev/null +++ b/db.py @@ -0,0 +1,149 @@ +import sqlite3 +import requests +from urllib.parse import urlparse, urljoin +from bs4 import BeautifulSoup + +DATABASE = "index.db" + +SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + + +def get_db(): + db = sqlite3.connect(DATABASE) + db.row_factory = sqlite3.Row + return db + + +def init_db(): + db = sqlite3.connect(DATABASE) + db.execute( + "CREATE TABLE IF NOT EXISTS pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " title TEXT," + " body TEXT," + " note TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " + "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS links (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " page_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " label TEXT," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS settings (" + " key TEXT PRIMARY KEY," + " value TEXT" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS subscriptions (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " name TEXT DEFAULT ''," + " auto_sync INTEGER DEFAULT 0," + " last_sync TEXT DEFAULT ''" + ")" + ) + db.executescript(""" + CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + """) + db.commit() + db.close() + + +def get_setting(key, default=""): + db = get_db() + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + db.close() + return row["value"] if row else default + + +def set_setting(key, value): + db = get_db() + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + db.close() + + +def get_site_name(): + return get_setting("site_name", "tinyweb") + + +def fetch_page(url): + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) + resp.raise_for_status() + soup = BeautifulSoup(resp.text, "html.parser") + + # extract links before stripping tags + domain = urlparse(url).netloc + seen = set() + links = [] + for a in soup.find_all("a", href=True): + href = urljoin(url, a["href"]).split("#")[0] + parsed = urlparse(href) + if parsed.netloc != domain: + continue + if any(href.lower().endswith(ext) for ext in SKIP_EXT): + continue + if parsed.query or "action=" in href: + continue + path = parsed.path.lower() + if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): + continue + if href in seen or href == url: + continue + seen.add(href) + label = a.get_text(strip=True) or href + links.append((href, label[:200])) + + for tag in soup(["script", "style", "nav", "footer", "header"]): + tag.decompose() + title = soup.title.string.strip() if soup.title and soup.title.string else url + body = soup.get_text(separator=" ", strip=True) + return title, body, links + + +def index_url(url, note=""): + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", + (url, title, body, note), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + return title diff --git a/gateway.py b/gateway.py new file mode 100644 index 0000000..0bde2d3 --- /dev/null +++ b/gateway.py @@ -0,0 +1,149 @@ +import sys +import time +import threading +import RNS +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import parse_qs, urlparse + +APP_NAME = "tinyweb" +ASPECTS = ["server"] +GATEWAY_PORT = 8080 +REQUEST_TIMEOUT = 60 + + +class GatewayState: + reticulum = None + destination = None + link = None + link_lock = threading.Lock() + + +def resolve_destination(dest_hash_hex): + dest_hash = bytes.fromhex(dest_hash_hex) + + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + print(f"Requesting path to {RNS.prettyhexrep(dest_hash)}...") + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + raise ConnectionError(f"Could not find path to {RNS.prettyhexrep(dest_hash)}") + + server_identity = RNS.Identity.recall(dest_hash) + GatewayState.destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, + ) + print(f"Resolved destination: {RNS.prettyhexrep(dest_hash)}") + + +def ensure_link(): + with GatewayState.link_lock: + if GatewayState.link and GatewayState.link.status == RNS.Link.ACTIVE: + return GatewayState.link + + print("Establishing link...") + link = RNS.Link(GatewayState.destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + + if link.status != RNS.Link.ACTIVE: + raise ConnectionError("Link establishment failed") + + GatewayState.link = link + print("Link established") + return link + + +class GatewayHandler(BaseHTTPRequestHandler): + + def _forward(self, method): + parsed = urlparse(self.path) + query = parse_qs(parsed.query) + + body = {} + if method == "POST": + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length).decode() + body = parse_qs(raw) + + request_data = { + "method": method, + "path": parsed.path, + "query": query, + "body": body, + "gateway_host": self.headers.get("Host", f"localhost:{GATEWAY_PORT}"), + } + + try: + link = ensure_link() + receipt = link.request( + "/tinyweb", + data=request_data, + timeout=REQUEST_TIMEOUT, + ) + + # Wait for the response + elapsed = 0 + done_statuses = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done_statuses and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + self.send_response(resp["status"]) + self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8")) + for k, v in resp.get("headers", {}).items(): + self.send_header(k, v) + self.end_headers() + resp_body = resp.get("body", "") + if resp_body: + self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body) + elif receipt.get_status() == RNS.RequestReceipt.FAILED: + self.send_error(504, "Request to TinyWeb server failed") + else: + self.send_error(504, "Request to TinyWeb server timed out") + + except ConnectionError as e: + GatewayState.link = None + self.send_error(502, f"Gateway error: {e}") + except Exception as e: + GatewayState.link = None + self.send_error(502, f"Gateway error: {e}") + + def do_GET(self): + self._forward("GET") + + def do_POST(self): + self._forward("POST") + + def log_message(self, format, *args): + print(f"[Gateway] {args[0]}") + + +def main(): + if len(sys.argv) < 2: + print(f"Usage: python gateway.py ") + print(f" The destination hash is printed by app.py on startup.") + sys.exit(1) + + dest_hash = sys.argv[1].replace("<", "").replace(">", "") + + GatewayState.reticulum = RNS.Reticulum() + resolve_destination(dest_hash) + + print(f"Gateway listening on http://localhost:{GATEWAY_PORT}") + print(f"Open http://localhost:{GATEWAY_PORT} in your browser") + HTTPServer(("127.0.0.1", GATEWAY_PORT), GatewayHandler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/handlers.py b/handlers.py new file mode 100644 index 0000000..8f17ee8 --- /dev/null +++ b/handlers.py @@ -0,0 +1,660 @@ +import json +from datetime import datetime +import requests + +from db import get_db, get_setting, set_setting, get_site_name, index_url +from templates import esc, snippet, wrap_page + + +def _respond(body_html, status=200): + return { + "status": status, + "content_type": "text/html; charset=utf-8", + "body": wrap_page(body_html), + "headers": {}, + } + + +def _redirect(location): + return { + "status": 302, + "content_type": "text/html; charset=utf-8", + "body": "", + "headers": {"Location": location}, + } + + +def _json_response(data, status=200, headers=None): + return { + "status": status, + "content_type": "application/json", + "body": json.dumps(data, indent=2), + "headers": headers or {}, + } + + +def _text_response(text, status=200, headers=None): + return { + "status": status, + "content_type": "text/plain", + "body": text, + "headers": headers or {}, + } + + +def _error(status): + return _respond(f"

    {status}

    ", status) + + +# --- Route handlers --- + + +def handle_search(query): + q = query.get("q", [""])[0].strip() + db = get_db() + count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + name = get_site_name() + + result_html = "" + trusted_html = "" + if q: + rows = db.execute( + "SELECT p.id, p.url, p.title, p.body, p.note " + "FROM pages_fts f JOIN pages p ON f.rowid = p.id " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", + (q,), + ).fetchall() + if rows: + for r in rows: + note_html = "" + if r["note"]: + note_html = f'
    {esc(r["note"])}
    ' + result_html += ( + f'
    ' + f'{esc(r["title"])}
    ' + f'{esc(r["url"])}
    ' + f'{esc(snippet(r["body"], q))}' + f'{note_html}' + f'
    ' + ) + else: + result_html = "

    No results in your index.

    " + + # search all linked pages from trusted sites + words = q.lower().split() + all_links = db.execute( + "SELECT l.url, l.label, p.title AS source_title " + "FROM links l JOIN pages p ON l.page_id = p.id", + ).fetchall() + indexed_urls = set(r["url"] for r in rows) if rows else set() + seen = set() + trusted = [] + for l in all_links: + if l["url"] in indexed_urls or l["url"] in seen: + continue + if any(w in l["label"].lower() for w in words): + seen.add(l["url"]) + trusted.append(l) + if len(trusted) >= 20: + break + + if trusted: + items = "" + for l in trusted: + items += ( + f'
  • {esc(l["label"])} ' + f'— from {esc(l["source_title"])}
    • ' + ) + trusted_html = ( + f'
    ' + f'from your trusted sites ({len(trusted)})' + f'
      {items}
    ' + f'
    ' + ) + + db.close() + return _respond( + f'

    {esc(name)}

    ' + f'
    ' + f'' + f' ' + f'
    ' + f'

    {count} page(s) indexed.' + f' + add url' + f' | browse' + f' | subscriptions' + f' | customize

    ' + f'
    {result_html}{trusted_html}' + ) + + +def handle_add_form(msg=""): + return _respond( + f"

    add url

    " + f'
    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_add_submit(body): + url = body.get("url", [""])[0].strip() + note = body.get("note", [""])[0].strip() + if not url: + return handle_add_form("URL is required.") + if not url.startswith(("http://", "https://")): + return handle_add_form("URL must start with http:// or https://") + try: + title = index_url(url, note) + return handle_add_form(f'Indexed: {esc(title)}') + except Exception as e: + return handle_add_form(f"Error: {esc(str(e))}") + + +def handle_pages(): + db = get_db() + rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() + db.close() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + items += ( + f'
  • {esc(r["title"])}{note_html} ' + f'({esc(r["url"])}) ' + f'edit ' + f'remove
    • ' + ) + return _respond( + f"

    indexed pages ({len(rows)})

    " + f"
      {items}
    " + f'

    export | import

    ' + f'back' + ) + + +def handle_edit_form(page_id, msg=""): + db = get_db() + row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() + db.close() + if not row: + return _error(404) + return _respond( + f"

    edit note

    " + f"

    {esc(row['title'])}
    " + f"{esc(row['url'])}

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_edit_submit(page_id, body): + note = body.get("note", [""])[0].strip() + db = get_db() + db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) + db.commit() + db.close() + return _redirect("/pages") + + +def handle_delete(page_id): + db = get_db() + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) + db.commit() + db.close() + return _redirect("/pages") + + +def handle_bookmark(query): + url = query.get("url", [""])[0].strip() + if not url or not url.startswith(("http://", "https://")): + return _text_response("error: invalid url", headers={"Access-Control-Allow-Origin": "*"}) + try: + title = index_url(url) + msg = f"ok: {title}" + except Exception as e: + msg = f"error: {e}" + return _text_response(msg, headers={"Access-Control-Allow-Origin": "*"}) + + +def handle_export(): + db = get_db() + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() + db.close() + data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] + return _json_response(data, headers={"Content-Disposition": "attachment; filename=tinyweb-export.json"}) + + +def handle_import_form(msg=""): + return _respond( + f"

    import

    " + f"

    Paste the contents of a tinyweb export file (JSON).

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_import_submit(body): + raw = body.get("data", [""])[0].strip() + if not raw: + return handle_import_form("Paste JSON data.") + try: + data = json.loads(raw) + except json.JSONDecodeError: + return handle_import_form("Invalid JSON.") + if not isinstance(data, list): + return handle_import_form("Expected a JSON array.") + + imported = 0 + errors = 0 + for entry in data: + url = entry.get("url", "").strip() + note = entry.get("note", "").strip() + if not url: + continue + try: + index_url(url, note) + imported += 1 + except Exception: + errors += 1 + + return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") + + +def handle_style_form(msg="", gateway_host=""): + css = get_setting("custom_css") + name = get_site_name() + sharing = get_setting("sharing_enabled", "0") + checked = " checked" if sharing == "1" else "" + host = gateway_host or "localhost:8080" + return _respond( + f"

    customize

    " + f"

    name your search engine

    " + f'
    ' + f'

    ' + f"

    sharing

    " + f'

    " + f"

    custom css

    " + f"

    Some classes you can target:

    " + f"
    "
+        f"body          - page background, font\n"
+        f"h1            - page titles\n"
+        f"input, button - search bar\n"
+        f"a             - links\n"
+        f".result       - each search result\n"
+        f".note         - your notes on results\n"
+        f".trusted      - trusted sites dropdown\n"
+        f"small         - url text\n"
+        f"ul, li        - browse page list"
+        f"
    " + f'

    ' + f'' + f"
    " + f"

    bookmarklet

    " + f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " + f'

    + save to {esc(name)}

    ' + f"

    {msg}

    " + f'back' + ) + + +def handle_style_submit(body): + css = body.get("css", [""])[0] + name = body.get("site_name", ["tinyweb"])[0].strip() + sharing = "1" if body.get("sharing_enabled") else "0" + set_setting("custom_css", css) + set_setting("site_name", name or "tinyweb") + set_setting("sharing_enabled", sharing) + return handle_style_form("Saved.") + + +def handle_api_sites(): + if get_setting("sharing_enabled", "0") != "1": + return _json_response( + {"error": "sharing disabled"}, + status=403, + headers={"Access-Control-Allow-Origin": "*"}, + ) + db = get_db() + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id DESC").fetchall() + db.close() + data = { + "name": get_site_name(), + "sites": [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows], + } + return _json_response(data, headers={"Access-Control-Allow-Origin": "*"}) + + +def handle_subscriptions(msg=""): + db = get_db() + subs = db.execute("SELECT * FROM subscriptions ORDER BY id DESC").fetchall() + db.close() + items = "" + for s in subs: + auto_label = "on" if s["auto_sync"] else "off" + last = s["last_sync"] or "never" + items += ( + f'' + f'{esc(s["name"] or "unknown")}
    {esc(s["url"])}' + f'{esc(last)}' + f'' + f'
    ' + f'
    ' + f'' + f'' + f'browse ' + f'
    ' + f'
    ' + f'
    ' + f'
    ' + f'' + f'' + ) + table = "" + if subs: + table = ( + f'' + f'{items}
    instancelast syncauto-syncactions
    ' + f'
    ' + f'
    ' + ) + return _respond( + f"

    subscriptions

    " + f'
    ' + f' ' + f'' + f'
    ' + f'

    {msg}

    ' + f'
    {table}' + f'
    back' + ) + + +def handle_subscription_add(body): + url = body.get("url", [""])[0].strip().rstrip("/") + if not url or not url.startswith(("http://", "https://")): + return handle_subscriptions("URL must start with http:// or https://") + try: + resp = requests.get(f"{url}/api/sites", timeout=5) + if resp.status_code == 403: + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + data = resp.json() + name = data.get("name", "") + except Exception as e: + return handle_subscriptions(f"Could not reach that instance: {esc(str(e))}") + db = get_db() + try: + db.execute( + "INSERT INTO subscriptions (url, name) VALUES (?, ?) " + "ON CONFLICT(url) DO UPDATE SET name=excluded.name", + (url, name), + ) + db.commit() + finally: + db.close() + return handle_subscriptions(f"Subscribed to {esc(name or url)}.") + + +def handle_subscription_browse(sub_id): + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + if not sub: + db.close() + return _error(404) + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + db.close() + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code == 403: + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + sites = resp.json().get("sites", []) + except Exception as e: + return handle_subscriptions(f"Could not fetch sites: {esc(str(e))}") + + new_items = "" + existing_items = "" + new_count = 0 + for s in sites: + if s["url"] in local_urls: + existing_items += ( + f'
  • {esc(s["title"])} ' + f'({esc(s["url"])}) — already indexed
    • ' + ) + else: + new_count += 1 + note_html = f' — {esc(s["note"])}' if s.get("note") else "" + new_items += ( + f'
    • ' + ) + + buttons = "" + if new_count: + buttons = ' ' + return _respond( + f'

    browsing: {esc(sub["name"] or sub["url"])}

    ' + f'

    {len(sites)} site(s) available, {new_count} new

    ' + f'
    ' + f'' + f'
      {new_items}
    ' + f'{buttons}' + f'
    ' + f'

    already indexed

      {existing_items}
    ' + f'back' + ) + + +def handle_subscription_pick(body): + sub_id = body.get("sub_id", [""])[0] + import_all = body.get("import_all", [""])[0] + + if import_all: + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + db.close() + if not sub: + return handle_subscriptions("Subscription not found.") + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + resp.raise_for_status() + sites = resp.json().get("sites", []) + except Exception as e: + return handle_subscriptions(f"Error: {esc(str(e))}") + urls = [s["url"] for s in sites if s["url"] not in local_urls] + else: + urls = body.get("urls", []) + + if not urls: + return handle_subscriptions("No sites selected.") + + imported = 0 + errors = 0 + for url in urls: + try: + index_url(url) + imported += 1 + except Exception: + errors += 1 + return handle_subscriptions(f"Imported {imported} page(s). {errors} error(s).") + + +def handle_subscription_sync(sub_id): + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + if not sub: + db.close() + return handle_subscriptions("Subscription not found.") + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code == 403: + db.close() + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + data = resp.json() + sites = data.get("sites", []) + remote_name = data.get("name", sub["name"]) + except Exception as e: + db.close() + return handle_subscriptions(f"Could not sync: {esc(str(e))}") + + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + synced = 0 + for s in sites: + if s["url"] in local_urls: + continue + try: + db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", + (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + ) + synced += 1 + except Exception: + pass + now = datetime.now().strftime("%Y-%m-%d %H:%M") + db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id)) + db.commit() + db.close() + return handle_subscriptions(f"Synced {synced} new site(s) from {esc(remote_name)}.") + + +def handle_subscription_autosync(sub_id): + db = get_db() + db.execute("UPDATE subscriptions SET auto_sync = 1 - auto_sync WHERE id = ?", (sub_id,)) + db.commit() + db.close() + return _redirect("/subscriptions") + + +def handle_subscription_delete(sub_id): + db = get_db() + db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,)) + db.commit() + db.close() + return _redirect("/subscriptions") + + +def handle_subscription_syncall(): + db = get_db() + subs = db.execute("SELECT * FROM subscriptions WHERE auto_sync = 1").fetchall() + db.close() + if not subs: + return handle_subscriptions("No subscriptions have auto-sync enabled.") + total = 0 + for sub in subs: + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code != 200: + continue + data = resp.json() + sites = data.get("sites", []) + remote_name = data.get("name", sub["name"]) + db = get_db() + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + for s in sites: + if s["url"] in local_urls: + continue + try: + db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", + (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + ) + except Exception: + pass + now = datetime.now().strftime("%Y-%m-%d %H:%M") + db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub["id"])) + db.commit() + db.close() + total += 1 + except Exception: + pass + return handle_subscriptions(f"Synced {total} subscription(s).") + + +# --- Dispatcher --- + + +def dispatch_request(data): + method = data.get("method", "GET") + path = data.get("path", "/") + query = data.get("query", {}) + body = data.get("body", {}) + gateway_host = data.get("gateway_host", "") + + def extract_id(prefix): + try: + return int(path[len(prefix):]) + except (ValueError, IndexError): + return None + + if method == "GET": + if path == "/": + return handle_search(query) + elif path == "/add": + return handle_add_form() + elif path == "/pages": + return handle_pages() + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_form(pid) if pid is not None else _error(400) + elif path.startswith("/delete/"): + pid = extract_id("/delete/") + return handle_delete(pid) if pid is not None else _error(400) + elif path == "/bookmark": + return handle_bookmark(query) + elif path == "/style": + return handle_style_form(gateway_host=gateway_host) + elif path == "/export": + return handle_export() + elif path == "/import": + return handle_import_form() + elif path == "/api/sites": + return handle_api_sites() + elif path == "/subscriptions": + return handle_subscriptions() + elif path.startswith("/subscriptions/browse/"): + sid = extract_id("/subscriptions/browse/") + return handle_subscription_browse(sid) if sid is not None else _error(400) + elif method == "POST": + if path == "/add": + return handle_add_submit(body) + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_submit(pid, body) if pid is not None else _error(400) + elif path == "/style": + return handle_style_submit(body) + elif path == "/import": + return handle_import_submit(body) + elif path == "/subscriptions/add": + return handle_subscription_add(body) + elif path == "/subscriptions/pick": + return handle_subscription_pick(body) + elif path.startswith("/subscriptions/sync/"): + sid = extract_id("/subscriptions/sync/") + return handle_subscription_sync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/autosync/"): + sid = extract_id("/subscriptions/autosync/") + return handle_subscription_autosync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/delete/"): + sid = extract_id("/subscriptions/delete/") + return handle_subscription_delete(sid) if sid is not None else _error(400) + elif path == "/subscriptions/syncall": + return handle_subscription_syncall() + + return _error(404) diff --git a/requirements.txt b/requirements.txt index 1190bd8..f63da5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests beautifulsoup4 +rns diff --git a/templates.py b/templates.py new file mode 100644 index 0000000..735a38e --- /dev/null +++ b/templates.py @@ -0,0 +1,21 @@ +import html +from db import get_setting + + +def esc(s): + return html.escape(str(s)) + + +def snippet(text, query, ctx=80): + pos = text.lower().find(query.lower()) + if pos == -1: + return text[:200] + start = max(0, pos - ctx) + end = min(len(text), pos + len(query) + ctx) + return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") + + +def wrap_page(body_html): + css = get_setting("custom_css") + style = f"" if css else "" + return f"{style}{body_html}" From 3e5cafb7f75da8386716f0941ccbc768b3ffddcd Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 22:17:51 -0700 Subject: [PATCH 006/194] ported everything to Reticulum mesh Replace HTTP server with Reticulum-native architecture. The server now speaks only Reticulum, with a client-side gateway providing browser access by translating HTTP to/from RNS requests. - Extract db layer (db.py), templates (templates.py), handlers (handlers.py) - app.py is now the RNS server with persistent identity and destination - gateway.py bridges HTTP on localhost:8080 to RNS link requests - Add rns dependency, add .gitignore --- .gitignore | 3 + app.py | 555 ++++----------------------------------- db.py | 149 +++++++++++ gateway.py | 149 +++++++++++ handlers.py | 660 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + templates.py | 21 ++ 7 files changed, 1027 insertions(+), 511 deletions(-) create mode 100644 .gitignore create mode 100644 db.py create mode 100644 gateway.py create mode 100644 handlers.py create mode 100644 templates.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..799f1c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +tinyweb_identity +index.db diff --git a/app.py b/app.py index dc83ae4..890bf0a 100644 --- a/app.py +++ b/app.py @@ -1,524 +1,57 @@ -import json -import sqlite3 -import html -import requests -from http.server import HTTPServer, BaseHTTPRequestHandler -from urllib.parse import parse_qs, urlparse, urljoin -from bs4 import BeautifulSoup +import os +import time +import RNS -DATABASE = "index.db" +from db import init_db +from handlers import dispatch_request + +APP_NAME = "tinyweb" +ASPECTS = ["server"] +IDENTITY_FILE = "tinyweb_identity" -def get_db(): - db = sqlite3.connect(DATABASE) - db.row_factory = sqlite3.Row - return db +def load_or_create_identity(): + if os.path.isfile(IDENTITY_FILE): + return RNS.Identity.from_file(IDENTITY_FILE) + identity = RNS.Identity() + identity.to_file(IDENTITY_FILE) + return identity -def init_db(): - db = sqlite3.connect(DATABASE) - db.execute( - "CREATE TABLE IF NOT EXISTS pages (" - " id INTEGER PRIMARY KEY AUTOINCREMENT," - " url TEXT UNIQUE NOT NULL," - " title TEXT," - " body TEXT," - " note TEXT DEFAULT ''" - ")" +def rns_request_handler(path, data, request_id, link_id, remote_identity, requested_at): + if data is None: + data = {"method": "GET", "path": "/", "query": {}, "body": {}, "gateway_host": ""} + return dispatch_request(data) + + +def main(): + init_db() + reticulum = RNS.Reticulum() + identity = load_or_create_identity() + + destination = RNS.Destination( + identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, ) - db.execute( - "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " - "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + + destination.register_request_handler( + "/tinyweb", + response_generator=rns_request_handler, + allow=RNS.Destination.ALLOW_ALL, ) - db.execute( - "CREATE TABLE IF NOT EXISTS links (" - " id INTEGER PRIMARY KEY AUTOINCREMENT," - " page_id INTEGER NOT NULL," - " url TEXT NOT NULL," - " label TEXT," - " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" - ")" - ) - db.execute( - "CREATE TABLE IF NOT EXISTS settings (" - " key TEXT PRIMARY KEY," - " value TEXT" - ")" - ) - db.executescript(""" - CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN - INSERT INTO pages_fts(rowid, title, body, url, note) - VALUES (new.id, new.title, new.body, new.url, new.note); - END; - CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN - INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) - VALUES ('delete', old.id, old.title, old.body, old.url, old.note); - END; - CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN - INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) - VALUES ('delete', old.id, old.title, old.body, old.url, old.note); - INSERT INTO pages_fts(rowid, title, body, url, note) - VALUES (new.id, new.title, new.body, new.url, new.note); - END; - """) - db.commit() - db.close() + destination.announce() -SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + print(f"TinyWeb Reticulum server running") + print(f"Destination hash: {RNS.prettyhexrep(destination.hash)}") + print(f"Share this hash with clients to connect via gateway.py") - -def fetch_page(url): - resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) - resp.raise_for_status() - soup = BeautifulSoup(resp.text, "html.parser") - - # extract links before stripping tags - domain = urlparse(url).netloc - seen = set() - links = [] - for a in soup.find_all("a", href=True): - href = urljoin(url, a["href"]).split("#")[0] - parsed = urlparse(href) - if parsed.netloc != domain: - continue - if any(href.lower().endswith(ext) for ext in SKIP_EXT): - continue - if parsed.query or "action=" in href: - continue - path = parsed.path.lower() - if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): - continue - if href in seen or href == url: - continue - seen.add(href) - label = a.get_text(strip=True) or href - links.append((href, label[:200])) - - for tag in soup(["script", "style", "nav", "footer", "header"]): - tag.decompose() - title = soup.title.string.strip() if soup.title and soup.title.string else url - body = soup.get_text(separator=" ", strip=True) - return title, body, links - - -def snippet(text, query, ctx=80): - pos = text.lower().find(query.lower()) - if pos == -1: - return text[:200] - start = max(0, pos - ctx) - end = min(len(text), pos + len(query) + ctx) - return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") - - -def esc(s): - return html.escape(str(s)) - - -def get_setting(key, default=""): - db = get_db() - row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() - db.close() - return row["value"] if row else default - - -def set_setting(key, value): - db = get_db() - db.execute( - "INSERT INTO settings (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value=excluded.value", - (key, value), - ) - db.commit() - db.close() - - -def get_site_name(): - return get_setting("site_name", "tinyweb") - - -def wrap_page(body_html): - css = get_setting("custom_css") - style = f"" if css else "" - return f"{style}{body_html}" - - -class Handler(BaseHTTPRequestHandler): - - def respond(self, body, status=200): - self.send_response(status) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.end_headers() - self.wfile.write(wrap_page(body).encode()) - - def do_GET(self): - parsed = urlparse(self.path) - path = parsed.path - params = parse_qs(parsed.query) - - if path == "/": - self.handle_search(params) - elif path == "/add": - self.handle_add_form() - elif path == "/pages": - self.handle_pages() - elif path.startswith("/delete/"): - self.handle_delete(path) - elif path.startswith("/edit/"): - self.handle_edit_form(path) - elif path == "/style": - self.handle_style_form() - elif path == "/bookmark": - self.handle_bookmark(params) - elif path == "/export": - self.handle_export() - elif path == "/import": - self.handle_import_form() - else: - self.respond("

    404

    ", 404) - - def do_POST(self): - length = int(self.headers.get("Content-Length", 0)) - body = self.rfile.read(length).decode() - params = parse_qs(body) - - if self.path == "/add": - self.handle_add_submit(params) - elif self.path.startswith("/edit/"): - self.handle_edit_submit(self.path, params) - elif self.path == "/style": - self.handle_style_submit(params) - elif self.path == "/import": - self.handle_import_submit(params) - else: - self.respond("

    404

    ", 404) - - def handle_search(self, params): - q = params.get("q", [""])[0].strip() - db = get_db() - count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - name = get_site_name() - - result_html = "" - trusted_html = "" - if q: - rows = db.execute( - "SELECT p.id, p.url, p.title, p.body, p.note " - "FROM pages_fts f JOIN pages p ON f.rowid = p.id " - "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", - (q,), - ).fetchall() - if rows: - for r in rows: - note_html = "" - if r["note"]: - note_html = f'
    {esc(r["note"])}
    ' - result_html += ( - f'
    ' - f'{esc(r["title"])}
    ' - f'{esc(r["url"])}
    ' - f'{esc(snippet(r["body"], q))}' - f'{note_html}' - f'
    ' - ) - else: - result_html = "

    No results in your index.

    " - - # search all linked pages from trusted sites - words = q.lower().split() - all_links = db.execute( - "SELECT l.url, l.label, p.title AS source_title " - "FROM links l JOIN pages p ON l.page_id = p.id", - ).fetchall() - indexed_urls = set(r["url"] for r in rows) if rows else set() - seen = set() - trusted = [] - for l in all_links: - if l["url"] in indexed_urls or l["url"] in seen: - continue - if any(w in l["label"].lower() for w in words): - seen.add(l["url"]) - trusted.append(l) - if len(trusted) >= 20: - break - - if trusted: - items = "" - for l in trusted: - items += ( - f'
  • {esc(l["label"])} ' - f'— from {esc(l["source_title"])}
    • ' - ) - trusted_html = ( - f'
    ' - f'from your trusted sites ({len(trusted)})' - f'
      {items}
    ' - f'
    ' - ) - - db.close() - self.respond( - f'

    {esc(name)}

    ' - f'
    ' - f'' - f' ' - f'
    ' - f'

    {count} page(s) indexed.' - f' + add url' - f' | browse' - f' | customize

    ' - f'
    {result_html}{trusted_html}' - ) - - def handle_add_form(self, msg=""): - self.respond( - f"

    add url

    " - f'
    ' - f'

    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_add_submit(self, params): - url = params.get("url", [""])[0].strip() - note = params.get("note", [""])[0].strip() - if not url: - return self.handle_add_form("URL is required.") - if not url.startswith(("http://", "https://")): - return self.handle_add_form("URL must start with http:// or https://") - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - self.handle_add_form(f'Indexed: {esc(title)}') - except Exception as e: - self.handle_add_form(f"Error: {esc(str(e))}") - - def handle_pages(self): - db = get_db() - rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() - db.close() - items = "" - for r in rows: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - items += ( - f'
  • {esc(r["title"])}{note_html} ' - f'({esc(r["url"])}) ' - f'edit ' - f'remove
    • ' - ) - self.respond( - f"

    indexed pages ({len(rows)})

    " - f"
      {items}
    " - f'

    export | import

    ' - f'back' - ) - - def handle_edit_form(self, path, msg=""): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - db = get_db() - row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() - db.close() - if not row: - return self.respond("

    404

    ", 404) - self.respond( - f"

    edit note

    " - f"

    {esc(row['title'])}
    " - f"{esc(row['url'])}

    " - f'
    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_edit_submit(self, path, params): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - note = params.get("note", [""])[0].strip() - db = get_db() - db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) - db.commit() - db.close() - self.send_response(302) - self.send_header("Location", "/pages") - self.end_headers() - - def handle_delete(self, path): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - db = get_db() - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) - db.commit() - db.close() - self.send_response(302) - self.send_header("Location", "/pages") - self.end_headers() - - def handle_bookmark(self, params): - url = params.get("url", [""])[0].strip() - if not url or not url.startswith(("http://", "https://")): - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Access-Control-Allow-Origin", "*") - self.end_headers() - self.wfile.write(b"error: invalid url") - return - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, '') " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body", - (url, title, body), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - msg = f"ok: {title}" - except Exception as e: - msg = f"error: {e}" - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Access-Control-Allow-Origin", "*") - self.end_headers() - self.wfile.write(msg.encode()) - - def handle_export(self): - db = get_db() - rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() - db.close() - data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Disposition", "attachment; filename=tinyweb-export.json") - self.end_headers() - self.wfile.write(json.dumps(data, indent=2).encode()) - - def handle_import_form(self, msg=""): - self.respond( - f"

    import

    " - f"

    Paste the contents of a tinyweb export file (JSON).

    " - f'
    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_import_submit(self, params): - raw = params.get("data", [""])[0].strip() - if not raw: - return self.handle_import_form("Paste JSON data.") - try: - data = json.loads(raw) - except json.JSONDecodeError: - return self.handle_import_form("Invalid JSON.") - if not isinstance(data, list): - return self.handle_import_form("Expected a JSON array.") - - imported = 0 - errors = 0 - for entry in data: - url = entry.get("url", "").strip() - note = entry.get("note", "").strip() - if not url: - continue - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - imported += 1 - except Exception: - errors += 1 - - self.handle_import_form(f"Imported {imported} page(s). {errors} error(s).") - - def handle_style_form(self, msg=""): - css = get_setting("custom_css") - name = get_site_name() - self.respond( - f"

    customize

    " - f"

    name your search engine

    " - f'
    ' - f'

    ' - f"

    custom css

    " - f"

    Some classes you can target:

    " - f"
    "
-            f"body          - page background, font\n"
-            f"h1            - page titles\n"
-            f"input, button - search bar\n"
-            f"a             - links\n"
-            f".result       - each search result\n"
-            f".note         - your notes on results\n"
-            f".trusted      - trusted sites dropdown\n"
-            f"small         - url text\n"
-            f"ul, li        - browse page list"
-            f"
    " - f'

    ' - f'' - f"
    " - f"

    bookmarklet

    " - f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " - f'

    r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}

    ' - f"

    {msg}

    " - f'back' - ) - - def handle_style_submit(self, params): - css = params.get("css", [""])[0] - name = params.get("site_name", ["tinyweb"])[0].strip() - set_setting("custom_css", css) - set_setting("site_name", name or "tinyweb") - self.handle_style_form("Saved.") + while True: + time.sleep(1) if __name__ == "__main__": - init_db() - print("running on http://0.0.0.0:5001") - HTTPServer(("0.0.0.0", 5001), Handler).serve_forever() + main() diff --git a/db.py b/db.py new file mode 100644 index 0000000..903f824 --- /dev/null +++ b/db.py @@ -0,0 +1,149 @@ +import sqlite3 +import requests +from urllib.parse import urlparse, urljoin +from bs4 import BeautifulSoup + +DATABASE = "index.db" + +SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + + +def get_db(): + db = sqlite3.connect(DATABASE) + db.row_factory = sqlite3.Row + return db + + +def init_db(): + db = sqlite3.connect(DATABASE) + db.execute( + "CREATE TABLE IF NOT EXISTS pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " title TEXT," + " body TEXT," + " note TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " + "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS links (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " page_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " label TEXT," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS settings (" + " key TEXT PRIMARY KEY," + " value TEXT" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS subscriptions (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " name TEXT DEFAULT ''," + " auto_sync INTEGER DEFAULT 0," + " last_sync TEXT DEFAULT ''" + ")" + ) + db.executescript(""" + CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + """) + db.commit() + db.close() + + +def get_setting(key, default=""): + db = get_db() + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + db.close() + return row["value"] if row else default + + +def set_setting(key, value): + db = get_db() + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + db.close() + + +def get_site_name(): + return get_setting("site_name", "tinyweb") + + +def fetch_page(url): + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) + resp.raise_for_status() + soup = BeautifulSoup(resp.text, "html.parser") + + # extract links before stripping tags + domain = urlparse(url).netloc + seen = set() + links = [] + for a in soup.find_all("a", href=True): + href = urljoin(url, a["href"]).split("#")[0] + parsed = urlparse(href) + if parsed.netloc != domain: + continue + if any(href.lower().endswith(ext) for ext in SKIP_EXT): + continue + if parsed.query or "action=" in href: + continue + path = parsed.path.lower() + if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): + continue + if href in seen or href == url: + continue + seen.add(href) + label = a.get_text(strip=True) or href + links.append((href, label[:200])) + + for tag in soup(["script", "style", "nav", "footer", "header"]): + tag.decompose() + title = soup.title.string.strip() if soup.title and soup.title.string else url + body = soup.get_text(separator=" ", strip=True) + return title, body, links + + +def index_url(url, note=""): + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", + (url, title, body, note), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + return title diff --git a/gateway.py b/gateway.py new file mode 100644 index 0000000..0bde2d3 --- /dev/null +++ b/gateway.py @@ -0,0 +1,149 @@ +import sys +import time +import threading +import RNS +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import parse_qs, urlparse + +APP_NAME = "tinyweb" +ASPECTS = ["server"] +GATEWAY_PORT = 8080 +REQUEST_TIMEOUT = 60 + + +class GatewayState: + reticulum = None + destination = None + link = None + link_lock = threading.Lock() + + +def resolve_destination(dest_hash_hex): + dest_hash = bytes.fromhex(dest_hash_hex) + + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + print(f"Requesting path to {RNS.prettyhexrep(dest_hash)}...") + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + raise ConnectionError(f"Could not find path to {RNS.prettyhexrep(dest_hash)}") + + server_identity = RNS.Identity.recall(dest_hash) + GatewayState.destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, + ) + print(f"Resolved destination: {RNS.prettyhexrep(dest_hash)}") + + +def ensure_link(): + with GatewayState.link_lock: + if GatewayState.link and GatewayState.link.status == RNS.Link.ACTIVE: + return GatewayState.link + + print("Establishing link...") + link = RNS.Link(GatewayState.destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + + if link.status != RNS.Link.ACTIVE: + raise ConnectionError("Link establishment failed") + + GatewayState.link = link + print("Link established") + return link + + +class GatewayHandler(BaseHTTPRequestHandler): + + def _forward(self, method): + parsed = urlparse(self.path) + query = parse_qs(parsed.query) + + body = {} + if method == "POST": + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length).decode() + body = parse_qs(raw) + + request_data = { + "method": method, + "path": parsed.path, + "query": query, + "body": body, + "gateway_host": self.headers.get("Host", f"localhost:{GATEWAY_PORT}"), + } + + try: + link = ensure_link() + receipt = link.request( + "/tinyweb", + data=request_data, + timeout=REQUEST_TIMEOUT, + ) + + # Wait for the response + elapsed = 0 + done_statuses = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done_statuses and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + self.send_response(resp["status"]) + self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8")) + for k, v in resp.get("headers", {}).items(): + self.send_header(k, v) + self.end_headers() + resp_body = resp.get("body", "") + if resp_body: + self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body) + elif receipt.get_status() == RNS.RequestReceipt.FAILED: + self.send_error(504, "Request to TinyWeb server failed") + else: + self.send_error(504, "Request to TinyWeb server timed out") + + except ConnectionError as e: + GatewayState.link = None + self.send_error(502, f"Gateway error: {e}") + except Exception as e: + GatewayState.link = None + self.send_error(502, f"Gateway error: {e}") + + def do_GET(self): + self._forward("GET") + + def do_POST(self): + self._forward("POST") + + def log_message(self, format, *args): + print(f"[Gateway] {args[0]}") + + +def main(): + if len(sys.argv) < 2: + print(f"Usage: python gateway.py ") + print(f" The destination hash is printed by app.py on startup.") + sys.exit(1) + + dest_hash = sys.argv[1].replace("<", "").replace(">", "") + + GatewayState.reticulum = RNS.Reticulum() + resolve_destination(dest_hash) + + print(f"Gateway listening on http://localhost:{GATEWAY_PORT}") + print(f"Open http://localhost:{GATEWAY_PORT} in your browser") + HTTPServer(("127.0.0.1", GATEWAY_PORT), GatewayHandler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/handlers.py b/handlers.py new file mode 100644 index 0000000..8f17ee8 --- /dev/null +++ b/handlers.py @@ -0,0 +1,660 @@ +import json +from datetime import datetime +import requests + +from db import get_db, get_setting, set_setting, get_site_name, index_url +from templates import esc, snippet, wrap_page + + +def _respond(body_html, status=200): + return { + "status": status, + "content_type": "text/html; charset=utf-8", + "body": wrap_page(body_html), + "headers": {}, + } + + +def _redirect(location): + return { + "status": 302, + "content_type": "text/html; charset=utf-8", + "body": "", + "headers": {"Location": location}, + } + + +def _json_response(data, status=200, headers=None): + return { + "status": status, + "content_type": "application/json", + "body": json.dumps(data, indent=2), + "headers": headers or {}, + } + + +def _text_response(text, status=200, headers=None): + return { + "status": status, + "content_type": "text/plain", + "body": text, + "headers": headers or {}, + } + + +def _error(status): + return _respond(f"

    {status}

    ", status) + + +# --- Route handlers --- + + +def handle_search(query): + q = query.get("q", [""])[0].strip() + db = get_db() + count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + name = get_site_name() + + result_html = "" + trusted_html = "" + if q: + rows = db.execute( + "SELECT p.id, p.url, p.title, p.body, p.note " + "FROM pages_fts f JOIN pages p ON f.rowid = p.id " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", + (q,), + ).fetchall() + if rows: + for r in rows: + note_html = "" + if r["note"]: + note_html = f'
    {esc(r["note"])}
    ' + result_html += ( + f'
    ' + f'{esc(r["title"])}
    ' + f'{esc(r["url"])}
    ' + f'{esc(snippet(r["body"], q))}' + f'{note_html}' + f'
    ' + ) + else: + result_html = "

    No results in your index.

    " + + # search all linked pages from trusted sites + words = q.lower().split() + all_links = db.execute( + "SELECT l.url, l.label, p.title AS source_title " + "FROM links l JOIN pages p ON l.page_id = p.id", + ).fetchall() + indexed_urls = set(r["url"] for r in rows) if rows else set() + seen = set() + trusted = [] + for l in all_links: + if l["url"] in indexed_urls or l["url"] in seen: + continue + if any(w in l["label"].lower() for w in words): + seen.add(l["url"]) + trusted.append(l) + if len(trusted) >= 20: + break + + if trusted: + items = "" + for l in trusted: + items += ( + f'
  • {esc(l["label"])} ' + f'— from {esc(l["source_title"])}
    • ' + ) + trusted_html = ( + f'
    ' + f'from your trusted sites ({len(trusted)})' + f'
      {items}
    ' + f'
    ' + ) + + db.close() + return _respond( + f'

    {esc(name)}

    ' + f'
    ' + f'' + f' ' + f'
    ' + f'

    {count} page(s) indexed.' + f' + add url' + f' | browse' + f' | subscriptions' + f' | customize

    ' + f'
    {result_html}{trusted_html}' + ) + + +def handle_add_form(msg=""): + return _respond( + f"

    add url

    " + f'
    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_add_submit(body): + url = body.get("url", [""])[0].strip() + note = body.get("note", [""])[0].strip() + if not url: + return handle_add_form("URL is required.") + if not url.startswith(("http://", "https://")): + return handle_add_form("URL must start with http:// or https://") + try: + title = index_url(url, note) + return handle_add_form(f'Indexed: {esc(title)}') + except Exception as e: + return handle_add_form(f"Error: {esc(str(e))}") + + +def handle_pages(): + db = get_db() + rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() + db.close() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + items += ( + f'
  • {esc(r["title"])}{note_html} ' + f'({esc(r["url"])}) ' + f'edit ' + f'remove
    • ' + ) + return _respond( + f"

    indexed pages ({len(rows)})

    " + f"
      {items}
    " + f'

    export | import

    ' + f'back' + ) + + +def handle_edit_form(page_id, msg=""): + db = get_db() + row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() + db.close() + if not row: + return _error(404) + return _respond( + f"

    edit note

    " + f"

    {esc(row['title'])}
    " + f"{esc(row['url'])}

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_edit_submit(page_id, body): + note = body.get("note", [""])[0].strip() + db = get_db() + db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) + db.commit() + db.close() + return _redirect("/pages") + + +def handle_delete(page_id): + db = get_db() + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) + db.commit() + db.close() + return _redirect("/pages") + + +def handle_bookmark(query): + url = query.get("url", [""])[0].strip() + if not url or not url.startswith(("http://", "https://")): + return _text_response("error: invalid url", headers={"Access-Control-Allow-Origin": "*"}) + try: + title = index_url(url) + msg = f"ok: {title}" + except Exception as e: + msg = f"error: {e}" + return _text_response(msg, headers={"Access-Control-Allow-Origin": "*"}) + + +def handle_export(): + db = get_db() + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() + db.close() + data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] + return _json_response(data, headers={"Content-Disposition": "attachment; filename=tinyweb-export.json"}) + + +def handle_import_form(msg=""): + return _respond( + f"

    import

    " + f"

    Paste the contents of a tinyweb export file (JSON).

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_import_submit(body): + raw = body.get("data", [""])[0].strip() + if not raw: + return handle_import_form("Paste JSON data.") + try: + data = json.loads(raw) + except json.JSONDecodeError: + return handle_import_form("Invalid JSON.") + if not isinstance(data, list): + return handle_import_form("Expected a JSON array.") + + imported = 0 + errors = 0 + for entry in data: + url = entry.get("url", "").strip() + note = entry.get("note", "").strip() + if not url: + continue + try: + index_url(url, note) + imported += 1 + except Exception: + errors += 1 + + return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") + + +def handle_style_form(msg="", gateway_host=""): + css = get_setting("custom_css") + name = get_site_name() + sharing = get_setting("sharing_enabled", "0") + checked = " checked" if sharing == "1" else "" + host = gateway_host or "localhost:8080" + return _respond( + f"

    customize

    " + f"

    name your search engine

    " + f'
    ' + f'

    ' + f"

    sharing

    " + f'

    " + f"

    custom css

    " + f"

    Some classes you can target:

    " + f"
    "
+        f"body          - page background, font\n"
+        f"h1            - page titles\n"
+        f"input, button - search bar\n"
+        f"a             - links\n"
+        f".result       - each search result\n"
+        f".note         - your notes on results\n"
+        f".trusted      - trusted sites dropdown\n"
+        f"small         - url text\n"
+        f"ul, li        - browse page list"
+        f"
    " + f'

    ' + f'' + f"
    " + f"

    bookmarklet

    " + f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " + f'

    + save to {esc(name)}

    ' + f"

    {msg}

    " + f'back' + ) + + +def handle_style_submit(body): + css = body.get("css", [""])[0] + name = body.get("site_name", ["tinyweb"])[0].strip() + sharing = "1" if body.get("sharing_enabled") else "0" + set_setting("custom_css", css) + set_setting("site_name", name or "tinyweb") + set_setting("sharing_enabled", sharing) + return handle_style_form("Saved.") + + +def handle_api_sites(): + if get_setting("sharing_enabled", "0") != "1": + return _json_response( + {"error": "sharing disabled"}, + status=403, + headers={"Access-Control-Allow-Origin": "*"}, + ) + db = get_db() + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id DESC").fetchall() + db.close() + data = { + "name": get_site_name(), + "sites": [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows], + } + return _json_response(data, headers={"Access-Control-Allow-Origin": "*"}) + + +def handle_subscriptions(msg=""): + db = get_db() + subs = db.execute("SELECT * FROM subscriptions ORDER BY id DESC").fetchall() + db.close() + items = "" + for s in subs: + auto_label = "on" if s["auto_sync"] else "off" + last = s["last_sync"] or "never" + items += ( + f'' + f'{esc(s["name"] or "unknown")}
    {esc(s["url"])}' + f'{esc(last)}' + f'' + f'
    ' + f'
    ' + f'' + f'' + f'browse ' + f'
    ' + f'
    ' + f'
    ' + f'
    ' + f'' + f'' + ) + table = "" + if subs: + table = ( + f'' + f'{items}
    instancelast syncauto-syncactions
    ' + f'
    ' + f'
    ' + ) + return _respond( + f"

    subscriptions

    " + f'
    ' + f' ' + f'' + f'
    ' + f'

    {msg}

    ' + f'
    {table}' + f'
    back' + ) + + +def handle_subscription_add(body): + url = body.get("url", [""])[0].strip().rstrip("/") + if not url or not url.startswith(("http://", "https://")): + return handle_subscriptions("URL must start with http:// or https://") + try: + resp = requests.get(f"{url}/api/sites", timeout=5) + if resp.status_code == 403: + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + data = resp.json() + name = data.get("name", "") + except Exception as e: + return handle_subscriptions(f"Could not reach that instance: {esc(str(e))}") + db = get_db() + try: + db.execute( + "INSERT INTO subscriptions (url, name) VALUES (?, ?) " + "ON CONFLICT(url) DO UPDATE SET name=excluded.name", + (url, name), + ) + db.commit() + finally: + db.close() + return handle_subscriptions(f"Subscribed to {esc(name or url)}.") + + +def handle_subscription_browse(sub_id): + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + if not sub: + db.close() + return _error(404) + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + db.close() + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code == 403: + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + sites = resp.json().get("sites", []) + except Exception as e: + return handle_subscriptions(f"Could not fetch sites: {esc(str(e))}") + + new_items = "" + existing_items = "" + new_count = 0 + for s in sites: + if s["url"] in local_urls: + existing_items += ( + f'
  • {esc(s["title"])} ' + f'({esc(s["url"])}) — already indexed
    • ' + ) + else: + new_count += 1 + note_html = f' — {esc(s["note"])}' if s.get("note") else "" + new_items += ( + f'
    • ' + ) + + buttons = "" + if new_count: + buttons = ' ' + return _respond( + f'

    browsing: {esc(sub["name"] or sub["url"])}

    ' + f'

    {len(sites)} site(s) available, {new_count} new

    ' + f'
    ' + f'' + f'
      {new_items}
    ' + f'{buttons}' + f'
    ' + f'

    already indexed

      {existing_items}
    ' + f'back' + ) + + +def handle_subscription_pick(body): + sub_id = body.get("sub_id", [""])[0] + import_all = body.get("import_all", [""])[0] + + if import_all: + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + db.close() + if not sub: + return handle_subscriptions("Subscription not found.") + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + resp.raise_for_status() + sites = resp.json().get("sites", []) + except Exception as e: + return handle_subscriptions(f"Error: {esc(str(e))}") + urls = [s["url"] for s in sites if s["url"] not in local_urls] + else: + urls = body.get("urls", []) + + if not urls: + return handle_subscriptions("No sites selected.") + + imported = 0 + errors = 0 + for url in urls: + try: + index_url(url) + imported += 1 + except Exception: + errors += 1 + return handle_subscriptions(f"Imported {imported} page(s). {errors} error(s).") + + +def handle_subscription_sync(sub_id): + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + if not sub: + db.close() + return handle_subscriptions("Subscription not found.") + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code == 403: + db.close() + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + data = resp.json() + sites = data.get("sites", []) + remote_name = data.get("name", sub["name"]) + except Exception as e: + db.close() + return handle_subscriptions(f"Could not sync: {esc(str(e))}") + + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + synced = 0 + for s in sites: + if s["url"] in local_urls: + continue + try: + db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", + (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + ) + synced += 1 + except Exception: + pass + now = datetime.now().strftime("%Y-%m-%d %H:%M") + db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id)) + db.commit() + db.close() + return handle_subscriptions(f"Synced {synced} new site(s) from {esc(remote_name)}.") + + +def handle_subscription_autosync(sub_id): + db = get_db() + db.execute("UPDATE subscriptions SET auto_sync = 1 - auto_sync WHERE id = ?", (sub_id,)) + db.commit() + db.close() + return _redirect("/subscriptions") + + +def handle_subscription_delete(sub_id): + db = get_db() + db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,)) + db.commit() + db.close() + return _redirect("/subscriptions") + + +def handle_subscription_syncall(): + db = get_db() + subs = db.execute("SELECT * FROM subscriptions WHERE auto_sync = 1").fetchall() + db.close() + if not subs: + return handle_subscriptions("No subscriptions have auto-sync enabled.") + total = 0 + for sub in subs: + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code != 200: + continue + data = resp.json() + sites = data.get("sites", []) + remote_name = data.get("name", sub["name"]) + db = get_db() + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + for s in sites: + if s["url"] in local_urls: + continue + try: + db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", + (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + ) + except Exception: + pass + now = datetime.now().strftime("%Y-%m-%d %H:%M") + db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub["id"])) + db.commit() + db.close() + total += 1 + except Exception: + pass + return handle_subscriptions(f"Synced {total} subscription(s).") + + +# --- Dispatcher --- + + +def dispatch_request(data): + method = data.get("method", "GET") + path = data.get("path", "/") + query = data.get("query", {}) + body = data.get("body", {}) + gateway_host = data.get("gateway_host", "") + + def extract_id(prefix): + try: + return int(path[len(prefix):]) + except (ValueError, IndexError): + return None + + if method == "GET": + if path == "/": + return handle_search(query) + elif path == "/add": + return handle_add_form() + elif path == "/pages": + return handle_pages() + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_form(pid) if pid is not None else _error(400) + elif path.startswith("/delete/"): + pid = extract_id("/delete/") + return handle_delete(pid) if pid is not None else _error(400) + elif path == "/bookmark": + return handle_bookmark(query) + elif path == "/style": + return handle_style_form(gateway_host=gateway_host) + elif path == "/export": + return handle_export() + elif path == "/import": + return handle_import_form() + elif path == "/api/sites": + return handle_api_sites() + elif path == "/subscriptions": + return handle_subscriptions() + elif path.startswith("/subscriptions/browse/"): + sid = extract_id("/subscriptions/browse/") + return handle_subscription_browse(sid) if sid is not None else _error(400) + elif method == "POST": + if path == "/add": + return handle_add_submit(body) + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_submit(pid, body) if pid is not None else _error(400) + elif path == "/style": + return handle_style_submit(body) + elif path == "/import": + return handle_import_submit(body) + elif path == "/subscriptions/add": + return handle_subscription_add(body) + elif path == "/subscriptions/pick": + return handle_subscription_pick(body) + elif path.startswith("/subscriptions/sync/"): + sid = extract_id("/subscriptions/sync/") + return handle_subscription_sync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/autosync/"): + sid = extract_id("/subscriptions/autosync/") + return handle_subscription_autosync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/delete/"): + sid = extract_id("/subscriptions/delete/") + return handle_subscription_delete(sid) if sid is not None else _error(400) + elif path == "/subscriptions/syncall": + return handle_subscription_syncall() + + return _error(404) diff --git a/requirements.txt b/requirements.txt index 1190bd8..f63da5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests beautifulsoup4 +rns diff --git a/templates.py b/templates.py new file mode 100644 index 0000000..735a38e --- /dev/null +++ b/templates.py @@ -0,0 +1,21 @@ +import html +from db import get_setting + + +def esc(s): + return html.escape(str(s)) + + +def snippet(text, query, ctx=80): + pos = text.lower().find(query.lower()) + if pos == -1: + return text[:200] + start = max(0, pos - ctx) + end = min(len(text), pos + len(query) + ctx) + return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") + + +def wrap_page(body_html): + css = get_setting("custom_css") + style = f"" if css else "" + return f"{style}{body_html}" From 8992c4ce20aad26ad8de3b24b76a48bc3d40ce51 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 22:51:22 -0700 Subject: [PATCH 007/194] wired up mesh subscriptions + search - Subscriptions now use Reticulum destination hashes instead of HTTP URLs - All subscription syncing happens over encrypted RNS links (rns_client.py) - Add remote_pages table for synced content from subscriptions - Search results now include pages from synced subscriptions, grouped by source - Remove HTTP dependency from subscription handlers --- db.py | 37 ++++++++++++- handlers.py | 142 +++++++++++++++++++++++++++++++------------------- rns_client.py | 78 +++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 56 deletions(-) create mode 100644 rns_client.py diff --git a/db.py b/db.py index 903f824..5a86a1c 100644 --- a/db.py +++ b/db.py @@ -47,12 +47,27 @@ def init_db(): db.execute( "CREATE TABLE IF NOT EXISTS subscriptions (" " id INTEGER PRIMARY KEY AUTOINCREMENT," - " url TEXT UNIQUE NOT NULL," + " dest_hash TEXT UNIQUE NOT NULL," " name TEXT DEFAULT ''," " auto_sync INTEGER DEFAULT 0," " last_sync TEXT DEFAULT ''" ")" ) + db.execute( + "CREATE TABLE IF NOT EXISTS remote_pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " subscription_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " title TEXT," + " note TEXT DEFAULT ''," + " FOREIGN KEY (subscription_id) REFERENCES subscriptions(id) ON DELETE CASCADE," + " UNIQUE(subscription_id, url)" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS remote_pages_fts " + "USING fts5(title, url, note, content=remote_pages, content_rowid=id)" + ) db.executescript(""" CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN INSERT INTO pages_fts(rowid, title, body, url, note) @@ -68,7 +83,27 @@ def init_db(): INSERT INTO pages_fts(rowid, title, body, url, note) VALUES (new.id, new.title, new.body, new.url, new.note); END; + CREATE TRIGGER IF NOT EXISTS remote_pages_ai AFTER INSERT ON remote_pages BEGIN + INSERT INTO remote_pages_fts(rowid, title, url, note) + VALUES (new.id, new.title, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS remote_pages_ad AFTER DELETE ON remote_pages BEGIN + INSERT INTO remote_pages_fts(remote_pages_fts, rowid, title, url, note) + VALUES ('delete', old.id, old.title, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS remote_pages_au AFTER UPDATE ON remote_pages BEGIN + INSERT INTO remote_pages_fts(remote_pages_fts, rowid, title, url, note) + VALUES ('delete', old.id, old.title, old.url, old.note); + INSERT INTO remote_pages_fts(rowid, title, url, note) + VALUES (new.id, new.title, new.url, new.note); + END; """) + # Migrate old subscriptions table if needed + cols = [row[1] for row in db.execute("PRAGMA table_info(subscriptions)").fetchall()] + if "url" in cols and "dest_hash" not in cols: + db.execute("ALTER TABLE subscriptions RENAME COLUMN url TO dest_hash") + db.commit() + db.commit() db.close() diff --git a/handlers.py b/handlers.py index 8f17ee8..ec5f6dd 100644 --- a/handlers.py +++ b/handlers.py @@ -1,9 +1,9 @@ import json from datetime import datetime -import requests from db import get_db, get_setting, set_setting, get_site_name, index_url from templates import esc, snippet, wrap_page +from rns_client import fetch_remote_sites def _respond(body_html, status=200): @@ -112,7 +112,42 @@ def handle_search(query): f'' ) + # search synced pages from subscriptions + remote_rows = db.execute( + "SELECT rp.url, rp.title, rp.note, s.name AS source_name " + "FROM remote_pages_fts rpf " + "JOIN remote_pages rp ON rpf.rowid = rp.id " + "JOIN subscriptions s ON rp.subscription_id = s.id " + "WHERE remote_pages_fts MATCH ? ORDER BY rank LIMIT 50", + (q,), + ).fetchall() + + remote_html = "" + if q and remote_rows: + # group by source + by_source = {} + for r in remote_rows: + source = r["source_name"] or "unknown" + by_source.setdefault(source, []).append(r) + for source, items in by_source.items(): + source_items = "" + for r in items: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + source_items += ( + f'
  • {esc(r["title"])}' + f'{note_html} ({esc(r["url"])})
    • ' + ) + remote_html += ( + f'
    ' + f'from {esc(source)} ({len(items)})' + f'
      {source_items}
    ' + f'
    ' + ) + db.close() + sub_count = "" + if q and remote_rows: + sub_count = f" + {len(remote_rows)} from subscriptions" return _respond( f'

    {esc(name)}

    ' f'
    ' @@ -124,7 +159,7 @@ def handle_search(query): f' | browse' f' | subscriptions' f' | customize

    ' - f'
    {result_html}{trusted_html}' + f'
    {result_html}{trusted_html}{remote_html}' ) @@ -348,7 +383,7 @@ def handle_subscriptions(msg=""): last = s["last_sync"] or "never" items += ( f'' - f'{esc(s["name"] or "unknown")}
    {esc(s["url"])}' + f'{esc(s["name"] or "unknown")}
    {esc(s["dest_hash"])}' f'{esc(last)}' f'' f'' @@ -374,7 +409,7 @@ def handle_subscriptions(msg=""): return _respond( f"

    subscriptions

    " f'' - f' ' + f' ' f'' f'
    ' f'

    {msg}

    ' @@ -384,29 +419,31 @@ def handle_subscriptions(msg=""): def handle_subscription_add(body): - url = body.get("url", [""])[0].strip().rstrip("/") - if not url or not url.startswith(("http://", "https://")): - return handle_subscriptions("URL must start with http:// or https://") + dest_hash = body.get("dest_hash", [""])[0].strip().replace("<", "").replace(">", "") + if not dest_hash or len(dest_hash) != 32: + return handle_subscriptions("Enter a valid 32-character destination hash.") try: - resp = requests.get(f"{url}/api/sites", timeout=5) - if resp.status_code == 403: - return handle_subscriptions("That instance has sharing disabled.") - resp.raise_for_status() - data = resp.json() + int(dest_hash, 16) + except ValueError: + return handle_subscriptions("Invalid destination hash (must be hex).") + try: + data = fetch_remote_sites(dest_hash) name = data.get("name", "") + except PermissionError: + return handle_subscriptions("That instance has sharing disabled.") except Exception as e: return handle_subscriptions(f"Could not reach that instance: {esc(str(e))}") db = get_db() try: db.execute( - "INSERT INTO subscriptions (url, name) VALUES (?, ?) " - "ON CONFLICT(url) DO UPDATE SET name=excluded.name", - (url, name), + "INSERT INTO subscriptions (dest_hash, name) VALUES (?, ?) " + "ON CONFLICT(dest_hash) DO UPDATE SET name=excluded.name", + (dest_hash, name), ) db.commit() finally: db.close() - return handle_subscriptions(f"Subscribed to {esc(name or url)}.") + return handle_subscriptions(f"Subscribed to {esc(name or dest_hash)}.") def handle_subscription_browse(sub_id): @@ -416,15 +453,24 @@ def handle_subscription_browse(sub_id): db.close() return _error(404) local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + + # Use locally synced data if available, otherwise fetch live + remote_rows = db.execute( + "SELECT url, title, note FROM remote_pages WHERE subscription_id = ?", + (sub_id,), + ).fetchall() db.close() - try: - resp = requests.get(f"{sub['url']}/api/sites", timeout=5) - if resp.status_code == 403: + + if remote_rows: + sites = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in remote_rows] + else: + try: + data = fetch_remote_sites(sub["dest_hash"]) + sites = data.get("sites", []) + except PermissionError: return handle_subscriptions("That instance has sharing disabled.") - resp.raise_for_status() - sites = resp.json().get("sites", []) - except Exception as e: - return handle_subscriptions(f"Could not fetch sites: {esc(str(e))}") + except Exception as e: + return handle_subscriptions(f"Could not fetch sites: {esc(str(e))}") new_items = "" existing_items = "" @@ -448,7 +494,7 @@ def handle_subscription_browse(sub_id): if new_count: buttons = ' ' return _respond( - f'

    browsing: {esc(sub["name"] or sub["url"])}

    ' + f'

    browsing: {esc(sub["name"] or sub["dest_hash"])}

    ' f'

    {len(sites)} site(s) available, {new_count} new

    ' f'
    ' f'' @@ -466,18 +512,12 @@ def handle_subscription_pick(body): if import_all: db = get_db() - sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + remote = db.execute( + "SELECT url FROM remote_pages WHERE subscription_id = ?", (sub_id,) + ).fetchall() db.close() - if not sub: - return handle_subscriptions("Subscription not found.") - try: - resp = requests.get(f"{sub['url']}/api/sites", timeout=5) - resp.raise_for_status() - sites = resp.json().get("sites", []) - except Exception as e: - return handle_subscriptions(f"Error: {esc(str(e))}") - urls = [s["url"] for s in sites if s["url"] not in local_urls] + urls = [r["url"] for r in remote if r["url"] not in local_urls] else: urls = body.get("urls", []) @@ -502,27 +542,24 @@ def handle_subscription_sync(sub_id): db.close() return handle_subscriptions("Subscription not found.") try: - resp = requests.get(f"{sub['url']}/api/sites", timeout=5) - if resp.status_code == 403: - db.close() - return handle_subscriptions("That instance has sharing disabled.") - resp.raise_for_status() - data = resp.json() + data = fetch_remote_sites(sub["dest_hash"]) sites = data.get("sites", []) remote_name = data.get("name", sub["name"]) + except PermissionError: + db.close() + return handle_subscriptions("That instance has sharing disabled.") except Exception as e: db.close() return handle_subscriptions(f"Could not sync: {esc(str(e))}") - local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + # Clear old remote pages for this subscription and re-insert + db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub_id,)) synced = 0 for s in sites: - if s["url"] in local_urls: - continue try: db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", - (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + "INSERT INTO remote_pages (subscription_id, url, title, note) VALUES (?, ?, ?, ?)", + (sub_id, s["url"], s["title"], s.get("note", "")), ) synced += 1 except Exception: @@ -531,7 +568,7 @@ def handle_subscription_sync(sub_id): db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id)) db.commit() db.close() - return handle_subscriptions(f"Synced {synced} new site(s) from {esc(remote_name)}.") + return handle_subscriptions(f"Synced {synced} site(s) from {esc(remote_name)}.") def handle_subscription_autosync(sub_id): @@ -559,21 +596,16 @@ def handle_subscription_syncall(): total = 0 for sub in subs: try: - resp = requests.get(f"{sub['url']}/api/sites", timeout=5) - if resp.status_code != 200: - continue - data = resp.json() + data = fetch_remote_sites(sub["dest_hash"]) sites = data.get("sites", []) remote_name = data.get("name", sub["name"]) db = get_db() - local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub["id"],)) for s in sites: - if s["url"] in local_urls: - continue try: db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", - (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + "INSERT INTO remote_pages (subscription_id, url, title, note) VALUES (?, ?, ?, ?)", + (sub["id"], s["url"], s["title"], s.get("note", "")), ) except Exception: pass diff --git a/rns_client.py b/rns_client.py new file mode 100644 index 0000000..32eeadc --- /dev/null +++ b/rns_client.py @@ -0,0 +1,78 @@ +import time +import RNS + +APP_NAME = "tinyweb" +ASPECTS = ["server"] +REQUEST_TIMEOUT = 30 + + +def fetch_remote_sites(dest_hash_hex): + """ + Connect to a remote TinyWeb instance over Reticulum and fetch its + shared sites. Returns the response dict from /api/sites, or raises + an exception on failure. + """ + dest_hash = bytes.fromhex(dest_hash_hex) + + # Resolve path if needed + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + raise ConnectionError(f"Could not find path to {dest_hash_hex}") + + server_identity = RNS.Identity.recall(dest_hash) + if server_identity is None: + raise ConnectionError(f"Could not recall identity for {dest_hash_hex}") + + destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, + ) + + # Establish link + link = RNS.Link(destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + + if link.status != RNS.Link.ACTIVE: + raise ConnectionError(f"Could not establish link to {dest_hash_hex}") + + try: + # Request /api/sites + request_data = { + "method": "GET", + "path": "/api/sites", + "query": {}, + "body": {}, + "gateway_host": "", + } + + receipt = link.request("/tinyweb", data=request_data, timeout=REQUEST_TIMEOUT) + + elapsed = 0 + done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT: + time.sleep(0.5) + elapsed += 0.5 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + if resp["status"] == 403: + raise PermissionError("That instance has sharing disabled.") + if resp["status"] != 200: + raise ConnectionError(f"Remote returned status {resp['status']}") + import json + return json.loads(resp["body"]) + else: + raise ConnectionError(f"Request failed or timed out") + finally: + link.teardown() From 8ae26a5597aa24a49f78ff1110629c33c20667e9 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 22:51:22 -0700 Subject: [PATCH 008/194] wired up mesh subscriptions + search - Subscriptions now use Reticulum destination hashes instead of HTTP URLs - All subscription syncing happens over encrypted RNS links (rns_client.py) - Add remote_pages table for synced content from subscriptions - Search results now include pages from synced subscriptions, grouped by source - Remove HTTP dependency from subscription handlers --- db.py | 37 ++++++++++++- handlers.py | 142 +++++++++++++++++++++++++++++++------------------- rns_client.py | 78 +++++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 56 deletions(-) create mode 100644 rns_client.py diff --git a/db.py b/db.py index 903f824..5a86a1c 100644 --- a/db.py +++ b/db.py @@ -47,12 +47,27 @@ def init_db(): db.execute( "CREATE TABLE IF NOT EXISTS subscriptions (" " id INTEGER PRIMARY KEY AUTOINCREMENT," - " url TEXT UNIQUE NOT NULL," + " dest_hash TEXT UNIQUE NOT NULL," " name TEXT DEFAULT ''," " auto_sync INTEGER DEFAULT 0," " last_sync TEXT DEFAULT ''" ")" ) + db.execute( + "CREATE TABLE IF NOT EXISTS remote_pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " subscription_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " title TEXT," + " note TEXT DEFAULT ''," + " FOREIGN KEY (subscription_id) REFERENCES subscriptions(id) ON DELETE CASCADE," + " UNIQUE(subscription_id, url)" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS remote_pages_fts " + "USING fts5(title, url, note, content=remote_pages, content_rowid=id)" + ) db.executescript(""" CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN INSERT INTO pages_fts(rowid, title, body, url, note) @@ -68,7 +83,27 @@ def init_db(): INSERT INTO pages_fts(rowid, title, body, url, note) VALUES (new.id, new.title, new.body, new.url, new.note); END; + CREATE TRIGGER IF NOT EXISTS remote_pages_ai AFTER INSERT ON remote_pages BEGIN + INSERT INTO remote_pages_fts(rowid, title, url, note) + VALUES (new.id, new.title, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS remote_pages_ad AFTER DELETE ON remote_pages BEGIN + INSERT INTO remote_pages_fts(remote_pages_fts, rowid, title, url, note) + VALUES ('delete', old.id, old.title, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS remote_pages_au AFTER UPDATE ON remote_pages BEGIN + INSERT INTO remote_pages_fts(remote_pages_fts, rowid, title, url, note) + VALUES ('delete', old.id, old.title, old.url, old.note); + INSERT INTO remote_pages_fts(rowid, title, url, note) + VALUES (new.id, new.title, new.url, new.note); + END; """) + # Migrate old subscriptions table if needed + cols = [row[1] for row in db.execute("PRAGMA table_info(subscriptions)").fetchall()] + if "url" in cols and "dest_hash" not in cols: + db.execute("ALTER TABLE subscriptions RENAME COLUMN url TO dest_hash") + db.commit() + db.commit() db.close() diff --git a/handlers.py b/handlers.py index 8f17ee8..ec5f6dd 100644 --- a/handlers.py +++ b/handlers.py @@ -1,9 +1,9 @@ import json from datetime import datetime -import requests from db import get_db, get_setting, set_setting, get_site_name, index_url from templates import esc, snippet, wrap_page +from rns_client import fetch_remote_sites def _respond(body_html, status=200): @@ -112,7 +112,42 @@ def handle_search(query): f'' ) + # search synced pages from subscriptions + remote_rows = db.execute( + "SELECT rp.url, rp.title, rp.note, s.name AS source_name " + "FROM remote_pages_fts rpf " + "JOIN remote_pages rp ON rpf.rowid = rp.id " + "JOIN subscriptions s ON rp.subscription_id = s.id " + "WHERE remote_pages_fts MATCH ? ORDER BY rank LIMIT 50", + (q,), + ).fetchall() + + remote_html = "" + if q and remote_rows: + # group by source + by_source = {} + for r in remote_rows: + source = r["source_name"] or "unknown" + by_source.setdefault(source, []).append(r) + for source, items in by_source.items(): + source_items = "" + for r in items: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + source_items += ( + f'
  • {esc(r["title"])}' + f'{note_html} ({esc(r["url"])})
    • ' + ) + remote_html += ( + f'
    ' + f'from {esc(source)} ({len(items)})' + f'
      {source_items}
    ' + f'
    ' + ) + db.close() + sub_count = "" + if q and remote_rows: + sub_count = f" + {len(remote_rows)} from subscriptions" return _respond( f'

    {esc(name)}

    ' f'' @@ -124,7 +159,7 @@ def handle_search(query): f' | browse' f' | subscriptions' f' | customize

    ' - f'
    {result_html}{trusted_html}' + f'
    {result_html}{trusted_html}{remote_html}' ) @@ -348,7 +383,7 @@ def handle_subscriptions(msg=""): last = s["last_sync"] or "never" items += ( f'' - f'{esc(s["name"] or "unknown")}
    {esc(s["url"])}' + f'{esc(s["name"] or "unknown")}
    {esc(s["dest_hash"])}' f'{esc(last)}' f'' f'' @@ -374,7 +409,7 @@ def handle_subscriptions(msg=""): return _respond( f"

    subscriptions

    " f'' - f' ' + f' ' f'' f'' f'

    {msg}

    ' @@ -384,29 +419,31 @@ def handle_subscriptions(msg=""): def handle_subscription_add(body): - url = body.get("url", [""])[0].strip().rstrip("/") - if not url or not url.startswith(("http://", "https://")): - return handle_subscriptions("URL must start with http:// or https://") + dest_hash = body.get("dest_hash", [""])[0].strip().replace("<", "").replace(">", "") + if not dest_hash or len(dest_hash) != 32: + return handle_subscriptions("Enter a valid 32-character destination hash.") try: - resp = requests.get(f"{url}/api/sites", timeout=5) - if resp.status_code == 403: - return handle_subscriptions("That instance has sharing disabled.") - resp.raise_for_status() - data = resp.json() + int(dest_hash, 16) + except ValueError: + return handle_subscriptions("Invalid destination hash (must be hex).") + try: + data = fetch_remote_sites(dest_hash) name = data.get("name", "") + except PermissionError: + return handle_subscriptions("That instance has sharing disabled.") except Exception as e: return handle_subscriptions(f"Could not reach that instance: {esc(str(e))}") db = get_db() try: db.execute( - "INSERT INTO subscriptions (url, name) VALUES (?, ?) " - "ON CONFLICT(url) DO UPDATE SET name=excluded.name", - (url, name), + "INSERT INTO subscriptions (dest_hash, name) VALUES (?, ?) " + "ON CONFLICT(dest_hash) DO UPDATE SET name=excluded.name", + (dest_hash, name), ) db.commit() finally: db.close() - return handle_subscriptions(f"Subscribed to {esc(name or url)}.") + return handle_subscriptions(f"Subscribed to {esc(name or dest_hash)}.") def handle_subscription_browse(sub_id): @@ -416,15 +453,24 @@ def handle_subscription_browse(sub_id): db.close() return _error(404) local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + + # Use locally synced data if available, otherwise fetch live + remote_rows = db.execute( + "SELECT url, title, note FROM remote_pages WHERE subscription_id = ?", + (sub_id,), + ).fetchall() db.close() - try: - resp = requests.get(f"{sub['url']}/api/sites", timeout=5) - if resp.status_code == 403: + + if remote_rows: + sites = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in remote_rows] + else: + try: + data = fetch_remote_sites(sub["dest_hash"]) + sites = data.get("sites", []) + except PermissionError: return handle_subscriptions("That instance has sharing disabled.") - resp.raise_for_status() - sites = resp.json().get("sites", []) - except Exception as e: - return handle_subscriptions(f"Could not fetch sites: {esc(str(e))}") + except Exception as e: + return handle_subscriptions(f"Could not fetch sites: {esc(str(e))}") new_items = "" existing_items = "" @@ -448,7 +494,7 @@ def handle_subscription_browse(sub_id): if new_count: buttons = ' ' return _respond( - f'

    browsing: {esc(sub["name"] or sub["url"])}

    ' + f'

    browsing: {esc(sub["name"] or sub["dest_hash"])}

    ' f'

    {len(sites)} site(s) available, {new_count} new

    ' f'
    ' f'' @@ -466,18 +512,12 @@ def handle_subscription_pick(body): if import_all: db = get_db() - sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + remote = db.execute( + "SELECT url FROM remote_pages WHERE subscription_id = ?", (sub_id,) + ).fetchall() db.close() - if not sub: - return handle_subscriptions("Subscription not found.") - try: - resp = requests.get(f"{sub['url']}/api/sites", timeout=5) - resp.raise_for_status() - sites = resp.json().get("sites", []) - except Exception as e: - return handle_subscriptions(f"Error: {esc(str(e))}") - urls = [s["url"] for s in sites if s["url"] not in local_urls] + urls = [r["url"] for r in remote if r["url"] not in local_urls] else: urls = body.get("urls", []) @@ -502,27 +542,24 @@ def handle_subscription_sync(sub_id): db.close() return handle_subscriptions("Subscription not found.") try: - resp = requests.get(f"{sub['url']}/api/sites", timeout=5) - if resp.status_code == 403: - db.close() - return handle_subscriptions("That instance has sharing disabled.") - resp.raise_for_status() - data = resp.json() + data = fetch_remote_sites(sub["dest_hash"]) sites = data.get("sites", []) remote_name = data.get("name", sub["name"]) + except PermissionError: + db.close() + return handle_subscriptions("That instance has sharing disabled.") except Exception as e: db.close() return handle_subscriptions(f"Could not sync: {esc(str(e))}") - local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + # Clear old remote pages for this subscription and re-insert + db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub_id,)) synced = 0 for s in sites: - if s["url"] in local_urls: - continue try: db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", - (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + "INSERT INTO remote_pages (subscription_id, url, title, note) VALUES (?, ?, ?, ?)", + (sub_id, s["url"], s["title"], s.get("note", "")), ) synced += 1 except Exception: @@ -531,7 +568,7 @@ def handle_subscription_sync(sub_id): db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id)) db.commit() db.close() - return handle_subscriptions(f"Synced {synced} new site(s) from {esc(remote_name)}.") + return handle_subscriptions(f"Synced {synced} site(s) from {esc(remote_name)}.") def handle_subscription_autosync(sub_id): @@ -559,21 +596,16 @@ def handle_subscription_syncall(): total = 0 for sub in subs: try: - resp = requests.get(f"{sub['url']}/api/sites", timeout=5) - if resp.status_code != 200: - continue - data = resp.json() + data = fetch_remote_sites(sub["dest_hash"]) sites = data.get("sites", []) remote_name = data.get("name", sub["name"]) db = get_db() - local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub["id"],)) for s in sites: - if s["url"] in local_urls: - continue try: db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", - (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + "INSERT INTO remote_pages (subscription_id, url, title, note) VALUES (?, ?, ?, ?)", + (sub["id"], s["url"], s["title"], s.get("note", "")), ) except Exception: pass diff --git a/rns_client.py b/rns_client.py new file mode 100644 index 0000000..32eeadc --- /dev/null +++ b/rns_client.py @@ -0,0 +1,78 @@ +import time +import RNS + +APP_NAME = "tinyweb" +ASPECTS = ["server"] +REQUEST_TIMEOUT = 30 + + +def fetch_remote_sites(dest_hash_hex): + """ + Connect to a remote TinyWeb instance over Reticulum and fetch its + shared sites. Returns the response dict from /api/sites, or raises + an exception on failure. + """ + dest_hash = bytes.fromhex(dest_hash_hex) + + # Resolve path if needed + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + raise ConnectionError(f"Could not find path to {dest_hash_hex}") + + server_identity = RNS.Identity.recall(dest_hash) + if server_identity is None: + raise ConnectionError(f"Could not recall identity for {dest_hash_hex}") + + destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, + ) + + # Establish link + link = RNS.Link(destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + + if link.status != RNS.Link.ACTIVE: + raise ConnectionError(f"Could not establish link to {dest_hash_hex}") + + try: + # Request /api/sites + request_data = { + "method": "GET", + "path": "/api/sites", + "query": {}, + "body": {}, + "gateway_host": "", + } + + receipt = link.request("/tinyweb", data=request_data, timeout=REQUEST_TIMEOUT) + + elapsed = 0 + done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT: + time.sleep(0.5) + elapsed += 0.5 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + if resp["status"] == 403: + raise PermissionError("That instance has sharing disabled.") + if resp["status"] != 200: + raise ConnectionError(f"Remote returned status {resp['status']}") + import json + return json.loads(resp["body"]) + else: + raise ConnectionError(f"Request failed or timed out") + finally: + link.teardown() From 4c203bbf2dc35770141ef3c9748f633397728284 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 23:01:54 -0700 Subject: [PATCH 009/194] single-command startup app.py now auto-starts the gateway HTTP server in a daemon thread, so users only need `python app.py` to get everything running. The gateway calls dispatch_request directly when co-located (local mode) instead of trying to establish an RNS link to itself. Bookmarklet hardcoded to localhost:8080. gateway.py still works standalone for connecting to remote instances. --- app.py | 18 +++++++++++++--- gateway.py | 61 +++++++++++++++++++++++++++++------------------------ handlers.py | 7 +++--- 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/app.py b/app.py index 890bf0a..9233911 100644 --- a/app.py +++ b/app.py @@ -1,9 +1,12 @@ import os import time +import threading import RNS +from http.server import HTTPServer from db import init_db from handlers import dispatch_request +from gateway import GatewayState, GatewayHandler, GATEWAY_PORT APP_NAME = "tinyweb" ASPECTS = ["server"] @@ -24,6 +27,14 @@ def rns_request_handler(path, data, request_id, link_id, remote_identity, reques return dispatch_request(data) +def start_gateway(reticulum): + GatewayState.reticulum = reticulum + GatewayState.local_dispatch = dispatch_request + server = HTTPServer(("127.0.0.1", GATEWAY_PORT), GatewayHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + def main(): init_db() reticulum = RNS.Reticulum() @@ -44,10 +55,11 @@ def main(): ) destination.announce() + start_gateway(reticulum) - print(f"TinyWeb Reticulum server running") - print(f"Destination hash: {RNS.prettyhexrep(destination.hash)}") - print(f"Share this hash with clients to connect via gateway.py") + print(f"TinyWeb running!") + print(f"Open http://localhost:{GATEWAY_PORT} in your browser") + print(f"Destination hash: {RNS.prettyhexrep(destination.hash)} (share this so friends can subscribe)") while True: time.sleep(1) diff --git a/gateway.py b/gateway.py index 0bde2d3..78516b3 100644 --- a/gateway.py +++ b/gateway.py @@ -16,6 +16,7 @@ class GatewayState: destination = None link = None link_lock = threading.Lock() + local_dispatch = None # set when running inside app.py def resolve_destination(dest_hash_hex): @@ -83,34 +84,40 @@ class GatewayHandler(BaseHTTPRequestHandler): } try: - link = ensure_link() - receipt = link.request( - "/tinyweb", - data=request_data, - timeout=REQUEST_TIMEOUT, - ) - - # Wait for the response - elapsed = 0 - done_statuses = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) - while receipt.get_status() not in done_statuses and elapsed < REQUEST_TIMEOUT: - time.sleep(0.1) - elapsed += 0.1 - - if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): - resp = receipt.get_response() - self.send_response(resp["status"]) - self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8")) - for k, v in resp.get("headers", {}).items(): - self.send_header(k, v) - self.end_headers() - resp_body = resp.get("body", "") - if resp_body: - self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body) - elif receipt.get_status() == RNS.RequestReceipt.FAILED: - self.send_error(504, "Request to TinyWeb server failed") + if GatewayState.local_dispatch: + resp = GatewayState.local_dispatch(request_data) else: - self.send_error(504, "Request to TinyWeb server timed out") + link = ensure_link() + receipt = link.request( + "/tinyweb", + data=request_data, + timeout=REQUEST_TIMEOUT, + ) + + # Wait for the response + elapsed = 0 + done_statuses = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done_statuses and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + elif receipt.get_status() == RNS.RequestReceipt.FAILED: + self.send_error(504, "Request to TinyWeb server failed") + return + else: + self.send_error(504, "Request to TinyWeb server timed out") + return + + self.send_response(resp["status"]) + self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8")) + for k, v in resp.get("headers", {}).items(): + self.send_header(k, v) + self.end_headers() + resp_body = resp.get("body", "") + if resp_body: + self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body) except ConnectionError as e: GatewayState.link = None diff --git a/handlers.py b/handlers.py index ec5f6dd..4550e15 100644 --- a/handlers.py +++ b/handlers.py @@ -308,12 +308,11 @@ def handle_import_submit(body): return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") -def handle_style_form(msg="", gateway_host=""): +def handle_style_form(msg=""): css = get_setting("custom_css") name = get_site_name() sharing = get_setting("sharing_enabled", "0") checked = " checked" if sharing == "1" else "" - host = gateway_host or "localhost:8080" return _respond( f"

    customize

    " f"

    name your search engine

    " @@ -340,7 +339,7 @@ def handle_style_form(msg="", gateway_host=""): f"" f"

    bookmarklet

    " f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " - f'

    + save to {esc(name)}

    ' + f'

    + save to {esc(name)}

    ' f"

    {msg}

    " f'back' ) @@ -651,7 +650,7 @@ def dispatch_request(data): elif path == "/bookmark": return handle_bookmark(query) elif path == "/style": - return handle_style_form(gateway_host=gateway_host) + return handle_style_form() elif path == "/export": return handle_export() elif path == "/import": From e545203a875bbfcf7c0c688c8832e7762be55044 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 23:01:54 -0700 Subject: [PATCH 010/194] single-command startup app.py now auto-starts the gateway HTTP server in a daemon thread, so users only need `python app.py` to get everything running. The gateway calls dispatch_request directly when co-located (local mode) instead of trying to establish an RNS link to itself. Bookmarklet hardcoded to localhost:8080. gateway.py still works standalone for connecting to remote instances. --- app.py | 18 +++++++++++++--- gateway.py | 61 +++++++++++++++++++++++++++++------------------------ handlers.py | 7 +++--- 3 files changed, 52 insertions(+), 34 deletions(-) diff --git a/app.py b/app.py index 890bf0a..9233911 100644 --- a/app.py +++ b/app.py @@ -1,9 +1,12 @@ import os import time +import threading import RNS +from http.server import HTTPServer from db import init_db from handlers import dispatch_request +from gateway import GatewayState, GatewayHandler, GATEWAY_PORT APP_NAME = "tinyweb" ASPECTS = ["server"] @@ -24,6 +27,14 @@ def rns_request_handler(path, data, request_id, link_id, remote_identity, reques return dispatch_request(data) +def start_gateway(reticulum): + GatewayState.reticulum = reticulum + GatewayState.local_dispatch = dispatch_request + server = HTTPServer(("127.0.0.1", GATEWAY_PORT), GatewayHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + def main(): init_db() reticulum = RNS.Reticulum() @@ -44,10 +55,11 @@ def main(): ) destination.announce() + start_gateway(reticulum) - print(f"TinyWeb Reticulum server running") - print(f"Destination hash: {RNS.prettyhexrep(destination.hash)}") - print(f"Share this hash with clients to connect via gateway.py") + print(f"TinyWeb running!") + print(f"Open http://localhost:{GATEWAY_PORT} in your browser") + print(f"Destination hash: {RNS.prettyhexrep(destination.hash)} (share this so friends can subscribe)") while True: time.sleep(1) diff --git a/gateway.py b/gateway.py index 0bde2d3..78516b3 100644 --- a/gateway.py +++ b/gateway.py @@ -16,6 +16,7 @@ class GatewayState: destination = None link = None link_lock = threading.Lock() + local_dispatch = None # set when running inside app.py def resolve_destination(dest_hash_hex): @@ -83,34 +84,40 @@ class GatewayHandler(BaseHTTPRequestHandler): } try: - link = ensure_link() - receipt = link.request( - "/tinyweb", - data=request_data, - timeout=REQUEST_TIMEOUT, - ) - - # Wait for the response - elapsed = 0 - done_statuses = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) - while receipt.get_status() not in done_statuses and elapsed < REQUEST_TIMEOUT: - time.sleep(0.1) - elapsed += 0.1 - - if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): - resp = receipt.get_response() - self.send_response(resp["status"]) - self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8")) - for k, v in resp.get("headers", {}).items(): - self.send_header(k, v) - self.end_headers() - resp_body = resp.get("body", "") - if resp_body: - self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body) - elif receipt.get_status() == RNS.RequestReceipt.FAILED: - self.send_error(504, "Request to TinyWeb server failed") + if GatewayState.local_dispatch: + resp = GatewayState.local_dispatch(request_data) else: - self.send_error(504, "Request to TinyWeb server timed out") + link = ensure_link() + receipt = link.request( + "/tinyweb", + data=request_data, + timeout=REQUEST_TIMEOUT, + ) + + # Wait for the response + elapsed = 0 + done_statuses = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done_statuses and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + elif receipt.get_status() == RNS.RequestReceipt.FAILED: + self.send_error(504, "Request to TinyWeb server failed") + return + else: + self.send_error(504, "Request to TinyWeb server timed out") + return + + self.send_response(resp["status"]) + self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8")) + for k, v in resp.get("headers", {}).items(): + self.send_header(k, v) + self.end_headers() + resp_body = resp.get("body", "") + if resp_body: + self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body) except ConnectionError as e: GatewayState.link = None diff --git a/handlers.py b/handlers.py index ec5f6dd..4550e15 100644 --- a/handlers.py +++ b/handlers.py @@ -308,12 +308,11 @@ def handle_import_submit(body): return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") -def handle_style_form(msg="", gateway_host=""): +def handle_style_form(msg=""): css = get_setting("custom_css") name = get_site_name() sharing = get_setting("sharing_enabled", "0") checked = " checked" if sharing == "1" else "" - host = gateway_host or "localhost:8080" return _respond( f"

    customize

    " f"

    name your search engine

    " @@ -340,7 +339,7 @@ def handle_style_form(msg="", gateway_host=""): f"" f"

    bookmarklet

    " f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " - f'

    + save to {esc(name)}

    ' + f'

    + save to {esc(name)}

    ' f"

    {msg}

    " f'back' ) @@ -651,7 +650,7 @@ def dispatch_request(data): elif path == "/bookmark": return handle_bookmark(query) elif path == "/style": - return handle_style_form(gateway_host=gateway_host) + return handle_style_form() elif path == "/export": return handle_export() elif path == "/import": From e9a6e0108e069597d99d21bdd72757607532894d Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 23:15:28 -0700 Subject: [PATCH 011/194] stripped tracking params, added tags URLs are cleaned of tracking parameters (utm_*, fbclid, gclid, etc.) before indexing. Tags can be added when saving or editing pages, browsed at /tags, and are included in search results. Tags are shared via /api/sites and preserved when syncing/importing from subscriptions. --- db.py | 39 +++++++++++- handlers.py | 180 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 192 insertions(+), 27 deletions(-) diff --git a/db.py b/db.py index 5a86a1c..b523c79 100644 --- a/db.py +++ b/db.py @@ -1,12 +1,26 @@ import sqlite3 import requests -from urllib.parse import urlparse, urljoin +from urllib.parse import urlparse, urljoin, parse_qs, urlencode, urlunparse from bs4 import BeautifulSoup DATABASE = "index.db" SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") +TRACKING_PARAMS = { + "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", + "fbclid", "gclid", "msclkid", "mc_cid", "mc_eid", "ref", "ref_src", + "ref_url", "_ga", "_gl", "yclid", "twclid", "igshid", +} + + +def clean_url(url): + parsed = urlparse(url) + params = parse_qs(parsed.query) + cleaned = {k: v for k, v in params.items() if k.lower() not in TRACKING_PARAMS} + new_query = urlencode(cleaned, doseq=True) + return urlunparse(parsed._replace(query=new_query)) + def get_db(): db = sqlite3.connect(DATABASE) @@ -60,6 +74,7 @@ def init_db(): " url TEXT NOT NULL," " title TEXT," " note TEXT DEFAULT ''," + " tags TEXT DEFAULT ''," " FOREIGN KEY (subscription_id) REFERENCES subscriptions(id) ON DELETE CASCADE," " UNIQUE(subscription_id, url)" ")" @@ -68,6 +83,21 @@ def init_db(): "CREATE VIRTUAL TABLE IF NOT EXISTS remote_pages_fts " "USING fts5(title, url, note, content=remote_pages, content_rowid=id)" ) + db.execute( + "CREATE TABLE IF NOT EXISTS tags (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " name TEXT UNIQUE NOT NULL" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS page_tags (" + " page_id INTEGER NOT NULL," + " tag_id INTEGER NOT NULL," + " PRIMARY KEY (page_id, tag_id)," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE," + " FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE" + ")" + ) db.executescript(""" CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN INSERT INTO pages_fts(rowid, title, body, url, note) @@ -104,6 +134,12 @@ def init_db(): db.execute("ALTER TABLE subscriptions RENAME COLUMN url TO dest_hash") db.commit() + # Migrate remote_pages: add tags column if missing + rp_cols = [row[1] for row in db.execute("PRAGMA table_info(remote_pages)").fetchall()] + if "tags" not in rp_cols: + db.execute("ALTER TABLE remote_pages ADD COLUMN tags TEXT DEFAULT ''") + db.commit() + db.commit() db.close() @@ -165,6 +201,7 @@ def fetch_page(url): def index_url(url, note=""): + url = clean_url(url) title, body, links = fetch_page(url) db = get_db() cur = db.execute( diff --git a/handlers.py b/handlers.py index 4550e15..42944a5 100644 --- a/handlers.py +++ b/handlers.py @@ -1,7 +1,7 @@ import json from datetime import datetime -from db import get_db, get_setting, set_setting, get_site_name, index_url +from db import get_db, get_setting, set_setting, get_site_name, index_url, clean_url from templates import esc, snippet, wrap_page from rns_client import fetch_remote_sites @@ -46,6 +46,38 @@ def _error(status): return _respond(f"

    {status}

    ", status) +# --- Tag helpers --- + + +def _get_page_tags(page_id, db=None): + close = False + if db is None: + db = get_db() + close = True + rows = db.execute( + "SELECT t.name FROM tags t JOIN page_tags pt ON t.id = pt.tag_id " + "WHERE pt.page_id = ? ORDER BY t.name", (page_id,) + ).fetchall() + if close: + db.close() + return [r["name"] for r in rows] + + +def _set_page_tags(page_id, tag_string, db=None): + close = False + if db is None: + db = get_db() + close = True + db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,)) + for name in (t.strip().lower() for t in tag_string.split(",") if t.strip()): + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,)) + tag_id = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()["id"] + db.execute("INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", (page_id, tag_id)) + if close: + db.commit() + db.close() + + # --- Route handlers --- @@ -69,12 +101,17 @@ def handle_search(query): note_html = "" if r["note"]: note_html = f'
    {esc(r["note"])}
    ' + tags = _get_page_tags(r["id"], db) + tags_html = "" + if tags: + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + tags_html = f'
    {tag_links}
    ' result_html += ( f'
    ' f'{esc(r["title"])}
    ' f'{esc(r["url"])}
    ' f'{esc(snippet(r["body"], q))}' - f'{note_html}' + f'{note_html}{tags_html}' f'
    ' ) else: @@ -157,6 +194,7 @@ def handle_search(query): f'

    {count} page(s) indexed.' f' + add url' f' | browse' + f' | tags' f' | subscriptions' f' | customize

    ' f'
    {result_html}{trusted_html}{remote_html}' @@ -169,6 +207,7 @@ def handle_add_form(msg=""): f'
    ' f'

    ' f'

    ' + f'

    ' f'' f"
    " f"

    {msg}

    " @@ -177,14 +216,22 @@ def handle_add_form(msg=""): def handle_add_submit(body): - url = body.get("url", [""])[0].strip() + url = clean_url(body.get("url", [""])[0].strip()) note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() if not url: return handle_add_form("URL is required.") if not url.startswith(("http://", "https://")): return handle_add_form("URL must start with http:// or https://") try: title = index_url(url, note) + if tags: + db = get_db() + row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() + if row: + _set_page_tags(row["id"], tags, db) + db.commit() + db.close() return handle_add_form(f'Indexed: {esc(title)}') except Exception as e: return handle_add_form(f"Error: {esc(str(e))}") @@ -193,16 +240,21 @@ def handle_add_submit(body): def handle_pages(): db = get_db() rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() - db.close() items = "" for r in rows: note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tags_html = "" + if tags: + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + tags_html = f' {tag_links}' items += ( - f'
  • {esc(r["title"])}{note_html} ' + f'
  • {esc(r["title"])}{note_html}{tags_html} ' f'({esc(r["url"])}) ' f'edit ' f'remove
    • ' ) + db.close() return _respond( f"

    indexed pages ({len(rows)})

    " f"
      {items}
    " @@ -214,15 +266,18 @@ def handle_pages(): def handle_edit_form(page_id, msg=""): db = get_db() row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() - db.close() if not row: + db.close() return _error(404) + tags = ", ".join(_get_page_tags(page_id, db)) + db.close() return _respond( - f"

    edit note

    " + f"

    edit page

    " f"

    {esc(row['title'])}
    " f"{esc(row['url'])}

    " f'
    ' f'

    ' + f'

    ' f'' f"
    " f"

    {msg}

    " @@ -232,8 +287,10 @@ def handle_edit_form(page_id, msg=""): def handle_edit_submit(page_id, body): note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() db = get_db() db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) + _set_page_tags(page_id, tags, db) db.commit() db.close() return _redirect("/pages") @@ -249,7 +306,7 @@ def handle_delete(page_id): def handle_bookmark(query): - url = query.get("url", [""])[0].strip() + url = clean_url(query.get("url", [""])[0].strip()) if not url or not url.startswith(("http://", "https://")): return _text_response("error: invalid url", headers={"Access-Control-Allow-Origin": "*"}) try: @@ -355,6 +412,51 @@ def handle_style_submit(body): return handle_style_form("Saved.") +def handle_tags(): + db = get_db() + rows = db.execute( + "SELECT t.name, COUNT(pt.page_id) AS cnt FROM tags t " + "JOIN page_tags pt ON t.id = pt.tag_id " + "GROUP BY t.id ORDER BY t.name" + ).fetchall() + db.close() + items = "" + for r in rows: + items += f'
  • {esc(r["name"])} ({r["cnt"]})
    • ' + return _respond( + f"

    tags

    " + f"
      {items}
    " if items else "

    No tags yet. Add tags when saving or editing pages.

    " + f'back' + ) + + +def handle_tag_browse(tag_name): + db = get_db() + rows = db.execute( + "SELECT p.id, p.url, p.title, p.note FROM pages p " + "JOIN page_tags pt ON p.id = pt.page_id " + "JOIN tags t ON t.id = pt.tag_id " + "WHERE t.name = ? ORDER BY p.id DESC", + (tag_name,), + ).fetchall() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + items += ( + f'
  • {esc(r["title"])}{note_html} {tag_links} ' + f'({esc(r["url"])})
    • ' + ) + db.close() + return _respond( + f'

    tag: {esc(tag_name)}

    ' + f'

    {len(rows)} page(s)

    ' + f'
      {items}
    ' + f'all tags | back' + ) + + def handle_api_sites(): if get_setting("sharing_enabled", "0") != "1": return _json_response( @@ -363,12 +465,13 @@ def handle_api_sites(): headers={"Access-Control-Allow-Origin": "*"}, ) db = get_db() - rows = db.execute("SELECT url, title, note FROM pages ORDER BY id DESC").fetchall() + rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() + sites = [] + for r in rows: + tags = _get_page_tags(r["id"], db) + sites.append({"url": r["url"], "title": r["title"], "note": r["note"], "tags": tags}) db.close() - data = { - "name": get_site_name(), - "sites": [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows], - } + data = {"name": get_site_name(), "sites": sites} return _json_response(data, headers={"Access-Control-Allow-Origin": "*"}) @@ -455,13 +558,16 @@ def handle_subscription_browse(sub_id): # Use locally synced data if available, otherwise fetch live remote_rows = db.execute( - "SELECT url, title, note FROM remote_pages WHERE subscription_id = ?", + "SELECT url, title, note, tags FROM remote_pages WHERE subscription_id = ?", (sub_id,), ).fetchall() db.close() if remote_rows: - sites = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in remote_rows] + sites = [] + for r in remote_rows: + tags = [t for t in r["tags"].split(",") if t] if r["tags"] else [] + sites.append({"url": r["url"], "title": r["title"], "note": r["note"], "tags": tags}) else: try: data = fetch_remote_sites(sub["dest_hash"]) @@ -483,9 +589,12 @@ def handle_subscription_browse(sub_id): else: new_count += 1 note_html = f' — {esc(s["note"])}' if s.get("note") else "" + tags_html = "" + if s.get("tags"): + tags_html = " " + " ".join(f'[{esc(t)}]' for t in s["tags"]) new_items += ( f'
    • ' ) @@ -509,16 +618,19 @@ def handle_subscription_pick(body): sub_id = body.get("sub_id", [""])[0] import_all = body.get("import_all", [""])[0] + # Build a url->tags map from remote_pages for this subscription + db = get_db() + remote_rows = db.execute( + "SELECT url, tags FROM remote_pages WHERE subscription_id = ?", (sub_id,) + ).fetchall() + remote_tags = {r["url"]: r["tags"] for r in remote_rows} + if import_all: - db = get_db() local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) - remote = db.execute( - "SELECT url FROM remote_pages WHERE subscription_id = ?", (sub_id,) - ).fetchall() - db.close() - urls = [r["url"] for r in remote if r["url"] not in local_urls] + urls = [r["url"] for r in remote_rows if r["url"] not in local_urls] else: urls = body.get("urls", []) + db.close() if not urls: return handle_subscriptions("No sites selected.") @@ -528,6 +640,15 @@ def handle_subscription_pick(body): for url in urls: try: index_url(url) + # Import tags from the remote page + tags_str = remote_tags.get(url, "") + if tags_str: + db = get_db() + row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() + if row: + _set_page_tags(row["id"], tags_str, db) + db.commit() + db.close() imported += 1 except Exception: errors += 1 @@ -556,9 +677,10 @@ def handle_subscription_sync(sub_id): synced = 0 for s in sites: try: + tags_str = ",".join(s.get("tags", [])) db.execute( - "INSERT INTO remote_pages (subscription_id, url, title, note) VALUES (?, ?, ?, ?)", - (sub_id, s["url"], s["title"], s.get("note", "")), + "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?)", + (sub_id, s["url"], s["title"], s.get("note", ""), tags_str), ) synced += 1 except Exception: @@ -602,9 +724,10 @@ def handle_subscription_syncall(): db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub["id"],)) for s in sites: try: + tags_str = ",".join(s.get("tags", [])) db.execute( - "INSERT INTO remote_pages (subscription_id, url, title, note) VALUES (?, ?, ?, ?)", - (sub["id"], s["url"], s["title"], s.get("note", "")), + "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?)", + (sub["id"], s["url"], s["title"], s.get("note", ""), tags_str), ) except Exception: pass @@ -655,6 +778,11 @@ def dispatch_request(data): return handle_export() elif path == "/import": return handle_import_form() + elif path == "/tags": + return handle_tags() + elif path.startswith("/tags/"): + tag_name = path[len("/tags/"):] + return handle_tag_browse(tag_name) if tag_name else _error(400) elif path == "/api/sites": return handle_api_sites() elif path == "/subscriptions": From 99a4e84ee1da60811e19f75527fbec7a92bba68e Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 23:15:28 -0700 Subject: [PATCH 012/194] stripped tracking params, added tags URLs are cleaned of tracking parameters (utm_*, fbclid, gclid, etc.) before indexing. Tags can be added when saving or editing pages, browsed at /tags, and are included in search results. Tags are shared via /api/sites and preserved when syncing/importing from subscriptions. --- db.py | 39 +++++++++++- handlers.py | 180 ++++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 192 insertions(+), 27 deletions(-) diff --git a/db.py b/db.py index 5a86a1c..b523c79 100644 --- a/db.py +++ b/db.py @@ -1,12 +1,26 @@ import sqlite3 import requests -from urllib.parse import urlparse, urljoin +from urllib.parse import urlparse, urljoin, parse_qs, urlencode, urlunparse from bs4 import BeautifulSoup DATABASE = "index.db" SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") +TRACKING_PARAMS = { + "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", + "fbclid", "gclid", "msclkid", "mc_cid", "mc_eid", "ref", "ref_src", + "ref_url", "_ga", "_gl", "yclid", "twclid", "igshid", +} + + +def clean_url(url): + parsed = urlparse(url) + params = parse_qs(parsed.query) + cleaned = {k: v for k, v in params.items() if k.lower() not in TRACKING_PARAMS} + new_query = urlencode(cleaned, doseq=True) + return urlunparse(parsed._replace(query=new_query)) + def get_db(): db = sqlite3.connect(DATABASE) @@ -60,6 +74,7 @@ def init_db(): " url TEXT NOT NULL," " title TEXT," " note TEXT DEFAULT ''," + " tags TEXT DEFAULT ''," " FOREIGN KEY (subscription_id) REFERENCES subscriptions(id) ON DELETE CASCADE," " UNIQUE(subscription_id, url)" ")" @@ -68,6 +83,21 @@ def init_db(): "CREATE VIRTUAL TABLE IF NOT EXISTS remote_pages_fts " "USING fts5(title, url, note, content=remote_pages, content_rowid=id)" ) + db.execute( + "CREATE TABLE IF NOT EXISTS tags (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " name TEXT UNIQUE NOT NULL" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS page_tags (" + " page_id INTEGER NOT NULL," + " tag_id INTEGER NOT NULL," + " PRIMARY KEY (page_id, tag_id)," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE," + " FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE" + ")" + ) db.executescript(""" CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN INSERT INTO pages_fts(rowid, title, body, url, note) @@ -104,6 +134,12 @@ def init_db(): db.execute("ALTER TABLE subscriptions RENAME COLUMN url TO dest_hash") db.commit() + # Migrate remote_pages: add tags column if missing + rp_cols = [row[1] for row in db.execute("PRAGMA table_info(remote_pages)").fetchall()] + if "tags" not in rp_cols: + db.execute("ALTER TABLE remote_pages ADD COLUMN tags TEXT DEFAULT ''") + db.commit() + db.commit() db.close() @@ -165,6 +201,7 @@ def fetch_page(url): def index_url(url, note=""): + url = clean_url(url) title, body, links = fetch_page(url) db = get_db() cur = db.execute( diff --git a/handlers.py b/handlers.py index 4550e15..42944a5 100644 --- a/handlers.py +++ b/handlers.py @@ -1,7 +1,7 @@ import json from datetime import datetime -from db import get_db, get_setting, set_setting, get_site_name, index_url +from db import get_db, get_setting, set_setting, get_site_name, index_url, clean_url from templates import esc, snippet, wrap_page from rns_client import fetch_remote_sites @@ -46,6 +46,38 @@ def _error(status): return _respond(f"

    {status}

    ", status) +# --- Tag helpers --- + + +def _get_page_tags(page_id, db=None): + close = False + if db is None: + db = get_db() + close = True + rows = db.execute( + "SELECT t.name FROM tags t JOIN page_tags pt ON t.id = pt.tag_id " + "WHERE pt.page_id = ? ORDER BY t.name", (page_id,) + ).fetchall() + if close: + db.close() + return [r["name"] for r in rows] + + +def _set_page_tags(page_id, tag_string, db=None): + close = False + if db is None: + db = get_db() + close = True + db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,)) + for name in (t.strip().lower() for t in tag_string.split(",") if t.strip()): + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,)) + tag_id = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()["id"] + db.execute("INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", (page_id, tag_id)) + if close: + db.commit() + db.close() + + # --- Route handlers --- @@ -69,12 +101,17 @@ def handle_search(query): note_html = "" if r["note"]: note_html = f'
    {esc(r["note"])}
    ' + tags = _get_page_tags(r["id"], db) + tags_html = "" + if tags: + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + tags_html = f'
    {tag_links}
    ' result_html += ( f'
    ' f'{esc(r["title"])}
    ' f'{esc(r["url"])}
    ' f'{esc(snippet(r["body"], q))}' - f'{note_html}' + f'{note_html}{tags_html}' f'
    ' ) else: @@ -157,6 +194,7 @@ def handle_search(query): f'

    {count} page(s) indexed.' f' + add url' f' | browse' + f' | tags' f' | subscriptions' f' | customize

    ' f'
    {result_html}{trusted_html}{remote_html}' @@ -169,6 +207,7 @@ def handle_add_form(msg=""): f'
    ' f'

    ' f'

    ' + f'

    ' f'' f"
    " f"

    {msg}

    " @@ -177,14 +216,22 @@ def handle_add_form(msg=""): def handle_add_submit(body): - url = body.get("url", [""])[0].strip() + url = clean_url(body.get("url", [""])[0].strip()) note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() if not url: return handle_add_form("URL is required.") if not url.startswith(("http://", "https://")): return handle_add_form("URL must start with http:// or https://") try: title = index_url(url, note) + if tags: + db = get_db() + row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() + if row: + _set_page_tags(row["id"], tags, db) + db.commit() + db.close() return handle_add_form(f'Indexed: {esc(title)}') except Exception as e: return handle_add_form(f"Error: {esc(str(e))}") @@ -193,16 +240,21 @@ def handle_add_submit(body): def handle_pages(): db = get_db() rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() - db.close() items = "" for r in rows: note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tags_html = "" + if tags: + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + tags_html = f' {tag_links}' items += ( - f'
  • {esc(r["title"])}{note_html} ' + f'
  • {esc(r["title"])}{note_html}{tags_html} ' f'({esc(r["url"])}) ' f'edit ' f'remove
    • ' ) + db.close() return _respond( f"

    indexed pages ({len(rows)})

    " f"
      {items}
    " @@ -214,15 +266,18 @@ def handle_pages(): def handle_edit_form(page_id, msg=""): db = get_db() row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() - db.close() if not row: + db.close() return _error(404) + tags = ", ".join(_get_page_tags(page_id, db)) + db.close() return _respond( - f"

    edit note

    " + f"

    edit page

    " f"

    {esc(row['title'])}
    " f"{esc(row['url'])}

    " f'
    ' f'

    ' + f'

    ' f'' f"
    " f"

    {msg}

    " @@ -232,8 +287,10 @@ def handle_edit_form(page_id, msg=""): def handle_edit_submit(page_id, body): note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() db = get_db() db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) + _set_page_tags(page_id, tags, db) db.commit() db.close() return _redirect("/pages") @@ -249,7 +306,7 @@ def handle_delete(page_id): def handle_bookmark(query): - url = query.get("url", [""])[0].strip() + url = clean_url(query.get("url", [""])[0].strip()) if not url or not url.startswith(("http://", "https://")): return _text_response("error: invalid url", headers={"Access-Control-Allow-Origin": "*"}) try: @@ -355,6 +412,51 @@ def handle_style_submit(body): return handle_style_form("Saved.") +def handle_tags(): + db = get_db() + rows = db.execute( + "SELECT t.name, COUNT(pt.page_id) AS cnt FROM tags t " + "JOIN page_tags pt ON t.id = pt.tag_id " + "GROUP BY t.id ORDER BY t.name" + ).fetchall() + db.close() + items = "" + for r in rows: + items += f'
  • {esc(r["name"])} ({r["cnt"]})
    • ' + return _respond( + f"

    tags

    " + f"
      {items}
    " if items else "

    No tags yet. Add tags when saving or editing pages.

    " + f'back' + ) + + +def handle_tag_browse(tag_name): + db = get_db() + rows = db.execute( + "SELECT p.id, p.url, p.title, p.note FROM pages p " + "JOIN page_tags pt ON p.id = pt.page_id " + "JOIN tags t ON t.id = pt.tag_id " + "WHERE t.name = ? ORDER BY p.id DESC", + (tag_name,), + ).fetchall() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + items += ( + f'
  • {esc(r["title"])}{note_html} {tag_links} ' + f'({esc(r["url"])})
    • ' + ) + db.close() + return _respond( + f'

    tag: {esc(tag_name)}

    ' + f'

    {len(rows)} page(s)

    ' + f'
      {items}
    ' + f'all tags | back' + ) + + def handle_api_sites(): if get_setting("sharing_enabled", "0") != "1": return _json_response( @@ -363,12 +465,13 @@ def handle_api_sites(): headers={"Access-Control-Allow-Origin": "*"}, ) db = get_db() - rows = db.execute("SELECT url, title, note FROM pages ORDER BY id DESC").fetchall() + rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() + sites = [] + for r in rows: + tags = _get_page_tags(r["id"], db) + sites.append({"url": r["url"], "title": r["title"], "note": r["note"], "tags": tags}) db.close() - data = { - "name": get_site_name(), - "sites": [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows], - } + data = {"name": get_site_name(), "sites": sites} return _json_response(data, headers={"Access-Control-Allow-Origin": "*"}) @@ -455,13 +558,16 @@ def handle_subscription_browse(sub_id): # Use locally synced data if available, otherwise fetch live remote_rows = db.execute( - "SELECT url, title, note FROM remote_pages WHERE subscription_id = ?", + "SELECT url, title, note, tags FROM remote_pages WHERE subscription_id = ?", (sub_id,), ).fetchall() db.close() if remote_rows: - sites = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in remote_rows] + sites = [] + for r in remote_rows: + tags = [t for t in r["tags"].split(",") if t] if r["tags"] else [] + sites.append({"url": r["url"], "title": r["title"], "note": r["note"], "tags": tags}) else: try: data = fetch_remote_sites(sub["dest_hash"]) @@ -483,9 +589,12 @@ def handle_subscription_browse(sub_id): else: new_count += 1 note_html = f' — {esc(s["note"])}' if s.get("note") else "" + tags_html = "" + if s.get("tags"): + tags_html = " " + " ".join(f'[{esc(t)}]' for t in s["tags"]) new_items += ( f'
    • ' ) @@ -509,16 +618,19 @@ def handle_subscription_pick(body): sub_id = body.get("sub_id", [""])[0] import_all = body.get("import_all", [""])[0] + # Build a url->tags map from remote_pages for this subscription + db = get_db() + remote_rows = db.execute( + "SELECT url, tags FROM remote_pages WHERE subscription_id = ?", (sub_id,) + ).fetchall() + remote_tags = {r["url"]: r["tags"] for r in remote_rows} + if import_all: - db = get_db() local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) - remote = db.execute( - "SELECT url FROM remote_pages WHERE subscription_id = ?", (sub_id,) - ).fetchall() - db.close() - urls = [r["url"] for r in remote if r["url"] not in local_urls] + urls = [r["url"] for r in remote_rows if r["url"] not in local_urls] else: urls = body.get("urls", []) + db.close() if not urls: return handle_subscriptions("No sites selected.") @@ -528,6 +640,15 @@ def handle_subscription_pick(body): for url in urls: try: index_url(url) + # Import tags from the remote page + tags_str = remote_tags.get(url, "") + if tags_str: + db = get_db() + row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() + if row: + _set_page_tags(row["id"], tags_str, db) + db.commit() + db.close() imported += 1 except Exception: errors += 1 @@ -556,9 +677,10 @@ def handle_subscription_sync(sub_id): synced = 0 for s in sites: try: + tags_str = ",".join(s.get("tags", [])) db.execute( - "INSERT INTO remote_pages (subscription_id, url, title, note) VALUES (?, ?, ?, ?)", - (sub_id, s["url"], s["title"], s.get("note", "")), + "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?)", + (sub_id, s["url"], s["title"], s.get("note", ""), tags_str), ) synced += 1 except Exception: @@ -602,9 +724,10 @@ def handle_subscription_syncall(): db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub["id"],)) for s in sites: try: + tags_str = ",".join(s.get("tags", [])) db.execute( - "INSERT INTO remote_pages (subscription_id, url, title, note) VALUES (?, ?, ?, ?)", - (sub["id"], s["url"], s["title"], s.get("note", "")), + "INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?)", + (sub["id"], s["url"], s["title"], s.get("note", ""), tags_str), ) except Exception: pass @@ -655,6 +778,11 @@ def dispatch_request(data): return handle_export() elif path == "/import": return handle_import_form() + elif path == "/tags": + return handle_tags() + elif path.startswith("/tags/"): + tag_name = path[len("/tags/"):] + return handle_tag_browse(tag_name) if tag_name else _error(400) elif path == "/api/sites": return handle_api_sites() elif path == "/subscriptions": From db60bfa70146ceaf0afc89026487e17dcf026764 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 23:21:02 -0700 Subject: [PATCH 013/194] added an about page with slow-web pitch Shows instance stats, destination hash for subscribing, and explains the slow web movement and how TinyWeb works. Destination hash is stored in settings on startup so the about page can display it. --- app.py | 3 ++- handlers.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 9233911..7d17007 100644 --- a/app.py +++ b/app.py @@ -4,7 +4,7 @@ import threading import RNS from http.server import HTTPServer -from db import init_db +from db import init_db, set_setting from handlers import dispatch_request from gateway import GatewayState, GatewayHandler, GATEWAY_PORT @@ -55,6 +55,7 @@ def main(): ) destination.announce() + set_setting("dest_hash", destination.hash.hex()) start_gateway(reticulum) print(f"TinyWeb running!") diff --git a/handlers.py b/handlers.py index 42944a5..2c0ec07 100644 --- a/handlers.py +++ b/handlers.py @@ -196,7 +196,8 @@ def handle_search(query): f' | browse' f' | tags' f' | subscriptions' - f' | customize

    ' + f' | customize' + f' | about

    ' f'
    {result_html}{trusted_html}{remote_html}' ) @@ -412,6 +413,58 @@ def handle_style_submit(body): return handle_style_form("Saved.") +def handle_about(): + name = get_site_name() + dest_hash = get_setting("dest_hash") + sharing = get_setting("sharing_enabled", "0") == "1" + db = get_db() + page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + tag_count = db.execute("SELECT count(*) FROM tags").fetchone()[0] + sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0] + db.close() + + sharing_html = ( + '

    This instance shares its index publicly. Subscribe to join the network.

    ' + if sharing else + '

    This instance is private.

    ' + ) + + hash_html = "" + if dest_hash: + hash_html = ( + f'

    subscribe

    ' + f'

    To subscribe to this instance, add this destination hash in your TinyWeb:

    ' + f'
    {esc(dest_hash)}
    ' + ) + + return _respond( + f'

    {esc(name)}

    ' + f'

    A personal search engine, built for the slow web.

    ' + f'

    TinyWeb is about taking back the internet. No algorithms, no ads, no tracking. ' + f'Just human-curated pages shared freely across a mesh network.

    ' + f'
      ' + f'
    • {page_count} page(s) indexed
      • ' + f'
    • {tag_count} tag(s)
      • ' + f'
    • {sub_count} subscription(s)
      • ' + f'
    ' + f'{sharing_html}' + f'{hash_html}' + f'

    what is the slow web?

    ' + f'

    The slow web is a movement for intentionality over speed, ' + f'human curation over algorithmic feeds, privacy over surveillance, ' + f'and community over corporations. Every page in this index was saved by a person ' + f'because they found it valuable — not because an algorithm told them to click.

    ' + f'

    how it works

    ' + f'
      ' + f'
    • Save pages you find valuable with the bookmarklet or /add
      • ' + f'
    • Search your personal index — queries never leave your machine
      • ' + f'
    • Subscribe to friends over Reticulum — encrypted, decentralized, works without the internet
      • ' + f'
    • Tag and organize your collection into curated lists
      • ' + f'
    ' + f'

    search | browse | tags

    ' + ) + + def handle_tags(): db = get_db() rows = db.execute( @@ -774,6 +827,8 @@ def dispatch_request(data): return handle_bookmark(query) elif path == "/style": return handle_style_form() + elif path == "/about": + return handle_about() elif path == "/export": return handle_export() elif path == "/import": From 1daf29af9af8a9088a88f690186e16fd37cad7c2 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 23:21:02 -0700 Subject: [PATCH 014/194] added an about page with slow-web pitch Shows instance stats, destination hash for subscribing, and explains the slow web movement and how TinyWeb works. Destination hash is stored in settings on startup so the about page can display it. --- app.py | 3 ++- handlers.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index 9233911..7d17007 100644 --- a/app.py +++ b/app.py @@ -4,7 +4,7 @@ import threading import RNS from http.server import HTTPServer -from db import init_db +from db import init_db, set_setting from handlers import dispatch_request from gateway import GatewayState, GatewayHandler, GATEWAY_PORT @@ -55,6 +55,7 @@ def main(): ) destination.announce() + set_setting("dest_hash", destination.hash.hex()) start_gateway(reticulum) print(f"TinyWeb running!") diff --git a/handlers.py b/handlers.py index 42944a5..2c0ec07 100644 --- a/handlers.py +++ b/handlers.py @@ -196,7 +196,8 @@ def handle_search(query): f' | browse' f' | tags' f' | subscriptions' - f' | customize

    ' + f' | customize' + f' | about

    ' f'
    {result_html}{trusted_html}{remote_html}' ) @@ -412,6 +413,58 @@ def handle_style_submit(body): return handle_style_form("Saved.") +def handle_about(): + name = get_site_name() + dest_hash = get_setting("dest_hash") + sharing = get_setting("sharing_enabled", "0") == "1" + db = get_db() + page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + tag_count = db.execute("SELECT count(*) FROM tags").fetchone()[0] + sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0] + db.close() + + sharing_html = ( + '

    This instance shares its index publicly. Subscribe to join the network.

    ' + if sharing else + '

    This instance is private.

    ' + ) + + hash_html = "" + if dest_hash: + hash_html = ( + f'

    subscribe

    ' + f'

    To subscribe to this instance, add this destination hash in your TinyWeb:

    ' + f'
    {esc(dest_hash)}
    ' + ) + + return _respond( + f'

    {esc(name)}

    ' + f'

    A personal search engine, built for the slow web.

    ' + f'

    TinyWeb is about taking back the internet. No algorithms, no ads, no tracking. ' + f'Just human-curated pages shared freely across a mesh network.

    ' + f'
      ' + f'
    • {page_count} page(s) indexed
      • ' + f'
    • {tag_count} tag(s)
      • ' + f'
    • {sub_count} subscription(s)
      • ' + f'
    ' + f'{sharing_html}' + f'{hash_html}' + f'

    what is the slow web?

    ' + f'

    The slow web is a movement for intentionality over speed, ' + f'human curation over algorithmic feeds, privacy over surveillance, ' + f'and community over corporations. Every page in this index was saved by a person ' + f'because they found it valuable — not because an algorithm told them to click.

    ' + f'

    how it works

    ' + f'
      ' + f'
    • Save pages you find valuable with the bookmarklet or /add
      • ' + f'
    • Search your personal index — queries never leave your machine
      • ' + f'
    • Subscribe to friends over Reticulum — encrypted, decentralized, works without the internet
      • ' + f'
    • Tag and organize your collection into curated lists
      • ' + f'
    ' + f'

    search | browse | tags

    ' + ) + + def handle_tags(): db = get_db() rows = db.execute( @@ -774,6 +827,8 @@ def dispatch_request(data): return handle_bookmark(query) elif path == "/style": return handle_style_form() + elif path == "/about": + return handle_about() elif path == "/export": return handle_export() elif path == "/import": From 5c14110b7dc698425d35e34bb1bb63fff7b496d0 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 23:38:15 -0700 Subject: [PATCH 015/194] bound to 0.0.0.0 for remote access --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 7d17007..6f520d2 100644 --- a/app.py +++ b/app.py @@ -30,7 +30,7 @@ def rns_request_handler(path, data, request_id, link_id, remote_identity, reques def start_gateway(reticulum): GatewayState.reticulum = reticulum GatewayState.local_dispatch = dispatch_request - server = HTTPServer(("127.0.0.1", GATEWAY_PORT), GatewayHandler) + server = HTTPServer(("0.0.0.0", GATEWAY_PORT), GatewayHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() From b76731eb60218617c356e24e5f75c9704e662b86 Mon Sep 17 00:00:00 2001 From: blankie Date: Wed, 25 Mar 2026 23:38:15 -0700 Subject: [PATCH 016/194] bound to 0.0.0.0 for remote access --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index 7d17007..6f520d2 100644 --- a/app.py +++ b/app.py @@ -30,7 +30,7 @@ def rns_request_handler(path, data, request_id, link_id, remote_identity, reques def start_gateway(reticulum): GatewayState.reticulum = reticulum GatewayState.local_dispatch = dispatch_request - server = HTTPServer(("127.0.0.1", GATEWAY_PORT), GatewayHandler) + server = HTTPServer(("0.0.0.0", GATEWAY_PORT), GatewayHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() From 2043a7cc7de69e5b31d35cd71752ce1a15f1d6d6 Mon Sep 17 00:00:00 2001 From: blankie Date: Thu, 26 Mar 2026 07:44:26 -0700 Subject: [PATCH 017/194] fixed stale tag count on about page Count tags from page_tags instead of the tags table, which retains orphaned rows when tags are removed from pages. --- handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers.py b/handlers.py index 2c0ec07..6e67f47 100644 --- a/handlers.py +++ b/handlers.py @@ -419,7 +419,7 @@ def handle_about(): sharing = get_setting("sharing_enabled", "0") == "1" db = get_db() page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - tag_count = db.execute("SELECT count(*) FROM tags").fetchone()[0] + tag_count = db.execute("SELECT count(DISTINCT tag_id) FROM page_tags").fetchone()[0] sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0] db.close() From 437584b09df487172c67d60a3a688566958d0679 Mon Sep 17 00:00:00 2001 From: blankie Date: Thu, 26 Mar 2026 07:44:26 -0700 Subject: [PATCH 018/194] fixed stale tag count on about page Count tags from page_tags instead of the tags table, which retains orphaned rows when tags are removed from pages. --- handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers.py b/handlers.py index 2c0ec07..6e67f47 100644 --- a/handlers.py +++ b/handlers.py @@ -419,7 +419,7 @@ def handle_about(): sharing = get_setting("sharing_enabled", "0") == "1" db = get_db() page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - tag_count = db.execute("SELECT count(*) FROM tags").fetchone()[0] + tag_count = db.execute("SELECT count(DISTINCT tag_id) FROM page_tags").fetchone()[0] sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0] db.close() From 612a836771325b6269dd7250097d6161958bc726 Mon Sep 17 00:00:00 2001 From: blankie Date: Thu, 26 Mar 2026 09:04:23 -0700 Subject: [PATCH 019/194] added custom template editor, cleaned up UI - Replace CSS-only customization with full HTML template editing - Users edit the entire page wrapper with {{content}} placeholder - Add /style?reset escape hatch to recover from broken templates - Move nav links to template, remove redundant nav from search page - Delete remote pages when unsubscribing from an instance --- handlers.py | 53 +++++++++++++++++++++------------------------------- templates.py | 27 ++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/handlers.py b/handlers.py index 6e67f47..4ef8939 100644 --- a/handlers.py +++ b/handlers.py @@ -2,15 +2,15 @@ import json from datetime import datetime from db import get_db, get_setting, set_setting, get_site_name, index_url, clean_url -from templates import esc, snippet, wrap_page +from templates import esc, snippet, wrap_page, DEFAULT_TEMPLATE from rns_client import fetch_remote_sites -def _respond(body_html, status=200): +def _respond(body_html, status=200, use_default=False): return { "status": status, "content_type": "text/html; charset=utf-8", - "body": wrap_page(body_html), + "body": wrap_page(body_html, use_default=use_default), "headers": {}, } @@ -186,19 +186,13 @@ def handle_search(query): if q and remote_rows: sub_count = f" + {len(remote_rows)} from subscriptions" return _respond( - f'

    {esc(name)}

    ' f'
    ' f'' f' ' f'
    ' - f'

    {count} page(s) indexed.' - f' + add url' - f' | browse' - f' | tags' - f' | subscriptions' - f' | customize' - f' | about

    ' - f'
    {result_html}{trusted_html}{remote_html}' + f'

    {count} pages indexed' + f' · + add url

    ' + f'{result_html}{trusted_html}{remote_html}' ) @@ -366,8 +360,11 @@ def handle_import_submit(body): return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") -def handle_style_form(msg=""): - css = get_setting("custom_css") +def handle_style_form(msg="", query=None): + if query and "reset" in query: + set_setting("custom_template", "") + msg = "Template reset to default." + template = get_setting("custom_template") or DEFAULT_TEMPLATE name = get_site_name() sharing = get_setting("sharing_enabled", "0") checked = " checked" if sharing == "1" else "" @@ -379,35 +376,26 @@ def handle_style_form(msg=""): f"

    sharing

    " f'

    " - f"

    custom css

    " - f"

    Some classes you can target:

    " - f"
    "
-        f"body          - page background, font\n"
-        f"h1            - page titles\n"
-        f"input, button - search bar\n"
-        f"a             - links\n"
-        f".result       - each search result\n"
-        f".note         - your notes on results\n"
-        f".trusted      - trusted sites dropdown\n"
-        f"small         - url text\n"
-        f"ul, li        - browse page list"
-        f"
    " - f'

    ' + f"

    custom html

    " + f"

    Edit the full page template. Use {esc('{{content}}')} " + f"where page content should appear.

    " + f'

    ' f'' f"" f"

    bookmarklet

    " f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " f'

    + save to {esc(name)}

    ' f"

    {msg}

    " - f'back' + f'back', + use_default=True, ) def handle_style_submit(body): - css = body.get("css", [""])[0] + template = body.get("template", [""])[0] name = body.get("site_name", ["tinyweb"])[0].strip() sharing = "1" if body.get("sharing_enabled") else "0" - set_setting("custom_css", css) + set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "") set_setting("site_name", name or "tinyweb") set_setting("sharing_enabled", sharing) return handle_style_form("Saved.") @@ -755,6 +743,7 @@ def handle_subscription_autosync(sub_id): def handle_subscription_delete(sub_id): db = get_db() + db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub_id,)) db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,)) db.commit() db.close() @@ -826,7 +815,7 @@ def dispatch_request(data): elif path == "/bookmark": return handle_bookmark(query) elif path == "/style": - return handle_style_form() + return handle_style_form(query=query) elif path == "/about": return handle_about() elif path == "/export": diff --git a/templates.py b/templates.py index 735a38e..372e736 100644 --- a/templates.py +++ b/templates.py @@ -15,7 +15,26 @@ def snippet(text, query, ctx=80): return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") -def wrap_page(body_html): - css = get_setting("custom_css") - style = f"" if css else "" - return f"{style}{body_html}" +DEFAULT_TEMPLATE = "\n\n\n\n{{content}}\n\n" + + +def _default_template(): + name = esc(get_setting("site_name", "tinyweb")) + return ( + "\n\n\n\n" + f'

    {name}' + ' | search | browse' + ' | tags | subscriptions' + ' | customize | about

    \n' + "
    \n{{content}}\n\n" + ) + + +def wrap_page(body_html, use_default=False): + if use_default: + template = _default_template() + else: + template = get_setting("custom_template") or _default_template() + if "{{content}}" not in template: + template = _default_template() + return template.replace("{{content}}", body_html) From ad21648d5bab94c7603a2e6305042e14691978dc Mon Sep 17 00:00:00 2001 From: blankie Date: Thu, 26 Mar 2026 09:04:23 -0700 Subject: [PATCH 020/194] added custom template editor, cleaned up UI - Replace CSS-only customization with full HTML template editing - Users edit the entire page wrapper with {{content}} placeholder - Add /style?reset escape hatch to recover from broken templates - Move nav links to template, remove redundant nav from search page - Delete remote pages when unsubscribing from an instance --- handlers.py | 53 +++++++++++++++++++++------------------------------- templates.py | 27 ++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/handlers.py b/handlers.py index 6e67f47..4ef8939 100644 --- a/handlers.py +++ b/handlers.py @@ -2,15 +2,15 @@ import json from datetime import datetime from db import get_db, get_setting, set_setting, get_site_name, index_url, clean_url -from templates import esc, snippet, wrap_page +from templates import esc, snippet, wrap_page, DEFAULT_TEMPLATE from rns_client import fetch_remote_sites -def _respond(body_html, status=200): +def _respond(body_html, status=200, use_default=False): return { "status": status, "content_type": "text/html; charset=utf-8", - "body": wrap_page(body_html), + "body": wrap_page(body_html, use_default=use_default), "headers": {}, } @@ -186,19 +186,13 @@ def handle_search(query): if q and remote_rows: sub_count = f" + {len(remote_rows)} from subscriptions" return _respond( - f'

    {esc(name)}

    ' f'
    ' f'' f' ' f'
    ' - f'

    {count} page(s) indexed.' - f' + add url' - f' | browse' - f' | tags' - f' | subscriptions' - f' | customize' - f' | about

    ' - f'
    {result_html}{trusted_html}{remote_html}' + f'

    {count} pages indexed' + f' · + add url

    ' + f'{result_html}{trusted_html}{remote_html}' ) @@ -366,8 +360,11 @@ def handle_import_submit(body): return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") -def handle_style_form(msg=""): - css = get_setting("custom_css") +def handle_style_form(msg="", query=None): + if query and "reset" in query: + set_setting("custom_template", "") + msg = "Template reset to default." + template = get_setting("custom_template") or DEFAULT_TEMPLATE name = get_site_name() sharing = get_setting("sharing_enabled", "0") checked = " checked" if sharing == "1" else "" @@ -379,35 +376,26 @@ def handle_style_form(msg=""): f"

    sharing

    " f'

    " - f"

    custom css

    " - f"

    Some classes you can target:

    " - f"
    "
-        f"body          - page background, font\n"
-        f"h1            - page titles\n"
-        f"input, button - search bar\n"
-        f"a             - links\n"
-        f".result       - each search result\n"
-        f".note         - your notes on results\n"
-        f".trusted      - trusted sites dropdown\n"
-        f"small         - url text\n"
-        f"ul, li        - browse page list"
-        f"
    " - f'

    ' + f"

    custom html

    " + f"

    Edit the full page template. Use {esc('{{content}}')} " + f"where page content should appear.

    " + f'

    ' f'' f"" f"

    bookmarklet

    " f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " f'

    + save to {esc(name)}

    ' f"

    {msg}

    " - f'back' + f'back', + use_default=True, ) def handle_style_submit(body): - css = body.get("css", [""])[0] + template = body.get("template", [""])[0] name = body.get("site_name", ["tinyweb"])[0].strip() sharing = "1" if body.get("sharing_enabled") else "0" - set_setting("custom_css", css) + set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "") set_setting("site_name", name or "tinyweb") set_setting("sharing_enabled", sharing) return handle_style_form("Saved.") @@ -755,6 +743,7 @@ def handle_subscription_autosync(sub_id): def handle_subscription_delete(sub_id): db = get_db() + db.execute("DELETE FROM remote_pages WHERE subscription_id = ?", (sub_id,)) db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,)) db.commit() db.close() @@ -826,7 +815,7 @@ def dispatch_request(data): elif path == "/bookmark": return handle_bookmark(query) elif path == "/style": - return handle_style_form() + return handle_style_form(query=query) elif path == "/about": return handle_about() elif path == "/export": diff --git a/templates.py b/templates.py index 735a38e..372e736 100644 --- a/templates.py +++ b/templates.py @@ -15,7 +15,26 @@ def snippet(text, query, ctx=80): return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") -def wrap_page(body_html): - css = get_setting("custom_css") - style = f"" if css else "" - return f"{style}{body_html}" +DEFAULT_TEMPLATE = "\n\n\n\n{{content}}\n\n" + + +def _default_template(): + name = esc(get_setting("site_name", "tinyweb")) + return ( + "\n\n\n\n" + f'

    {name}' + ' | search | browse' + ' | tags | subscriptions' + ' | customize | about

    \n' + "
    \n{{content}}\n\n" + ) + + +def wrap_page(body_html, use_default=False): + if use_default: + template = _default_template() + else: + template = get_setting("custom_template") or _default_template() + if "{{content}}" not in template: + template = _default_template() + return template.replace("{{content}}", body_html) From 8c7dd70921ab2a272ae104bb9f6b45fc8cdcdce3 Mon Sep 17 00:00:00 2001 From: blankie Date: Thu, 26 Mar 2026 10:11:32 -0700 Subject: [PATCH 021/194] created themes folder with kodama template Save the custom kodama template to themes/kodama.html so it's version-controlled as a file rather than only living in the database. Stop tracking index.db since it's runtime data, not source code. --- .gitignore | 1 + themes/kodama.html | 746 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 747 insertions(+) create mode 100644 themes/kodama.html diff --git a/.gitignore b/.gitignore index 799f1c1..3917026 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ tinyweb_identity index.db +index.db diff --git a/themes/kodama.html b/themes/kodama.html new file mode 100644 index 0000000..d48b577 --- /dev/null +++ b/themes/kodama.html @@ -0,0 +1,746 @@ + + + + + + + + + + + +
    + +
    +
    + {{content}} +
    +
    +
    curated by hand · shared over mesh
    +
    +
    +
    + + + \ No newline at end of file From 77b52b98099740a6799ce22923a4f9f0a28828a4 Mon Sep 17 00:00:00 2001 From: blankie Date: Thu, 26 Mar 2026 10:11:32 -0700 Subject: [PATCH 022/194] created themes folder with kodama template Save the custom kodama template to themes/kodama.html so it's version-controlled as a file rather than only living in the database. Stop tracking index.db since it's runtime data, not source code. --- .gitignore | 1 + themes/kodama.html | 746 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 747 insertions(+) create mode 100644 themes/kodama.html diff --git a/.gitignore b/.gitignore index 799f1c1..3917026 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ tinyweb_identity index.db +index.db diff --git a/themes/kodama.html b/themes/kodama.html new file mode 100644 index 0000000..d48b577 --- /dev/null +++ b/themes/kodama.html @@ -0,0 +1,746 @@ + + + + + + + + + + + +
    + +
    +
    + {{content}} +
    +
    +
    curated by hand · shared over mesh
    +
    +
    +
    + + + \ No newline at end of file From 35d0a6d444b4442410c7742a72fb03842fd7dc20 Mon Sep 17 00:00:00 2001 From: blankie Date: Thu, 26 Mar 2026 10:54:22 -0700 Subject: [PATCH 023/194] hardened CSRF, SSRF, FTS5 - CSRF: Generate random token at startup, include as hidden field in all 11 POST forms, validate at top of POST dispatch (returns 403) - SSRF: Block private/internal IP ranges (127/8, 10/8, 172.16/12, 192.168/16, 169.254/16, ::1, fc00::/7) by resolving hostname before fetch. Remove verify=False from requests.get(). - DELETE: Change /delete/ from GET (instant delete) to GET (confirmation page) + POST (actual delete) to prevent accidental deletion from prefetchers/crawlers. - FTS5: Wrap search input in double quotes to neutralize FTS5 operators (AND, OR, NOT, *, column:). Add try/except fallback. --- db.py | 35 ++++++++++++++++++++- handlers.py | 91 ++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 106 insertions(+), 20 deletions(-) diff --git a/db.py b/db.py index b523c79..c9ea195 100644 --- a/db.py +++ b/db.py @@ -1,3 +1,5 @@ +import socket +import ipaddress import sqlite3 import requests from urllib.parse import urlparse, urljoin, parse_qs, urlencode, urlunparse @@ -5,6 +7,36 @@ from bs4 import BeautifulSoup DATABASE = "index.db" +BLOCKED_NETWORKS = [ + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("169.254.0.0/16"), + ipaddress.ip_network("0.0.0.0/8"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("fc00::/7"), + ipaddress.ip_network("fe80::/10"), +] + + +def _validate_url_target(url): + """Resolve hostname and block private/internal IPs to prevent SSRF.""" + parsed = urlparse(url) + hostname = parsed.hostname + port = parsed.port or (443 if parsed.scheme == "https" else 80) + if not hostname: + raise ValueError(f"No hostname in URL: {url}") + try: + addrs = socket.getaddrinfo(hostname, port, proto=socket.IPPROTO_TCP) + except socket.gaierror: + raise ValueError(f"Cannot resolve hostname: {hostname}") + for family, type_, proto, canonname, sockaddr in addrs: + ip = ipaddress.ip_address(sockaddr[0]) + for network in BLOCKED_NETWORKS: + if ip in network: + raise ValueError(f"URL resolves to blocked address: {ip}") + SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") TRACKING_PARAMS = { @@ -167,7 +199,8 @@ def get_site_name(): def fetch_page(url): - resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) + _validate_url_target(url) + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}) resp.raise_for_status() soup = BeautifulSoup(resp.text, "html.parser") diff --git a/handlers.py b/handlers.py index 4ef8939..0045767 100644 --- a/handlers.py +++ b/handlers.py @@ -1,10 +1,28 @@ import json +import secrets from datetime import datetime from db import get_db, get_setting, set_setting, get_site_name, index_url, clean_url from templates import esc, snippet, wrap_page, DEFAULT_TEMPLATE from rns_client import fetch_remote_sites +_csrf_token = secrets.token_hex(32) + + +def _csrf_field(): + return f'' + + +def _check_csrf(body): + token = body.get("_csrf", [""])[0] + return secrets.compare_digest(token, _csrf_token) + + +def _sanitize_fts_query(query): + """Escape user input for safe use in FTS5 MATCH.""" + escaped = query.replace('"', '""') + return f'"{escaped}"' + def _respond(body_html, status=200, use_default=False): return { @@ -90,12 +108,15 @@ def handle_search(query): result_html = "" trusted_html = "" if q: - rows = db.execute( - "SELECT p.id, p.url, p.title, p.body, p.note " - "FROM pages_fts f JOIN pages p ON f.rowid = p.id " - "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", - (q,), - ).fetchall() + try: + rows = db.execute( + "SELECT p.id, p.url, p.title, p.body, p.note " + "FROM pages_fts f JOIN pages p ON f.rowid = p.id " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", + (_sanitize_fts_query(q),), + ).fetchall() + except Exception: + rows = [] if rows: for r in rows: note_html = "" @@ -150,14 +171,17 @@ def handle_search(query): ) # search synced pages from subscriptions - remote_rows = db.execute( - "SELECT rp.url, rp.title, rp.note, s.name AS source_name " - "FROM remote_pages_fts rpf " - "JOIN remote_pages rp ON rpf.rowid = rp.id " - "JOIN subscriptions s ON rp.subscription_id = s.id " - "WHERE remote_pages_fts MATCH ? ORDER BY rank LIMIT 50", - (q,), - ).fetchall() + try: + remote_rows = db.execute( + "SELECT rp.url, rp.title, rp.note, s.name AS source_name " + "FROM remote_pages_fts rpf " + "JOIN remote_pages rp ON rpf.rowid = rp.id " + "JOIN subscriptions s ON rp.subscription_id = s.id " + "WHERE remote_pages_fts MATCH ? ORDER BY rank LIMIT 50", + (_sanitize_fts_query(q),), + ).fetchall() + except Exception: + remote_rows = [] remote_html = "" if q and remote_rows: @@ -200,6 +224,7 @@ def handle_add_form(msg=""): return _respond( f"

    add url

    " f'
    ' + f'{_csrf_field()}' f'

    ' f'

    ' f'

    ' @@ -271,6 +296,7 @@ def handle_edit_form(page_id, msg=""): f"

    {esc(row['title'])}
    " f"{esc(row['url'])}

    " f'' + f'{_csrf_field()}' f'

    ' f'

    ' f'' @@ -291,6 +317,24 @@ def handle_edit_submit(page_id, body): return _redirect("/pages") +def handle_delete_confirm(page_id): + db = get_db() + row = db.execute("SELECT id, url, title FROM pages WHERE id = ?", (page_id,)).fetchone() + db.close() + if not row: + return _error(404) + return _respond( + f"

    confirm delete

    " + f"

    Remove {esc(row['title'])}
    " + f"{esc(row['url'])}

    " + f'' + f'{_csrf_field()}' + f'' + f"
    " + f' cancel' + ) + + def handle_delete(page_id): db = get_db() db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) @@ -325,6 +369,7 @@ def handle_import_form(msg=""): f"

    import

    " f"

    Paste the contents of a tinyweb export file (JSON).

    " f'
    ' + f'{_csrf_field()}' f'

    ' f'' f"
    " @@ -372,6 +417,7 @@ def handle_style_form(msg="", query=None): f"

    customize

    " f"

    name your search engine

    " f'
    ' + f'{_csrf_field()}' f'

    ' f"

    sharing

    " f'