No results in your index.
" - - # search all linked pages from trusted sites - words = q.lower().split() - all_links = db.execute( - "SELECT l.url, l.label, p.title AS source_title " - "FROM links l JOIN pages p ON l.page_id = p.id", - ).fetchall() - indexed_urls = set(r["url"] for r in rows) if rows else set() - seen = set() - trusted = [] - for l in all_links: - if l["url"] in indexed_urls or l["url"] in seen: - continue - if any(w in l["label"].lower() for w in words): - seen.add(l["url"]) - trusted.append(l) - if len(trusted) >= 20: - break - - if trusted: - items = "" - for l in trusted: - items += ( - f'{count} page(s) indexed.' - f' + add url' - f' | browse' - f' | customize
' - f'{msg}
" - f'back' - ) - - def handle_add_submit(self, params): - url = params.get("url", [""])[0].strip() - note = params.get("note", [""])[0].strip() - if not url: - return self.handle_add_form("URL is required.") - if not url.startswith(("http://", "https://")): - return self.handle_add_form("URL must start with http:// or https://") - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - self.handle_add_form(f'Indexed: {esc(title)}') - except Exception as e: - self.handle_add_form(f"Error: {esc(str(e))}") - - def handle_pages(self): - db = get_db() - rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() - db.close() - items = "" - for r in rows: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - items += ( - f'{esc(row['title'])}
"
- f"{esc(row['url'])}
{msg}
" - f'back' - ) - - def handle_edit_submit(self, path, params): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("Paste the contents of a tinyweb export file (JSON).
" - f'" - f"{msg}
" - f'back' - ) + lora_block = "" + if get_setting("lora_enabled", "0") == "1": + lora_port = get_setting("lora_port", "") + if lora_port: + lora_frequency = get_setting("lora_frequency", "867200000") + lora_bandwidth = get_setting("lora_bandwidth", "125000") + lora_txpower = get_setting("lora_txpower", "7") + lora_sf = get_setting("lora_sf", "8") + lora_cr = get_setting("lora_cr", "5") + lora_block = f""" + [[RNode LoRa]] + type = RNodeInterface + enabled = yes + port = {lora_port} + frequency = {lora_frequency} + bandwidth = {lora_bandwidth} + txpower = {lora_txpower} + spreadingfactor = {lora_sf} + codingrate = {lora_cr} +""" - def handle_import_submit(self, params): - raw = params.get("data", [""])[0].strip() - if not raw: - return self.handle_import_form("Paste JSON data.") - try: - data = json.loads(raw) - except json.JSONDecodeError: - return self.handle_import_form("Invalid JSON.") - if not isinstance(data, list): - return self.handle_import_form("Expected a JSON array.") + os.makedirs(config_dir, exist_ok=True) + with open(config_file, "w") as f: + f.write(f"""{managed_sentinel} +[reticulum] + enable_transport = False + share_instance = No - imported = 0 - errors = 0 - for entry in data: - url = entry.get("url", "").strip() - note = entry.get("note", "").strip() - if not url: - continue - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - imported += 1 - except Exception: - errors += 1 +[logging] + loglevel = 4 - self.handle_import_form(f"Imported {imported} page(s). {errors} error(s).") +[interfaces] + [[Default Interface]] + type = AutoInterface + enabled = Yes +{tcp_block}{lora_block}""") + print(f"Created Reticulum config at {config_file}") - def handle_style_form(self, msg=""): - css = get_setting("custom_css") - name = get_site_name() - self.respond( - f"Drag this link to your bookmarks bar. Click it on any page to index it instantly.
" - f'' - f"{msg}
" - f'back' - ) - def handle_style_submit(self, params): - css = params.get("css", [""])[0] - name = params.get("site_name", ["tinyweb"])[0].strip() - set_setting("custom_css", css) - set_setting("site_name", name or "tinyweb") - self.handle_style_form("Saved.") +def _preload_embeddings(): + """Pre-load the embedding model and build the HNSW index in background.""" + if get_setting("semantic_search", "0") != "1": + print("Semantic search disabled.") + return + try: + from embeddings import _get_session, _get_reranker, build_index + _get_session() + build_index() + if get_setting("use_reranker", "0") == "1": + _get_reranker() + print("Semantic search ready (with reranker).") + else: + print("Semantic search ready.") + except Exception as e: + print(f"Semantic search unavailable: {e}") + + +def main(): + parser = argparse.ArgumentParser(prog="tinyweb", description="Personal decentralized search engine") + parser.add_argument("--version", "-v", action="store_true", help="Show version") + parser.add_argument("--port", "-p", type=int, default=None, help="HTTP gateway port (default: 8080)") + parser.add_argument( + "--bind", "-b", default="127.0.0.1", + help="Address to bind the HTTP gateway to (default: 127.0.0.1). " + "Use 0.0.0.0 to expose to the LAN; note that the web UI has no authentication.", + ) + args = parser.parse_args() + + if args.version: + print(f"TinyWeb {get_version()}") + return + + bind_host = args.bind + port = args.port or 8080 + gateway.GATEWAY_PORT = find_available_port(port, host=bind_host) + + init_db() + transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) + transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))) + threading.Thread(target=_preload_embeddings, daemon=True).start() + config_dir = os.environ.get("RNS_CONFIG_DIR") + ensure_rns_config(config_dir, transport_host, transport_port) + reticulum = RNS.Reticulum(configdir=config_dir) + identity = load_or_create_identity() + + destination = RNS.Destination( + identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, + ) + + destination.register_request_handler( + "/tinyweb", + response_generator=rns_request_handler, + allow=RNS.Destination.ALLOW_ALL, + ) + + # Initialize forum plugin if available + forum = None + try: + from tinyweb_forum import ForumPlugin + from db import get_site_name + forum = ForumPlugin(DATA_DIR, identity, reticulum, site_name=get_site_name()) + if get_setting("forum_enabled", "0") == "1": + forum.enable() + templates_mod.FORUM_ENABLED = True + handlers_mod.forum_plugin = forum + print(f"Forum plugin: {'enabled' if forum.is_enabled() else 'available (enable in settings)'}") + except ImportError: + print("Forum plugin not installed (pip install tinyweb[forum])") + except Exception as e: + print(f"Forum plugin error: {e}") + + # Brief delay to ensure all interfaces (especially TCP) are fully ready + time.sleep(2) + destination.announce() + set_setting("dest_hash", destination.hash.hex()) + start_gateway(reticulum, bind_host=bind_host) + + print(f"TinyWeb running!") + if bind_host in ("0.0.0.0", "::"): + print(f"Open http://localhost:{gateway.GATEWAY_PORT} in your browser") + print(f"WARNING: listening on {bind_host} — the web UI has no authentication. " + "Anyone on your network can control this instance.") + else: + print(f"Open http://{bind_host}:{gateway.GATEWAY_PORT} in your browser") + print(f"Destination hash: {RNS.prettyhexrep(destination.hash)} (share this so friends can subscribe)") + + while True: + time.sleep(1) if __name__ == "__main__": - init_db() - print("running on http://localhost:5001") - HTTPServer(("localhost", 5001), Handler).serve_forever() + main() diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..9a2f26e --- /dev/null +++ b/conftest.py @@ -0,0 +1,128 @@ +"""Shared pytest fixtures for TinyWeb tests. + +Three fixtures cover most tests: `temp_db` swaps the SQLite path to a +per-test tempfile, `seeded_db` layers sample rows on top, and `csrf_session` +primes the thread-local CSRF token that handlers read. +""" +import socket +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent)) + +import db as db_module +import handlers as handlers_module + + +@pytest.fixture +def temp_db(tmp_path, monkeypatch): + """Isolated SQLite DB per test. + + Swaps `db.DATABASE` and `db.DATA_DIR` to a tempdir, clears the connection + pool before and after so state doesn't leak across tests, and calls + `init_db()` so every schema object exists. + """ + data_dir = tmp_path / "tinyweb" + data_dir.mkdir() + db_path = data_dir / "index.db" + + monkeypatch.setattr(db_module, "DATA_DIR", str(data_dir)) + monkeypatch.setattr(db_module, "DATABASE", str(db_path)) + + with db_module._pool_lock: + for conn in db_module._pool: + try: + conn.close() + except Exception: + pass + db_module._pool.clear() + + db_module.init_db() + yield db_path + + with db_module._pool_lock: + for conn in db_module._pool: + try: + conn.close() + except Exception: + pass + db_module._pool.clear() + + +@pytest.fixture +def seeded_db(temp_db): + """A temp DB with a small, realistic set of pages/tags/links.""" + db = db_module.get_db() + try: + rows = [ + ("https://example.com/rust-intro", "Rust Intro", "A gentle introduction to rust borrow checker.", "notes on ownership"), + ("https://example.com/python-tips", "Python Tips", "Daily python tricks for readable code.", ""), + ("https://example.com/ocaml-why", "Why OCaml", "Type systems and inference in ocaml.", "private thoughts"), + ("https://news.example.org/mesh", "Mesh Networking", "Reticulum and LoRa for decentralized networks.", ""), + ] + for url, title, body, note in rows: + db.execute( + "INSERT INTO pages (url, title, body, note, last_modified) " + "VALUES (?, ?, ?, ?, '2026-04-01T00:00:00')", + (url, title, body, note), + ) + db.commit() + page_ids = { + row["url"]: row["id"] + for row in db.execute("SELECT id, url FROM pages").fetchall() + } + tag_rows = [ + (page_ids["https://example.com/rust-intro"], ["rust", "public"]), + (page_ids["https://example.com/python-tips"], ["python"]), + (page_ids["https://example.com/ocaml-why"], ["ocaml", "private"]), + (page_ids["https://news.example.org/mesh"], ["mesh", "public"]), + ] + for pid, tags in tag_rows: + for name in tags: + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,)) + tid = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()[0] + db.execute( + "INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", + (pid, tid), + ) + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_ids["https://example.com/rust-intro"], "https://example.com/rust-advanced", "advanced rust guide"), + ) + db.commit() + finally: + db_module.return_db(db) + return temp_db + + +@pytest.fixture +def csrf_session(monkeypatch): + """Prime the CSRF thread-local so handler code that calls _get_csrf_token works.""" + token = "test-csrf-token" + handlers_module._request_local.csrf_token = token + yield token + if hasattr(handlers_module._request_local, "csrf_token"): + del handlers_module._request_local.csrf_token + + +def patch_dns_fail(monkeypatch): + """Make every socket.getaddrinfo call raise gaierror for the rest of this test.""" + def boom(*args, **kwargs): + raise socket.gaierror("test: DNS disabled") + monkeypatch.setattr(socket, "getaddrinfo", boom) + + +def patch_dns_ok(monkeypatch, address="93.184.216.34"): + """Make every getaddrinfo return a single public IP for the rest of this test.""" + def ok(host, port, *args, **kwargs): + return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (address, port or 80))] + monkeypatch.setattr(socket, "getaddrinfo", ok) + + +def patch_dns_private(monkeypatch, address="127.0.0.1"): + """Make every getaddrinfo return a private/blocked IP for the rest of this test.""" + def private(host, port, *args, **kwargs): + return [(socket.AF_INET, socket.SOCK_STREAM, 0, "", (address, port or 80))] + monkeypatch.setattr(socket, "getaddrinfo", private) diff --git a/db.py b/db.py new file mode 100644 index 0000000..ec13254 --- /dev/null +++ b/db.py @@ -0,0 +1,449 @@ +import socket +import ipaddress +import sqlite3 +import requests +import os +from urllib.parse import urlparse, urljoin, parse_qs, urlencode, urlunparse, quote +from bs4 import BeautifulSoup + +DATA_DIR = os.path.expanduser("~/.tinyweb") +DATABASE = os.path.join(DATA_DIR, "index.db") + +BLOCKED_NETWORKS = [ + ipaddress.ip_network("127.0.0.0/8"), + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("169.254.0.0/16"), + ipaddress.ip_network("0.0.0.0/8"), + ipaddress.ip_network("::1/128"), + ipaddress.ip_network("fc00::/7"), + ipaddress.ip_network("fe80::/10"), +] + + +def _is_blocked_response(html, status_code): + """Check if response is a CDN challenge/block page.""" + if status_code == 403: + return True + html_lower = html.lower() + if "just a moment" in html_lower or "cloudflare" in html_lower: + return True + if "enable javascript and cookies" in html_lower: + return True + if "request rejected" in html_lower: + return True + if "access denied" in html_lower: + return True + return False + + +def _validate_url_target(url): + """Resolve hostname and block private/internal IPs to prevent SSRF.""" + parsed = urlparse(url) + hostname = parsed.hostname + port = parsed.port or (443 if parsed.scheme == "https" else 80) + if not hostname: + raise ValueError(f"No hostname in URL: {url}") + try: + addrs = socket.getaddrinfo(hostname, port, proto=socket.IPPROTO_TCP) + except socket.gaierror: + raise ValueError(f"Cannot resolve hostname: {hostname}") + for family, type_, proto, canonname, sockaddr in addrs: + ip = ipaddress.ip_address(sockaddr[0]) + for network in BLOCKED_NETWORKS: + if ip in network: + raise ValueError(f"URL resolves to blocked address: {ip}") + +SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + +TRACKING_PARAMS = { + "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", + "fbclid", "gclid", "msclkid", "mc_cid", "mc_eid", "ref", "ref_src", + "ref_url", "_ga", "_gl", "yclid", "twclid", "igshid", +} + + +def clean_url(url): + parsed = urlparse(url) + + # Prefer https + scheme = "https" if parsed.scheme in ("http", "https") else parsed.scheme + + # Normalize hostname: lowercase, strip www (only if non-www resolves) + hostname = (parsed.hostname or "").lower() + original_hostname = hostname + if hostname.startswith("www."): + hostname = hostname[4:] + port = parsed.port or (443 if scheme == "https" else 80) + try: + socket.getaddrinfo(hostname, port, proto=socket.IPPROTO_TCP) + except socket.gaierror: + hostname = original_hostname + + # Preserve explicit non-default ports + port = parsed.port + if port and ((scheme == "https" and port == 443) or (scheme == "http" and port == 80)): + port = None + netloc = f"{hostname}:{port}" if port else hostname + + # Strip trailing slash (keep root "/" as-is) + path = parsed.path.rstrip("/") or "/" + + # Remove tracking params and sort remaining for consistent ordering + params = parse_qs(parsed.query) + cleaned = sorted( + ((k, sorted(v)) for k, v in params.items() if k.lower() not in TRACKING_PARAMS), + key=lambda x: x[0], + ) + new_query = urlencode(cleaned, doseq=True, quote_via=quote) + + return urlunparse((scheme, netloc, path, "", new_query, "")) + + +_pool = [] +_pool_lock = __import__("threading").Lock() +_POOL_SIZE = 16 + + +def get_db(): + with _pool_lock: + if _pool: + db = _pool.pop() + try: + db.execute("SELECT 1") + return db + except Exception: + pass + db = sqlite3.connect(DATABASE, timeout=10) + db.execute("PRAGMA journal_mode=WAL") + db.execute("PRAGMA foreign_keys = ON") + db.row_factory = sqlite3.Row + return db + + +def return_db(db): + try: + db.rollback() + except Exception: + try: + db.close() + except Exception: + pass + return + with _pool_lock: + if len(_pool) < _POOL_SIZE: + _pool.append(db) + else: + db.close() + + +def init_db(): + os.makedirs(DATA_DIR, exist_ok=True) + db = sqlite3.connect(DATABASE) + db.execute( + "CREATE TABLE IF NOT EXISTS pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " title TEXT," + " body TEXT," + " note TEXT DEFAULT ''," + " last_modified TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now'))," + " reticulum_dest TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " + "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS links (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " page_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " label TEXT," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS settings (" + " key TEXT PRIMARY KEY," + " value TEXT" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS subscriptions (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " dest_hash TEXT UNIQUE NOT NULL," + " name TEXT DEFAULT ''," + " auto_sync INTEGER DEFAULT 0," + " last_sync TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS remote_pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " subscription_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " title TEXT," + " note TEXT DEFAULT ''," + " tags TEXT DEFAULT ''," + " FOREIGN KEY (subscription_id) REFERENCES subscriptions(id) ON DELETE CASCADE," + " UNIQUE(subscription_id, url)" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS remote_pages_fts " + "USING fts5(title, url, note, content=remote_pages, content_rowid=id)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS tags (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " name TEXT UNIQUE NOT NULL" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS page_tags (" + " page_id INTEGER NOT NULL," + " tag_id INTEGER NOT NULL," + " PRIMARY KEY (page_id, tag_id)," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE," + " FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE" + ")" + ) + db.executescript(""" + CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS remote_pages_ai AFTER INSERT ON remote_pages BEGIN + INSERT INTO remote_pages_fts(rowid, title, url, note) + VALUES (new.id, new.title, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS remote_pages_ad AFTER DELETE ON remote_pages BEGIN + INSERT INTO remote_pages_fts(remote_pages_fts, rowid, title, url, note) + VALUES ('delete', old.id, old.title, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS remote_pages_au AFTER UPDATE ON remote_pages BEGIN + INSERT INTO remote_pages_fts(remote_pages_fts, rowid, title, url, note) + VALUES ('delete', old.id, old.title, old.url, old.note); + INSERT INTO remote_pages_fts(rowid, title, url, note) + VALUES (new.id, new.title, new.url, new.note); + END; + """) + # Migrate old subscriptions table if needed + cols = [row[1] for row in db.execute("PRAGMA table_info(subscriptions)").fetchall()] + if "url" in cols and "dest_hash" not in cols: + db.execute("ALTER TABLE subscriptions RENAME COLUMN url TO dest_hash") + db.commit() + + # Migrate remote_pages: add tags column if missing + rp_cols = [row[1] for row in db.execute("PRAGMA table_info(remote_pages)").fetchall()] + if "tags" not in rp_cols: + db.execute("ALTER TABLE remote_pages ADD COLUMN tags TEXT DEFAULT ''") + db.commit() + + # Migrate pages: add last_modified column if missing + page_cols = [row[1] for row in db.execute("PRAGMA table_info(pages)").fetchall()] + if "last_modified" not in page_cols: + db.execute("ALTER TABLE pages ADD COLUMN last_modified TEXT DEFAULT ''") + db.execute("UPDATE pages SET last_modified = strftime('%Y-%m-%dT%H:%M:%S','now') WHERE last_modified = ''") + db.commit() + + # Migrate pages: add summary column if missing + if "summary" not in page_cols: + db.execute("ALTER TABLE pages ADD COLUMN summary TEXT DEFAULT ''") + db.commit() + + # Migrate pages: add reticulum_dest column if missing + if "reticulum_dest" not in page_cols: + db.execute("ALTER TABLE pages ADD COLUMN reticulum_dest TEXT DEFAULT ''") + db.commit() + + # Chunks table for semantic search embeddings + db.execute( + "CREATE TABLE IF NOT EXISTS chunks (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " page_id INTEGER," + " remote_page_id INTEGER," + " chunk_index INTEGER NOT NULL," + " chunk_text TEXT NOT NULL," + " embedding BLOB NOT NULL," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE," + " FOREIGN KEY (remote_page_id) REFERENCES remote_pages(id) ON DELETE CASCADE" + ")" + ) + db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_page ON chunks(page_id)") + db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_remote ON chunks(remote_page_id)") + db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_page_idx ON chunks(page_id, chunk_index)") + db.execute("CREATE INDEX IF NOT EXISTS idx_pages_url ON pages(url)") + db.execute("CREATE INDEX IF NOT EXISTS idx_pages_modified ON pages(last_modified)") + db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_page ON page_tags(page_id)") + db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_tag ON page_tags(tag_id)") + + # Migrate custom_template: replace hardcoded forum link with {{forum_link}} placeholder + cur = db.execute("SELECT value FROM settings WHERE key='custom_template'") + row = cur.fetchone() + if row: + updated = row[0].replace('forum', "{{forum_link}}") + if updated != row[0]: + db.execute("UPDATE settings SET value=? WHERE key='custom_template'", (updated,)) + db.commit() + + # Migrate custom_template: replace hardcoded site name with {{site_name}} placeholder + cur = db.execute("SELECT value FROM settings WHERE key='custom_template'") + row = cur.fetchone() + if row and '{{site_name}}' not in row[0]: + updated = row[0].replace('href="/">tinyweb', 'href="/">{{site_name}}') + if updated != row[0]: + db.execute("UPDATE settings SET value=? WHERE key='custom_template'", (updated,)) + db.commit() + + db.execute("PRAGMA journal_mode=WAL") + db.execute("PRAGMA synchronous=NORMAL") + db.execute("PRAGMA cache_size=-64000") + db.commit() + db.close() + + +def get_setting(key, default=""): + db = get_db() + try: + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + return row["value"] if row else default + finally: + return_db(db) + + +def vacuum_db(): + """Run VACUUM and WAL checkpoint to reclaim space after deletions.""" + db = get_db() + try: + db.execute("PRAGMA wal_checkpoint(TRUNCATE)") + db.execute("VACUUM") + finally: + return_db(db) + + +def set_setting(key, value): + db = get_db() + try: + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + finally: + return_db(db) + + +def get_site_name(): + return get_setting("site_name", "tinyweb") + + +def fetch_page(url): + _validate_url_target(url) + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False) + + if _is_blocked_response(resp.text, resp.status_code): + raise Exception(f"Site blocks automated access: {resp.status_code}") + + # Follow redirects manually, re-validating each target + max_redirects = 5 + while resp.is_redirect and max_redirects > 0: + redirect_url = resp.headers.get("Location") + if not redirect_url: + break + redirect_url = urljoin(url, redirect_url) + _validate_url_target(redirect_url) + url = redirect_url + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False) + max_redirects -= 1 + resp.raise_for_status() + soup = BeautifulSoup(resp.text, "html.parser") + + # extract links before stripping tags + domain = urlparse(url).netloc + seen = set() + links = [] + for a in soup.find_all("a", href=True): + href = urljoin(url, a["href"]).split("#")[0] + parsed = urlparse(href) + if parsed.netloc != domain: + continue + if any(href.lower().endswith(ext) for ext in SKIP_EXT): + continue + if parsed.query or "action=" in href: + continue + path = parsed.path.lower() + if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): + continue + if href in seen or href == url: + continue + seen.add(href) + label = a.get_text(strip=True) or href + links.append((href, label[:200])) + + # Extract meta description before stripping tags (case-insensitive) + meta_desc = "" + for m in soup.find_all("meta"): + name = (m.get("name") or "").lower() + prop = (m.get("property") or "").lower() + content = (m.get("content") or "").strip() + if not content: + continue + if name == "description" and len(content) > len(meta_desc): + meta_desc = content + elif prop == "og:description" and not meta_desc: + meta_desc = content + + for tag in soup(["script", "style", "nav", "footer", "header", "noscript", "aside"]): + tag.decompose() + title = soup.title.string.strip() if soup.title and soup.title.string else url + + body = soup.get_text(separator=" ", strip=True) + return title, body, links, meta_desc + + + +def index_url(url, note="", reticulum_dest=""): + url = clean_url(url) + title, body, links, meta_desc = fetch_page(url) + summary = meta_desc if meta_desc and len(meta_desc) > 20 else "" + db = get_db() + try: + now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + db.execute( + "INSERT INTO pages (url, title, body, note, last_modified, summary, reticulum_dest) VALUES (?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, " + "note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary, reticulum_dest=excluded.reticulum_dest", + (url, title, body, note, now, summary, reticulum_dest), + ) + page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0] + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + if get_setting("semantic_search", "0") == "1": + try: + from embeddings import store_embeddings + store_embeddings(page_id, title, body, db) + except Exception: + pass # embedding generation is best-effort + finally: + return_db(db) + return title diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..151a79c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +services: + tinyweb: + build: . + ports: + - "8080:8080" + volumes: + - tinyweb-data:/data + restart: unless-stopped + # Connect to another Reticulum instance over TCP. + # Required on macOS (Docker can't do LAN auto-discovery). + # On Linux, auto-discovery works with network_mode: host. + # environment: + # - RNS_TCP_HOST=10.0.0.100 + # - RNS_TCP_PORT=4242 + +volumes: + tinyweb-data: diff --git a/embeddings.py b/embeddings.py new file mode 100644 index 0000000..03f6f13 --- /dev/null +++ b/embeddings.py @@ -0,0 +1,586 @@ +"""Semantic search using Snowflake arctic-embed-s via ONNX Runtime + hnswlib.""" + +import os +import re +import threading +import numpy as np + +DATA_DIR = os.path.expanduser("~/.tinyweb") +MODEL_ID = "Snowflake/snowflake-arctic-embed-s" +MODEL_DIR = os.path.join(DATA_DIR, "models", "snowflake-arctic-embed-s") +RERANKER_DIR = os.path.join(DATA_DIR, "models", "cross-encoder") +HNSW_PATH = os.path.join(DATA_DIR, "index.hnsw") +DIMS = 384 +MAX_TOKENS = 512 +QUERY_PREFIX = "Represent this sentence for searching relevant passages: " + +_session = None +_tokenizer = None +_lock = threading.Lock() + +_reranker_session = None +_reranker_tokenizer = None +_reranker_lock = threading.Lock() + +# Live HNSW index and chunk-id mapping +_hnsw_index = None +_hnsw_ids = [] # maps internal HNSW label -> chunks.id +_hnsw_lock = threading.Lock() + + +# --------------------------------------------------------------------------- +# Model download & loading +# --------------------------------------------------------------------------- + +def _ensure_model(): + """Download the ONNX model and tokenizer from HuggingFace if not present.""" + os.makedirs(MODEL_DIR, exist_ok=True) + model_path = os.path.join(MODEL_DIR, "model.onnx") + tokenizer_path = os.path.join(MODEL_DIR, "tokenizer.json") + if os.path.exists(model_path) and os.path.exists(tokenizer_path): + return + from huggingface_hub import hf_hub_download + os.makedirs(MODEL_DIR, exist_ok=True) + files = { + "onnx/model_quantized.onnx": "model.onnx", + "tokenizer.json": "tokenizer.json", + "tokenizer_config.json": "tokenizer_config.json", + } + for remote, local in files.items(): + target = os.path.join(MODEL_DIR, local) + if os.path.exists(target): + continue + cached = hf_hub_download(repo_id=MODEL_ID, filename=remote) + # hf_hub_download returns the cached file path; copy to our model dir + import shutil + shutil.copy2(cached, target) + + +def _get_session(): + """Return (onnxruntime.InferenceSession, tokenizers.Tokenizer) singleton.""" + global _session, _tokenizer + if _session is not None: + return _session, _tokenizer + with _lock: + if _session is not None: + return _session, _tokenizer + _ensure_model() + import onnxruntime as ort + from tokenizers import Tokenizer + _session = ort.InferenceSession( + os.path.join(MODEL_DIR, "model.onnx"), + providers=["CPUExecutionProvider"], + ) + _tokenizer = Tokenizer.from_file(os.path.join(MODEL_DIR, "tokenizer.json")) + _tokenizer.enable_truncation(max_length=MAX_TOKENS) + _tokenizer.enable_padding(pad_id=0, pad_token="[PAD]", length=None) + return _session, _tokenizer + + +def _get_reranker(): + """Return (onnxruntime.InferenceSession, tokenizers.Tokenizer) for the cross-encoder reranker.""" + global _reranker_session, _reranker_tokenizer + if _reranker_session is not None: + return _reranker_session, _reranker_tokenizer + with _reranker_lock: + if _reranker_session is not None: + return _reranker_session, _reranker_tokenizer + model_path = os.path.join(RERANKER_DIR, "model.onnx") + tok_path = os.path.join(RERANKER_DIR, "tokenizer.json") + if not os.path.exists(model_path) or not os.path.exists(tok_path): + return None, None + import onnxruntime as ort + from tokenizers import Tokenizer + _reranker_session = ort.InferenceSession( + model_path, providers=["CPUExecutionProvider"], + ) + _reranker_tokenizer = Tokenizer.from_file(tok_path) + _reranker_tokenizer.enable_truncation(max_length=512) + _reranker_tokenizer.enable_padding(pad_id=0, pad_token="[PAD]", length=None) + return _reranker_session, _reranker_tokenizer + + +def rerank(query, documents, limit=10): + """Score query-document pairs with the cross-encoder and return reranked indices. + + Args: + query: search query string + documents: list of document texts to score against the query + limit: max results to return + + Returns: list of (original_index, score) sorted by score descending. + """ + session, tokenizer = _get_reranker() + if session is None: + return [(i, 0.0) for i in range(min(limit, len(documents)))] + + # Cross-encoder takes (query, document) pairs — encode as pair sequences + pairs = [[query, doc] for doc in documents] + encodings = tokenizer.encode_batch(pairs) + + input_ids = np.array([e.ids for e in encodings], dtype=np.int64) + attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64) + token_type_ids = np.array([e.type_ids for e in encodings], dtype=np.int64) + + outputs = session.run( + None, + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids, + }, + ) + # Output is logits — higher = more relevant + scores = outputs[0].flatten() + ranked = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) + return [(i, float(scores[i])) for i in ranked[:limit]] + + +# --------------------------------------------------------------------------- +# Text chunking +# --------------------------------------------------------------------------- + +_SENTENCE_RE = re.compile(r'(?<=[.!?])\s+') + + +def chunk_text(title, body): + """Split body into chunks, each prefixed with title for context. + + Strategy: split on double newlines (paragraphs). If a paragraph exceeds + MAX_TOKENS words, split at sentence boundaries. Each chunk is prefixed + with the page title. + """ + if not body or not body.strip(): + return [f"{title}"] if title else [] + + prefix = f"{title}: " if title else "" + # Rough word budget for chunk body (leave room for prefix) + prefix_words = len(prefix.split()) + max_words = MAX_TOKENS - prefix_words # approximate; tokenizer may differ + + paragraphs = re.split(r'\n\s*\n', body.strip()) + chunks = [] + + for para in paragraphs: + para = para.strip() + if len(para) < 20: + continue + words = para.split() + if len(words) <= max_words: + chunks.append(prefix + para) + else: + # Split paragraph into sentences + sentences = _SENTENCE_RE.split(para) + current = [] + current_len = 0 + for sent in sentences: + sent_words = len(sent.split()) + if current_len + sent_words > max_words and current: + chunks.append(prefix + " ".join(current)) + current = [] + current_len = 0 + if sent_words > max_words: + # Sentence too long — use sliding window + s_words = sent.split() + for i in range(0, len(s_words), max_words - 50): + window = s_words[i:i + max_words] + chunks.append(prefix + " ".join(window)) + else: + current.append(sent) + current_len += sent_words + if current: + chunks.append(prefix + " ".join(current)) + + if not chunks and title: + chunks = [title] + + return chunks + + +# --------------------------------------------------------------------------- +# Embedding +# --------------------------------------------------------------------------- + +def embed(texts, is_query=False): + """Encode texts into L2-normalized float32 embeddings (N, 384). + + For queries, prepend the model's query prefix. + Processes in batches of 32 to limit memory usage. + """ + if not texts: + return np.empty((0, DIMS), dtype=np.float32) + + session, tokenizer = _get_session() + + if is_query: + texts = [QUERY_PREFIX + t for t in texts] + + batch_size = 32 + all_embeddings = [] + + for start in range(0, len(texts), batch_size): + batch = texts[start:start + batch_size] + encodings = tokenizer.encode_batch(batch) + input_ids = np.array([e.ids for e in encodings], dtype=np.int64) + attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64) + token_type_ids = np.zeros_like(input_ids) + + outputs = session.run( + None, + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids, + }, + ) + emb = outputs[0][:, 0, :] + all_embeddings.append(emb) + + embeddings = np.concatenate(all_embeddings, axis=0) + norms = np.linalg.norm(embeddings, axis=1, keepdims=True) + norms = np.maximum(norms, 1e-12) + embeddings = embeddings / norms + return _maybe_compress(embeddings.astype(np.float32)) + + +def _maybe_compress(embeddings): + """Compress embeddings to float16 if compression is enabled.""" + try: + from db import get_setting + if get_setting("compress_embeddings", "0") == "1": + return embeddings.astype(np.float16) + except Exception: + pass + return embeddings + + +def _decompress(embeddings): + """Decompress float16 embeddings to float32 if needed.""" + if embeddings.dtype == np.float16: + return embeddings.astype(np.float32) + return embeddings + + +def _blob_to_vec(buf): + """Decode a stored embedding blob to a float32 vector, inferring dtype from length.""" + if len(buf) == DIMS * 2: + return np.frombuffer(buf, dtype=np.float16).astype(np.float32) + return np.frombuffer(buf, dtype=np.float32) + + +# --------------------------------------------------------------------------- +# HNSW index management +# --------------------------------------------------------------------------- + +BATCH_SIZE = 50000 + +def build_index(db=None): + """Load all embeddings from chunks table and build HNSW index in batches.""" + import hnswlib + global _hnsw_index, _hnsw_ids + + from db import get_db, return_db + own_db = db is None + if own_db: + db = get_db() + + try: + total = db.execute("SELECT COUNT(*) FROM chunks").fetchone()[0] + if total == 0: + with _hnsw_lock: + _hnsw_index = None + _hnsw_ids = [] + return + + all_ids = [] + all_embeddings = [] + + for offset in range(0, total, BATCH_SIZE): + rows = db.execute( + "SELECT id, embedding FROM chunks ORDER BY id LIMIT ? OFFSET ?", + (BATCH_SIZE, offset), + ).fetchall() + for r in rows: + emb = _blob_to_vec(r["embedding"]) + all_ids.append(r["id"]) + all_embeddings.append(emb) + finally: + if own_db: + return_db(db) + + if not all_ids: + with _hnsw_lock: + _hnsw_index = None + _hnsw_ids = [] + return + + matrix = np.stack(all_embeddings) + n = len(all_ids) + ids = all_ids + + index = hnswlib.Index(space="cosine", dim=DIMS) + index.init_index(max_elements=max(n, 1024), ef_construction=200, M=16) + index.add_items(matrix, list(range(n))) + index.set_ef(50) + + with _hnsw_lock: + _hnsw_index = index + _hnsw_ids = ids + + +def _add_to_index(chunk_ids, embeddings_matrix): + """Add new embeddings to the live HNSW index.""" + import hnswlib + global _hnsw_index, _hnsw_ids + + with _hnsw_lock: + if _hnsw_index is None: + index = hnswlib.Index(space="cosine", dim=DIMS) + index.init_index(max_elements=1024, ef_construction=200, M=16) + index.set_ef(50) + _hnsw_index = index + _hnsw_ids = [] + + current_max = _hnsw_index.get_max_elements() + needed = len(_hnsw_ids) + len(chunk_ids) + if needed > current_max: + _hnsw_index.resize_index(max(needed * 2, current_max * 2)) + + labels = list(range(len(_hnsw_ids), len(_hnsw_ids) + len(chunk_ids))) + _hnsw_index.add_items(embeddings_matrix, labels) + _hnsw_ids.extend(chunk_ids) + + +# --------------------------------------------------------------------------- +# Store embeddings for pages +# --------------------------------------------------------------------------- + +def store_embeddings(page_id, title, body, db): + """Chunk, embed, and store embeddings for a page. Adds to HNSW index.""" + chunks = chunk_text(title, body) + if not chunks: + return + + embeddings_matrix = embed(chunks) + embeddings_matrix = _decompress(embeddings_matrix) + + db.execute("DELETE FROM chunks WHERE page_id = ?", (page_id,)) + + new_ids = [] + for i, (text, emb) in enumerate(zip(chunks, embeddings_matrix)): + cursor = db.execute( + "INSERT INTO chunks (page_id, remote_page_id, chunk_index, chunk_text, embedding) " + "VALUES (?, NULL, ?, ?, ?)", + (page_id, i, text, emb.tobytes()), + ) + new_ids.append(cursor.lastrowid) + db.commit() + + _add_to_index(new_ids, embeddings_matrix) + + +def store_remote_embeddings(remote_page_id, title, note, db): + """Store a single embedding for a remote page (title + note).""" + text = f"{title}: {note}" if note else (title or "") + if not text.strip(): + return + + embeddings_matrix = embed([text]) + embeddings_matrix = _decompress(embeddings_matrix) + + db.execute("DELETE FROM chunks WHERE remote_page_id = ?", (remote_page_id,)) + cursor = db.execute( + "INSERT INTO chunks (page_id, remote_page_id, chunk_index, chunk_text, embedding) " + "VALUES (NULL, ?, 0, ?, ?)", + (remote_page_id, text, embeddings_matrix[0].tobytes()), + ) + db.commit() + + _add_to_index([cursor.lastrowid], embeddings_matrix) + + +# --------------------------------------------------------------------------- +# Search +# --------------------------------------------------------------------------- + +def semantic_search(query_text, limit=100, db=None): + """Search for pages by semantic similarity. + + Returns: [(page_id, score, best_chunk_text), ...] sorted by score desc. + Groups by page_id, taking the max chunk score per page. + """ + if _hnsw_index is None or not _hnsw_ids: + return [] + + query_emb = embed([query_text], is_query=True) + + with _hnsw_lock: + if _hnsw_index is None or not _hnsw_ids: + return [] + k = min(limit * 3, len(_hnsw_ids)) # oversample to account for grouping + if k == 0: + return [] + labels, distances = _hnsw_index.knn_query(query_emb, k=k) + + # Map HNSW labels back to chunk IDs + chunk_ids = [_hnsw_ids[int(lbl)] for lbl in labels[0]] + # cosine distance -> similarity: hnswlib returns 1-cosine for "cosine" space + scores = [1.0 - float(d) for d in distances[0]] + + # Fetch chunk details from DB + from db import get_db, return_db + own_db = db is None + if own_db: + db = get_db() + try: + placeholders = ",".join("?" * len(chunk_ids)) + rows = db.execute( + f"SELECT id, page_id, chunk_text FROM chunks WHERE id IN ({placeholders})", + chunk_ids, + ).fetchall() + finally: + if own_db: + return_db(db) + + chunk_map = {r["id"]: r for r in rows} + + # Group by page_id, keep best score and chunk text per page + page_best = {} # page_id -> (score, chunk_text) + for cid, score in zip(chunk_ids, scores): + chunk = chunk_map.get(cid) + if not chunk or chunk["page_id"] is None: + continue + pid = chunk["page_id"] + if pid not in page_best or score > page_best[pid][0]: + page_best[pid] = (score, chunk["chunk_text"]) + + results = [(pid, score, text) for pid, (score, text) in page_best.items()] + results.sort(key=lambda x: x[1], reverse=True) + return results[:limit] + + +def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=False): + """Merge BM25 and semantic results via RRF, optionally rerank with cross-encoder. + + Default (two-stage): BM25 + semantic fused via RRF. + With use_reranker=True (three-stage): rerank top 20 with cross-encoder. + + Returns: [(page_id, best_chunk_text), ...] in ranked order. + """ + k = 60 # RRF constant + + sem_results = semantic_search(query_text, limit=100, db=db) + + best_chunks = {} # page_id -> chunk_text + for _rank, (pid, _score, chunk_text) in enumerate(sem_results): + if pid not in best_chunks: + best_chunks[pid] = chunk_text + + # When BM25 has no hits, use raw semantic similarity scores directly + # (RRF rank positions distort nearly-equal scores) + if not bm25_ranked_ids: + fused_ids = [(pid, score) for pid, score, _ in sem_results] + else: + rrf_scores = {} + for rank, pid in enumerate(bm25_ranked_ids): + rrf_scores[pid] = rrf_scores.get(pid, 0) + 1.0 / (k + rank + 1) + for rank, (pid, _score, chunk_text) in enumerate(sem_results): + rrf_scores[pid] = rrf_scores.get(pid, 0) + 1.0 / (k + rank + 1) + fused_ids = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True) + + fused = fused_ids + all_ids = [pid for pid, _ in fused] + + if not all_ids: + return [] + + if not use_reranker: + return [(pid, best_chunks.get(pid, "")) for pid in all_ids[:limit]] + + # --- Rerank top 20, append next 10 from RRF order --- + rerank_ids = all_ids[:20] + tail_ids = all_ids[20:30] + + from db import get_db, return_db + own_db = db is None + if own_db: + db = get_db() + try: + placeholders = ",".join("?" * len(rerank_ids)) + rows = db.execute( + f"SELECT id, title, body FROM pages WHERE id IN ({placeholders})", + rerank_ids, + ).fetchall() + finally: + if own_db: + return_db(db) + + page_map = {r["id"]: r for r in rows} + + doc_texts = [] + ordered_ids = [] + for pid in rerank_ids: + page = page_map.get(pid) + if not page: + continue + chunk = best_chunks.get(pid, "") + body_preview = chunk[:200] if chunk else page["body"][:200] + doc = f"{page['title']}. {body_preview}" + doc_texts.append(doc) + ordered_ids.append(pid) + + if not doc_texts: + return [] + + try: + reranked = rerank(query_text, doc_texts, limit=20) + results = [(ordered_ids[idx], best_chunks.get(ordered_ids[idx], "")) for idx, _score in reranked] + except Exception: + results = [(pid, best_chunks.get(pid, "")) for pid in ordered_ids[:20]] + + # Append next 10 from RRF order (no reranking) + reranked_set = {pid for pid, _ in results} + for pid in tail_ids: + if pid not in reranked_set: + results.append((pid, best_chunks.get(pid, ""))) + + return results[:30] + + +# --------------------------------------------------------------------------- +# Reindex +# --------------------------------------------------------------------------- + +def reindex_all(db=None, progress_callback=None): + """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index.""" + from db import get_db, return_db + own_db = db is None + if own_db: + db = get_db() + try: + # Clear existing chunks so everything is regenerated + db.execute("DELETE FROM chunks") + db.commit() + + rows = db.execute( + "SELECT p.id, p.title, p.body, p.summary FROM pages p" + ).fetchall() + + total = len(rows) + for i, row in enumerate(rows): + store_embeddings(row["id"], row["title"], row["body"], db) + if progress_callback: + progress_callback(i + 1, total) + + # Also handle remote pages + remote_rows = db.execute( + "SELECT rp.id, rp.title, rp.note FROM remote_pages rp" + ).fetchall() + + for rp in remote_rows: + store_remote_embeddings(rp["id"], rp["title"], rp["note"], db) + finally: + if own_db: + return_db(db) + + build_index(db) diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..1f49fcb --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Generate Reticulum config with optional TCP peer +# Set RNS_TCP_HOST and RNS_TCP_PORT env vars to connect to a remote instance + +CONFIG_DIR="/data/.reticulum" +CONFIG_FILE="$CONFIG_DIR/config" + +mkdir -p "$CONFIG_DIR" + +if [ ! -f "$CONFIG_FILE" ]; then + cat > "$CONFIG_FILE" <{" | ".join(parts)}
' + + +# --- Tag helpers --- + + +def _get_page_tags(page_id, db=None): + close = False + if db is None: + db = get_db() + close = True + rows = db.execute( + "SELECT t.name FROM tags t JOIN page_tags pt ON t.id = pt.tag_id " + "WHERE pt.page_id = ? ORDER BY t.name", (page_id,) + ).fetchall() + if close: + return_db(db) + return [r["name"] for r in rows] + + +def _set_page_tags(page_id, tag_string, db=None): + close = False + if db is None: + db = get_db() + close = True + db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,)) + for name in (t.strip().lower() for t in tag_string.split(",") if t.strip()): + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,)) + tag_id = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()["id"] + db.execute("INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", (page_id, tag_id)) + if close: + db.commit() + return_db(db) + + +def _cleanup_orphaned_tags(db): + """Delete tags that have no page associations.""" + db.execute("DELETE FROM tags WHERE id NOT IN (SELECT DISTINCT tag_id FROM page_tags)") + + +# --- Route handlers --- + + +def handle_search(query): + q = query.get("q", [""])[0].strip() + page = _paginate(query) + offset = (page - 1) * PER_PAGE + db = get_db() + try: + count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + name = get_site_name() + + result_html = "" + trusted_html = "" + if q: + # BM25 keyword search with column weights: title=10, body=1, url=5, note=3 + try: + fts_q = _sanitize_fts_query(q) + bm25_rows = db.execute( + "SELECT p.id, p.url, p.title, p.body, p.note " + "FROM pages_fts f JOIN pages p ON f.rowid = p.id " + "WHERE pages_fts MATCH ? " + "ORDER BY bm25(pages_fts, 10.0, 1.0, 5.0, 3.0) LIMIT 100", + (fts_q,), + ).fetchall() + except Exception: + bm25_rows = [] + + # Hybrid search: merge BM25 + semantic via RRF + bm25_ids = [r["id"] for r in bm25_rows] + chunk_snippets = {} # page_id -> best chunk text + if get_setting("semantic_search", "0") == "1": + try: + from embeddings import hybrid_search + use_reranker = get_setting("use_reranker", "1") == "1" + fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker) + fused_ids = [pid for pid, _ in fused] + chunk_snippets = {pid: text for pid, text in fused if text} + except Exception: + fused_ids = bm25_ids + else: + fused_ids = bm25_ids + + total_results = len(fused_ids) + page_ids = fused_ids[offset:offset + PER_PAGE] + + if page_ids: + # Fetch rows in fused order + placeholders = ",".join("?" * len(page_ids)) + all_rows = db.execute( + f"SELECT id, url, title, body, note, summary FROM pages WHERE id IN ({placeholders})", + page_ids, + ).fetchall() + row_map = {r["id"]: r for r in all_rows} + rows = [row_map[pid] for pid in page_ids if pid in row_map] + else: + rows = [] + + if rows: + for r in rows: + note_html = "" + if r["note"]: + note_html = f'No results in your index.
" + + # search all linked pages from trusted sites + words = q.lower().split() + all_links = db.execute( + "SELECT l.url, l.label, p.title AS source_title " + "FROM links l JOIN pages p ON l.page_id = p.id", + ).fetchall() + indexed_urls = set(r["url"] for r in rows) if rows else set() + seen = set() + trusted = [] + for l in all_links: + if l["url"] in indexed_urls or l["url"] in seen: + continue + if any(w in l["label"].lower() for w in words): + seen.add(l["url"]) + trusted.append(l) + if len(trusted) >= 20: + break + + if trusted: + items = "" + for l in trusted: + items += ( + f'Your index is empty.
' + 'tinyweb is a personal search engine for pages you save. ' + 'The index stays on your machine; so does every search.
' + 'From here: add a page, ' + 'get the bookmarklet, or ' + 'subscribe to another instance.
' + 'Subscribe to a friend's TinyWeb instance to sync their index
" + f'" + f"" + f"{msg}
" + f'back' + ) + url_value = f'value="{esc(prefill_url)}" ' if prefill_url else "" + return _respond( + f"Add a site to your index
" + f'" + f"{msg}
" + f'back' + ) + + +def handle_add_submit(body): + input_type = body.get("input_type", ["url"])[0] + url = body.get("url", [""])[0].strip() + reticulum_dest = body.get("reticulum_dest", [""])[0].strip().replace("<", "").replace(">", "") + note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + + if input_type == "url": + if not url: + return handle_add_form("URL is required.") + url = clean_url(url) + if not url.startswith(("http://", "https://")): + return handle_add_form("URL must start with http:// or https://") + else: + if not reticulum_dest: + return handle_add_form("Reticulum destination hash is required.") + if len(reticulum_dest) != 32 or not all(c in "0123456789abcdefABCDEF" for c in reticulum_dest): + return handle_add_form("Invalid reticulum destination hash. Must be 32 hex characters.") + url = f"reticulum:{reticulum_dest}" + + try: + title = index_url(url, note, reticulum_dest if reticulum_dest else "") + if tags: + db = get_db() + try: + row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() + if row: + _set_page_tags(row["id"], tags, db) + db.commit() + finally: + return_db(db) + + return handle_add_form(f'Indexed: {esc(url)}') + + except ValueError as e: + return handle_add_form(f"Error: {esc(str(e))}") + + except Exception as e: + error_msg = str(e).lower() + # Check if it's a block response + if "block" in error_msg or "cloudflare" in error_msg or "403" in error_msg: + # Show manual entry form for blocked sites + return _respond( + f"{esc(url)} blocks automated access. " + f"You can still save it manually:
" + f'" + f'back' + ) + return handle_add_form(f"Error: could not fetch or index that URL. {esc(str(e)[:100])}") + + +def handle_add_manual_submit(body): + url = clean_url(body.get("url", [""])[0].strip()) + note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + manual_title = body.get("manual_title", [""])[0].strip() + manual_desc = body.get("manual_description", [""])[0].strip() + + if not url: + return handle_add_form("URL is required.") + + if not manual_title: + return handle_add_form("Title is required for manual entry.") + + db = get_db() + try: + now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + + db.execute( + "INSERT INTO pages (url, title, body, note, last_modified, summary) VALUES (?, ?, ?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, " + "note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary", + (url, manual_title, manual_desc, note, now, manual_desc[:200]), + ) + + # Get the page ID + page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0] + + # Add tags if provided + if tags: + _set_page_tags(page_id, tags, db) + + db.commit() + + # Generate embeddings for this page (if semantic search is enabled) + if get_setting("semantic_search", "0") == "1": + try: + from embeddings import store_embeddings + # Pass the page_id, title, description, and db connection + store_embeddings(page_id, manual_title, manual_desc, db) + db.commit() + except Exception as e: + # Log error but don't fail the whole operation + print(f"Error generating embeddings: {e}") + + return handle_add_form(f'Added manually: {esc(manual_title)}') + finally: + return_db(db) + + +def handle_pages(query=None): + msg = query.get("msg", [""])[0] if query else "" + msg_html = f'{esc(msg)}
' if msg else "" + page = _paginate(query or {}) + offset = (page - 1) * BROWSE_PER_PAGE + db = get_db() + try: + total = db.execute("SELECT count(*) FROM pages").fetchone()[0] + rows = db.execute( + "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?", + (BROWSE_PER_PAGE, offset), + ).fetchall() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tags_html = "" + if tags: + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + tags_html = f' {tag_links}' + items += ( + f'Remove the following {n} page{'' if n == 1 else 's'}?
" + f"{esc(row['title'])}
"
+ f"{esc(row['url'])}
{msg}
" + f'back' + ) + + +def handle_edit_submit(page_id, body): + title = body.get("title", [""])[0].strip() + summary = body.get("summary", [""])[0].strip() + note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + + db = get_db() + try: + db.execute( + "UPDATE pages SET title = ?, summary = ?, note = ? WHERE id = ?", + (title, summary, note, page_id) + ) + + _set_page_tags(page_id, tags, db) + _cleanup_orphaned_tags(db) + + db.commit() + + finally: + return_db(db) + + return _redirect("/pages") + + +def handle_delete_confirm(page_id): + db = get_db() + try: + row = db.execute("SELECT id, url, title FROM pages WHERE id = ?", (page_id,)).fetchone() + finally: + return_db(db) + if not row: + return _error(404) + return _respond( + f"Remove {esc(row['title'])}
"
+ f"{esc(row['url'])}
Paste the contents of a tinyweb export file (JSON).
" + f'" + f"{msg}
" + f'back' + ) + + +def handle_import_submit(body): + raw = body.get("data", [""])[0].strip() + if not raw: + return handle_import_form("Paste JSON data.") + try: + data = json.loads(raw) + except json.JSONDecodeError: + return handle_import_form("Invalid JSON.") + if not isinstance(data, list): + return handle_import_form("Expected a JSON array.") + + MAX_IMPORT = 100 + if len(data) > MAX_IMPORT: + return handle_import_form(f"Too many entries. Maximum is {MAX_IMPORT}.") + + imported = 0 + errors = 0 + for entry in data: + url = entry.get("url", "").strip() + note = entry.get("note", "").strip() + if not url: + continue + try: + index_url(url, note) + imported += 1 + except Exception: + errors += 1 + + return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") + + +def handle_style_form(msg=""): + template = get_setting("custom_template") or DEFAULT_TEMPLATE + name = get_site_name() + sharing = get_setting("sharing_enabled", "0") + checked = " checked" if sharing == "1" else "" + sharing_mode = get_setting("sharing_mode", "exclude_private") + forum = get_setting("forum_enabled", "0") + forum_checked = " checked" if forum == "1" else "" + exclude_checked = " checked" if sharing_mode != "require_public" else "" + require_checked = " checked" if sharing_mode == "require_public" else "" + shared_count = _count_shared_pages() + semantic = get_setting("semantic_search", "0") + semantic_checked = " checked" if semantic == "1" else "" + reranker = get_setting("use_reranker", "0") + reranker_checked = " checked" if reranker == "1" else "" + disabled = "" if semantic == "1" else " disabled" + dimmed = ' style="opacity:0.4"' if semantic != "1" else "" + tcp_enabled = get_setting("tcp_enabled", "1") + tcp_checked = " checked" if tcp_enabled == "1" else "" + tcp_disabled = "" if tcp_enabled == "1" else " disabled" + transport_host = get_setting("transport_host", "reticulum.derickphan.com") + transport_port = get_setting("transport_port", "4242") + compress = get_setting("compress_embeddings", "0") + compress_checked = " checked" if compress == "1" else "" + lora_enabled = get_setting("lora_enabled", "0") + lora_checked = " checked" if lora_enabled == "1" else "" + lora_disabled = "" if lora_enabled == "1" else " disabled" + lora_dimmed = ' style="opacity:0.4"' if lora_enabled != "1" else "" + lora_port = get_setting("lora_port", "") + lora_frequency = get_setting("lora_frequency", "867200000") + lora_bandwidth = get_setting("lora_bandwidth", "125000") + lora_txpower = get_setting("lora_txpower", "7") + lora_sf = get_setting("lora_sf", "8") + lora_cr = get_setting("lora_cr", "5") + if forum_plugin is not None: + forum_section = ( + f"tinyweb-forum — "
+ f'more info.Drag this link to your bookmarks bar. Click it on any page to index it instantly.
" + f'' + f"{msg}
" + f'back', + use_default=True, + ) + + +def handle_style_submit(body): + template = body.get("template", [""])[0].replace("\r\n", "\n").replace("\r", "\n") + name = body.get("site_name", ["tinyweb"])[0].strip() + sharing = "1" if body.get("sharing_enabled") else "0" + sharing_mode = body.get("sharing_mode", ["exclude_private"])[0] + if sharing_mode not in ("exclude_private", "require_public"): + sharing_mode = "exclude_private" + set_setting("sharing_mode", sharing_mode) + semantic = "1" if body.get("semantic_search") else "0" + reranker = "1" if body.get("use_reranker") else "0" + compress = "1" if body.get("compress_embeddings") else "0" + tcp_enabled = "1" if body.get("tcp_enabled") else "0" + transport_host = body.get("transport_host", [""])[0].strip() + transport_port = body.get("transport_port", [""])[0].strip() + set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "") + set_setting("site_name", name or "tinyweb") + set_setting("sharing_enabled", sharing) + set_setting("semantic_search", semantic) + set_setting("use_reranker", reranker) + set_setting("compress_embeddings", compress) + set_setting("tcp_enabled", tcp_enabled) + if transport_host: + set_setting("transport_host", transport_host) + if transport_port: + set_setting("transport_port", transport_port) + lora_enabled = "1" if body.get("lora_enabled") else "0" + set_setting("lora_enabled", lora_enabled) + set_setting("lora_port", body.get("lora_port", [""])[0].strip()) + set_setting("lora_frequency", body.get("lora_frequency", ["867200000"])[0].strip()) + set_setting("lora_bandwidth", body.get("lora_bandwidth", ["125000"])[0].strip()) + set_setting("lora_txpower", body.get("lora_txpower", ["7"])[0].strip()) + set_setting("lora_sf", body.get("lora_sf", ["8"])[0].strip()) + set_setting("lora_cr", body.get("lora_cr", ["5"])[0].strip()) + forum_enabled = "1" if body.get("forum_enabled") else "0" + current_forum = get_setting("forum_enabled", "0") + if forum_enabled != current_forum: + if forum_enabled == "1" and forum_plugin is None: + return handle_style_form( + "Forum plugin not installed. Run: pip install tinyweb-forum" + ) + if forum_enabled == "1": + forum_plugin.enable() + try: + forum_plugin.fdb.set_setting("forum_enabled", "1") + except Exception: + pass + else: + forum_plugin.disable() + try: + forum_plugin.fdb.set_setting("forum_enabled", "0") + except Exception: + pass + set_setting("forum_enabled", forum_enabled) + templates_mod.FORUM_ENABLED = (forum_enabled == "1") + return handle_style_form("Saved. Restart TinyWeb for mesh network changes to take effect.") + + +def handle_about(): + name = get_site_name() + dest_hash = get_setting("dest_hash") + sharing = get_setting("sharing_enabled", "0") == "1" + db = get_db() + try: + page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + tag_count = db.execute("SELECT count(DISTINCT tag_id) FROM page_tags").fetchone()[0] + sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0] + finally: + return_db(db) + + sharing_html = ( + 'This instance shares its index publicly. Subscribe to join the network.
' + if sharing else + 'This instance is private.
' + ) + + hash_html = "" + if dest_hash: + hash_html = ( + f'To subscribe to this instance, add this destination hash in your TinyWeb:
' + f'{esc(dest_hash)}'
+ )
+
+ return _respond(
+ f'A personal search engine, built for the slow web.
' + f'TinyWeb is about taking back the internet. No algorithms, no ads, no tracking. ' + f'Just human-curated pages shared freely across a mesh network.
' + f'Everything is stored locally under ~/.tinyweb/:
tinyweb_identity — your permanent mesh identity. '
+ f'If you lose this file, your destination hash changes and subscribers '
+ f'have to re-subscribe to the new one.index.db — your full reading history: every page, '
+ f'note, tag, and synced remote page.models/ — the semantic search model if you enabled it '
+ f'(redownloadable, safe to delete).Back up ~/.tinyweb/ periodically. '
+ f'Copying the whole directory to another device preserves your identity and index together. '
+ f'The export page gives you a JSON dump of pages only — '
+ f'it does not preserve your identity or subscription state, so it is a migration aid, '
+ f'not a substitute for a full backup.
The slow web is a movement for intentionality over speed, ' + f'human curation over algorithmic feeds, privacy over surveillance, ' + f'and community over corporations. Every page in this index was saved by a person ' + f'because they found it valuable — not because an algorithm told them to click.
' + f'No tags yet. Add tags when saving or editing pages.
" + f'back' + ) + + +def handle_tag_browse(tag_name, query=None): + page = _paginate(query or {}) + offset = (page - 1) * BROWSE_PER_PAGE + db = get_db() + try: + total = db.execute( + "SELECT count(*) FROM page_tags pt JOIN tags t ON t.id = pt.tag_id WHERE t.name = ?", + (tag_name,), + ).fetchone()[0] + rows = db.execute( + "SELECT p.id, p.url, p.title, p.note FROM pages p " + "JOIN page_tags pt ON p.id = pt.page_id " + "JOIN tags t ON t.id = pt.tag_id " + "WHERE t.name = ? ORDER BY p.id DESC LIMIT ? OFFSET ?", + (tag_name, BROWSE_PER_PAGE, offset), + ).fetchall() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + items += ( + f'{total} page(s)
' + f'public"
+ if mode == "require_public"
+ else "all pages except those tagged private"
+ )
+ sharing_on = get_setting("sharing_enabled", "0") == "1"
+ status = (
+ 'Sharing is enabled. Subscribers see the pages listed below.
' + if sharing_on else + 'Sharing is disabled. Nothing is actually being shared right now; ' + 'this is the list that would be exposed if you enabled it.
' + ) + db = get_db() + try: + sites = _shared_sites(db) + finally: + return_db(db) + if not sites: + body = ( + "Rule: {mode_label}.
" + f"{status}" + "No pages match the current rule.
" + '' + ) + return _respond(body) + rows = "" + for s in sites: + tags_html = "" + if s["tags"]: + tags_html = " " + " ".join(f"[{esc(t)}]" for t in s["tags"]) + note_html = f' — {esc(s["note"])}' if s["note"] else "" + rows += ( + f'Rule: {mode_label}.
" + f"{status}" + f"{len(sites)} page(s) visible to subscribers.
" + f"{msg}
' + f'{len(sites)} site(s) available, {new_count} new
' + f'' + f'Semantic search is disabled. Enable it in settings to use embeddings.
" + f'' + ) + db = get_db() + try: + total_pages = db.execute("SELECT count(*) FROM pages").fetchone()[0] + pages_with_chunks = db.execute( + "SELECT count(DISTINCT page_id) FROM chunks WHERE page_id IS NOT NULL" + ).fetchone()[0] + finally: + return_db(db) + progress = get_setting("reindex_progress", "") + status_html = "" + if progress: + status_html = f'' + elif _reindex_thread and _reindex_thread.is_alive(): + status_html = '' + return _respond( + f"{pages_with_chunks} of {total_pages} pages have embeddings.
" + f'{status_html}' + f'' + f'' + ) + + +def handle_reindex_submit(body): + global _reindex_thread + if _reindex_thread and _reindex_thread.is_alive(): + return handle_reindex_form() + + def _run(): + try: + from embeddings import reindex_all + def progress(current, total): + set_setting("reindex_progress", f"{current}/{total}") + reindex_all(progress_callback=progress) + except Exception: + pass + finally: + set_setting("reindex_progress", "") + + _reindex_thread = threading.Thread(target=_run, daemon=True) + _reindex_thread.start() + return _redirect("/reindex") + + +# --- Dispatcher --- + + +def _dispatch_inner(data): + method = data.get("method", "GET") + path = data.get("path", "/") + query = data.get("query", {}) + body = data.get("body", {}) + gateway_host = data.get("gateway_host", "") + + def extract_id(prefix): + try: + return int(path[len(prefix):]) + except (ValueError, IndexError): + return None + + if method == "GET": + if path == "/": + return handle_search(query) + elif path == "/add": + action_type = query.get("type", ["index"])[0] + prefill_url = query.get("url", [""])[0].strip() + return handle_add_form( + action_type=action_type if action_type == "subscribe" else "index", + prefill_url=prefill_url, + ) + elif path == "/pages": + return handle_pages(query) + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_form(pid) if pid is not None else _error(400) + elif path.startswith("/delete/"): + pid = extract_id("/delete/") + return handle_delete_confirm(pid) if pid is not None else _error(400) + elif path == "/bookmark": + return handle_bookmark(query) + elif path == "/style": + return handle_style_form() + elif path == "/share/preview": + return handle_share_preview() + elif path == "/about": + return handle_about() + elif path == "/export": + return handle_export(query) + elif path == "/import": + return handle_import_form() + elif path == "/tags": + return handle_tags() + elif path.startswith("/tags/"): + tag_name = unquote(path[len("/tags/"):]) + return handle_tag_browse(tag_name, query) if tag_name else _error(400) + elif path == "/reindex": + return handle_reindex_form() + elif path == "/api/sites": + return handle_api_sites(query) + elif path == "/subscriptions": + return handle_subscriptions() + elif path.startswith("/subscriptions/browse/"): + sid = extract_id("/subscriptions/browse/") + return handle_subscription_browse(sid) if sid is not None else _error(400) + elif path.startswith("/forum"): + if forum_plugin and forum_plugin.is_enabled(): + return forum_plugin.handle(method, path, query, {}, data.get("cookies", {})) + return _error(404) + elif method == "POST": + if path.startswith("/forum"): + if forum_plugin and forum_plugin.is_enabled(): + return forum_plugin.handle(method, path, query, body, data.get("cookies", {})) + return _error(404) + if not _check_csrf(body): + return _respond("Invalid or missing CSRF token.
", status=403) + if path == "/add": + return handle_add_submit(body) + elif path == "/pages/bulk": + return handle_bulk_action(body) + elif path == "/add/manual": + return handle_add_manual_submit(body) + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_submit(pid, body) if pid is not None else _error(400) + elif path.startswith("/delete/"): + pid = extract_id("/delete/") + return handle_delete(pid) if pid is not None else _error(400) + elif path == "/style": + return handle_style_submit(body) + elif path == "/style/reset": + set_setting("custom_template", "") + return handle_style_form("Template reset to default.") + elif path == "/style/vacuum": + from db import vacuum_db + vacuum_db() + return handle_style_form("Database vacuumed.") + elif path == "/import": + return handle_import_submit(body) + elif path == "/reindex": + return handle_reindex_submit(body) + elif path == "/subscriptions/add": + return handle_subscription_add(body) + elif path == "/subscriptions/pick": + return handle_subscription_pick(body) + elif path.startswith("/subscriptions/sync/"): + sid = extract_id("/subscriptions/sync/") + return handle_subscription_sync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/autosync/"): + sid = extract_id("/subscriptions/autosync/") + return handle_subscription_autosync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/delete/"): + sid = extract_id("/subscriptions/delete/") + return handle_subscription_delete(sid) if sid is not None else _error(400) + elif path == "/subscriptions/syncall": + return handle_subscription_syncall() + + return _error(404) + + +def dispatch_request(data): + path = data.get("path", "/") + cookies = data.get("cookies", {}) + + # Forum handles its own CSRF — skip main CSRF to avoid cookie conflicts + if path.startswith("/forum") and forum_plugin and forum_plugin.is_enabled(): + resp = _dispatch_inner(data) + resp.setdefault("headers", {}) + resp["headers"]["X-Frame-Options"] = "DENY" + resp["headers"]["X-Content-Type-Options"] = "nosniff" + if resp.get("content_type", "").startswith("text/html"): + resp["body"] = wrap_page(resp.get("body", "")) + resp["headers"]["Content-Security-Policy"] = ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline'; " + "style-src 'self' 'unsafe-inline'; " + "img-src * data:; " + "frame-ancestors 'none'; " + "form-action 'self'; " + "base-uri 'self'" + ) + return resp + + csrf_token = cookies.get("_csrf", "") + if not csrf_token: + csrf_token = secrets.token_hex(32) + _request_local.csrf_token = csrf_token + + resp = _dispatch_inner(data) + + resp.setdefault("headers", {}) + resp["headers"]["Set-Cookie"] = f"_csrf={csrf_token}; SameSite=Strict; HttpOnly; Path=/" + resp["headers"]["X-Frame-Options"] = "DENY" + resp["headers"]["X-Content-Type-Options"] = "nosniff" + if resp.get("content_type", "").startswith("text/html"): + resp["headers"]["Content-Security-Policy"] = ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline'; " + "style-src 'self' 'unsafe-inline'; " + "img-src * data:; " + "frame-ancestors 'none'; " + "form-action 'self'; " + "base-uri 'self'" + ) + return resp diff --git a/index.db b/index.db deleted file mode 100644 index 797cd93..0000000 Binary files a/index.db and /dev/null differ diff --git a/pyinstaller.spec b/pyinstaller.spec new file mode 100644 index 0000000..431155f --- /dev/null +++ b/pyinstaller.spec @@ -0,0 +1,81 @@ +# -*- mode: python ; coding: utf-8 -*- + +import os +import sys + +block_cipher = None + +# Hidden imports that PyInstaller can't detect automatically +hiddenimports = [ + "RNS", + "RNS.Destination", + "RNS.Identity", + "RNS.Reticulum", + "onnxruntime", + "onnxruntime.capi.onnxruntime_pybind11_state", + "tokenizers", + "huggingface_hub", + "hnswlib", + "bs4", + "beautifulsoup4", + "numpy", + "requests", +] + +# Data files to include +datas = [ + ("themes", "themes"), +] + +# Exclude unnecessary modules +excludes = [ + "test", + "tests", + "tkinter", + "matplotlib", + "scipy", + "pandas", + "IPython", + "jupyter", + "notebook", +] + +a = Analysis( + ["app.py"], + pathex=[], + binaries=[], + datas=datas, + hiddenimports=hiddenimports, + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=excludes, + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False, +) + +pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) + +exe = EXE( + pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name="TinyWeb", + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, +) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..16d6cc5 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +testpaths = tests +python_files = test_*.py +filterwarnings = + ignore::DeprecationWarning diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..26b77f6 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest diff --git a/requirements.txt b/requirements.txt index 1190bd8..29da454 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,9 @@ +# forum: pip install tinyweb-forum (optional, adds URL discussion board) requests beautifulsoup4 +rns +onnxruntime +tokenizers +hnswlib +numpy +huggingface_hub diff --git a/rns_client.py b/rns_client.py new file mode 100644 index 0000000..98df406 --- /dev/null +++ b/rns_client.py @@ -0,0 +1,110 @@ +import json +import time +import RNS + +APP_NAME = "tinyweb" +ASPECTS = ["server"] + +# Two-tier timeout profiles: fast first, then slow for LoRa/multi-hop links +_TIMEOUT_TIERS = [ + {"path": 15, "link": 15, "request": 30, "poll": 0.25}, + {"path": 60, "link": 60, "request": 120, "poll": 1.0}, +] + + +def fetch_remote_sites(dest_hash_hex, since=""): + """ + Connect to a remote TinyWeb instance over Reticulum and fetch its + shared sites. Returns the response dict from /api/sites, or raises + an exception on failure. Pass `since` as ISO timestamp for delta sync. + + Uses progressive timeouts: tries fast first, then retries with longer + timeouts for slow links (LoRa, multi-hop). + """ + last_error = None + for tier in _TIMEOUT_TIERS: + try: + return _fetch(dest_hash_hex, since, tier) + except PermissionError: + raise # Don't retry permission errors + except Exception as e: + last_error = e + continue + raise ConnectionError( + f"Could not reach {dest_hash_hex} after {len(_TIMEOUT_TIERS)} attempts: {last_error}" + ) + + +def _fetch(dest_hash_hex, since, timeouts): + """Single fetch attempt with the given timeout profile.""" + dest_hash = bytes.fromhex(dest_hash_hex) + poll = timeouts["poll"] + + # Resolve path if needed + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < timeouts["path"]: + time.sleep(poll) + elapsed += poll + if not RNS.Transport.has_path(dest_hash): + raise ConnectionError( + f"Could not find path to {dest_hash_hex} ({timeouts['path']}s timeout)" + ) + + server_identity = RNS.Identity.recall(dest_hash) + if server_identity is None: + raise ConnectionError(f"Could not recall identity for {dest_hash_hex}") + + destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, + ) + + # Establish link + link = RNS.Link(destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < timeouts["link"]: + time.sleep(poll) + elapsed += poll + + if link.status != RNS.Link.ACTIVE: + raise ConnectionError( + f"Could not establish link to {dest_hash_hex} ({timeouts['link']}s timeout)" + ) + + try: + query = {"since": [since]} if since else {} + request_data = { + "method": "GET", + "path": "/api/sites", + "query": query, + "body": {}, + "gateway_host": "", + } + + req_timeout = timeouts["request"] + receipt = link.request("/tinyweb", data=request_data, timeout=req_timeout) + + elapsed = 0 + done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done and elapsed < req_timeout: + time.sleep(poll) + elapsed += poll + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + if resp["status"] == 403: + raise PermissionError("That instance has sharing disabled.") + if resp["status"] != 200: + raise ConnectionError(f"Remote returned status {resp['status']}") + return json.loads(resp["body"]) + else: + raise ConnectionError( + f"Request failed or timed out ({req_timeout}s timeout)" + ) + finally: + link.teardown() diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..d8df32c --- /dev/null +++ b/start.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exec /nix/store/vhgmnrmvvfdiw0kc2xz8px7rvg60lszc-python3-3.13.12-env/bin/python /home/lichenblankie/apps/tinyweb/app.py --bind 0.0.0.0 \ No newline at end of file diff --git a/templates.py b/templates.py new file mode 100644 index 0000000..ff533cc --- /dev/null +++ b/templates.py @@ -0,0 +1,37 @@ +import html +from db import get_setting + +FORUM_ENABLED = False + +def esc(s): + return html.escape(str(s)) + + +DEFAULT_TEMPLATE = "\n\n\n\n\n\n{{content}}\n\n" + + +def _default_template(): + name = esc(get_setting("site_name", "tinyweb")) + forum_link = ' | forum' if FORUM_ENABLED else "" + return ( + '\n\n\n\n\n\n' + f'{name}' + ' | search | browse' + ' | tags | subscriptions' + f'{forum_link}' + ' | customize | about
\n' + "body content
+ """ + title, body, links, meta = _fetch_with_html(monkeypatch, "https://example.com/", html) + assert meta == "the real description" + + +def test_og_description_fallback(monkeypatch): + """When there's no , og:description wins.""" + html = """ + + +body
+ """ + _, _, _, meta = _fetch_with_html(monkeypatch, "https://example.com/", html) + assert meta == "open graph fallback" diff --git a/tests/test_pagination.py b/tests/test_pagination.py new file mode 100644 index 0000000..05077e0 --- /dev/null +++ b/tests/test_pagination.py @@ -0,0 +1,58 @@ +"""Tests for `_paginate` and `_page_nav`.""" +from handlers import _paginate, _page_nav, PER_PAGE + + +def test_paginate_default_is_one(): + assert _paginate({}) == 1 + + +def test_paginate_reads_query_string(): + assert _paginate({"p": ["3"]}) == 3 + + +def test_paginate_clamps_to_one(): + assert _paginate({"p": ["0"]}) == 1 + assert _paginate({"p": ["-5"]}) == 1 + + +def test_paginate_handles_bad_input(): + assert _paginate({"p": ["not-a-number"]}) == 1 + assert _paginate({"p": []}) == 1 + + +def test_paginate_custom_key(): + assert _paginate({"batch": ["7"]}, key="batch") == 7 + + +def test_page_nav_empty_when_single_page(): + assert _page_nav(1, PER_PAGE, "/?q=foo") == "" + assert _page_nav(1, 0, "/?q=foo") == "" + + +def test_page_nav_shows_next_on_first_page(): + out = _page_nav(1, PER_PAGE * 3, "/?q=foo") + assert "next" in out + assert "prev" not in out + assert "page 1 of 3" in out + + +def test_page_nav_shows_both_in_middle(): + out = _page_nav(2, PER_PAGE * 3, "/?q=foo") + assert "next" in out + assert "prev" in out + + +def test_page_nav_shows_prev_on_last_page(): + out = _page_nav(3, PER_PAGE * 3, "/?q=foo") + assert "next" not in out + assert "prev" in out + assert "page 3 of 3" in out + + +def test_page_nav_handles_query_string_separator(): + # when base_url already has ?, pagination links must use & + out = _page_nav(1, PER_PAGE * 2, "/?q=foo") + assert "&p=2" in out + # when base_url has no ?, pagination links use ? + out = _page_nav(1, PER_PAGE * 2, "/pages") + assert "?p=2" in out diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..f8a5df7 --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,107 @@ +"""Aggregator of regression tests tied to specific bug-fix commits. + +Each test here guards against a specific bug that was once shipped. Running +just this file gives a one-line-per-bug audit: + + pytest tests/test_regressions.py -v + +The test bodies are intentionally small; for the exhaustive behavior of each +module, see the topical test files (test_fts_sanitizer.py, test_url_cleanup.py, +etc.). This file's job is to make the bug catalog scannable. +""" +import socket +from unittest.mock import patch + +import pytest + +import app as app_module +import db as db_module +import handlers as handlers_module +from conftest import patch_dns_fail, patch_dns_ok +from db import clean_url +from handlers import _sanitize_fts_query, handle_bulk_action + + +def test_6ffd38d_clean_url_preserves_www_when_bare_domain_fails(monkeypatch): + """6ffd38d: `clean_url` used to strip `www.` unconditionally; for sites that + only serve at `www.`, this produced unreachable clean URLs.""" + patch_dns_fail(monkeypatch) + assert clean_url("https://www.example.com/page") == "https://www.example.com/page" + + +def test_1bc695f_fts_sanitizer_strips_colon(): + """1bc695f: FTS5 colon is a column filter — must not appear in sanitized output.""" + assert ":" not in _sanitize_fts_query("title:secret body:exposed") + + +@pytest.mark.parametrize("op", ["AND", "OR", "NOT", "NEAR"]) +def test_1bc695f_fts_sanitizer_drops_operator_words(op): + """1bc695f: operator words (AND/OR/NOT/NEAR) would be interpreted as FTS5 + operators if they landed on the unquoted last token.""" + out = _sanitize_fts_query(f"foo {op} bar") + # operator itself should not appear in the output + tokens = out.replace('"', '').split() + assert op not in [t.rstrip("*") for t in tokens] + + +def test_1bc695f_gateway_rejects_oversize_body(): + """1bc695f: 16 MiB body-size cap prevents memory-exhaustion DoS.""" + from tests.test_gateway_limits import FakeGatewayHandler + from gateway import MAX_BODY_SIZE + h = FakeGatewayHandler( + path="/add", method="POST", + headers={"Content-Length": str(MAX_BODY_SIZE + 1)}, + ) + h._forward("POST") + assert h._captured["error"] and h._captured["error"][0] == 413 + + +def test_1bc695f_mesh_rejects_non_whitelisted_paths(): + """1bc695f: Reticulum callers are limited to GET /api/sites; CSRF cannot + authenticate mesh callers.""" + resp = app_module.rns_request_handler( + path="/tinyweb", + data={"method": "POST", "path": "/add", "query": {}, "body": {}, "gateway_host": ""}, + request_id="x", link_id="y", remote_identity=None, requested_at=0, + ) + assert resp["status"] == 403 + + +def test_1bc695f_pool_returns_clean_connection(temp_db, monkeypatch): + """1bc695f: uncommitted transactions on a pooled connection used to leak + into the next consumer.""" + from db import get_db, return_db + db = get_db() + db.execute( + "INSERT INTO pages (url, title, body) VALUES (?, ?, ?)", + ("https://leak.example.com/", "should not persist", "body"), + ) + return_db(db) # no commit + db2 = get_db() + try: + urls = {r["url"] for r in db2.execute("SELECT url FROM pages").fetchall()} + finally: + return_db(db2) + assert "https://leak.example.com/" not in urls + + +def test_8dffd8c_bulk_delete_requires_confirmation(seeded_db, csrf_session): + """8dffd8c: bulk delete without confirmed=1 must render a confirm page + instead of deleting — the JS confirm on /pages is a first-line filter only.""" + from db import get_db, return_db + db = get_db() + try: + pid = db.execute("SELECT id FROM pages LIMIT 1").fetchone()["id"] + count_before = db.execute("SELECT count(*) FROM pages").fetchone()[0] + finally: + return_db(db) + + resp = handle_bulk_action({"ids": [str(pid)], "action": ["delete"]}) + assert "confirm delete" in resp["body"].lower() + + db = get_db() + try: + count_after = db.execute("SELECT count(*) FROM pages").fetchone()[0] + finally: + return_db(db) + assert count_before == count_after, "bulk delete ran without confirmation" diff --git a/tests/test_sharing_logic.py b/tests/test_sharing_logic.py new file mode 100644 index 0000000..c9c06d4 --- /dev/null +++ b/tests/test_sharing_logic.py @@ -0,0 +1,38 @@ +"""Tests for `_page_is_shared`. + +This function decides whether a page is exposed over Reticulum to +subscribers. Getting it wrong means either a privacy leak or silently +hiding pages the user meant to share — both are worth a regression net. +""" +import pytest + +from handlers import _page_is_shared + + +@pytest.mark.parametrize("mode", ["exclude_private", "require_public"]) +def test_private_tag_always_excludes(mode): + """`private` tag overrides every mode — the most important invariant.""" + assert _page_is_shared(["private"], mode) is False + assert _page_is_shared(["public", "private"], mode) is False + + +def test_exclude_private_defaults_to_shared(): + assert _page_is_shared([], "exclude_private") is True + assert _page_is_shared(["random-tag"], "exclude_private") is True + + +def test_require_public_needs_public_tag(): + assert _page_is_shared([], "require_public") is False + assert _page_is_shared(["rust"], "require_public") is False + assert _page_is_shared(["public"], "require_public") is True + + +def test_require_public_still_vetoes_private(): + # public AND private → private wins. + assert _page_is_shared(["public", "private"], "require_public") is False + + +def test_unknown_mode_treated_as_exclude_private(): + """The default mode is 'exclude_private'; unknown modes fall through to it.""" + assert _page_is_shared([], "totally-bogus-mode") is True + assert _page_is_shared(["private"], "totally-bogus-mode") is False diff --git a/tests/test_ssrf.py b/tests/test_ssrf.py new file mode 100644 index 0000000..807f9bd --- /dev/null +++ b/tests/test_ssrf.py @@ -0,0 +1,64 @@ +"""Tests for `_validate_url_target` — SSRF prevention. + +Any URL the app fetches must resolve to a public IP; private/internal/ +loopback addresses must be rejected so attacker-controlled URLs cannot +reach internal services via our HTTP client. +""" +import socket +from unittest.mock import patch + +import pytest + +from db import _validate_url_target + + +def _mock_getaddrinfo(address): + """Return a function suitable as a socket.getaddrinfo replacement.""" + def f(host, port, *args, **kwargs): + family = socket.AF_INET6 if ":" in address else socket.AF_INET + return [(family, socket.SOCK_STREAM, 0, "", (address, port or 80))] + return f + + +@pytest.mark.parametrize("blocked_ip", [ + "127.0.0.1", + "127.1.2.3", + "10.0.0.1", + "10.255.255.255", + "172.16.0.1", + "172.31.255.255", + "192.168.0.1", + "192.168.255.255", + "169.254.169.254", + "0.0.0.0", + "::1", + "fc00::1", + "fe80::1", +]) +def test_blocks_private_and_loopback(monkeypatch, blocked_ip): + monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo(blocked_ip)) + with pytest.raises(ValueError, match="blocked"): + _validate_url_target("https://evil.example.com/internal") + + +def test_allows_public_ipv4(monkeypatch): + monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo("8.8.8.8")) + _validate_url_target("https://dns.example.com/") # does not raise + + +def test_allows_public_ipv6(monkeypatch): + monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo("2001:4860:4860::8888")) + _validate_url_target("https://v6.example.com/") # does not raise + + +def test_rejects_unresolvable_hostname(monkeypatch): + def boom(*args, **kwargs): + raise socket.gaierror("no such host") + monkeypatch.setattr(socket, "getaddrinfo", boom) + with pytest.raises(ValueError, match="Cannot resolve"): + _validate_url_target("https://does-not-exist.example.com/") + + +def test_rejects_missing_hostname(): + with pytest.raises(ValueError, match="No hostname"): + _validate_url_target("http:///path-only") diff --git a/tests/test_url_cleanup.py b/tests/test_url_cleanup.py new file mode 100644 index 0000000..1eef72b --- /dev/null +++ b/tests/test_url_cleanup.py @@ -0,0 +1,101 @@ +"""Tests for `clean_url` — URL normalization and tracking-param stripping. + +Clean URLs are the deduplication key in the pages table, so any change to +this function can silently cause duplicate rows or mask legitimate saves. +""" +import pytest + +from conftest import patch_dns_ok, patch_dns_fail +from db import clean_url, TRACKING_PARAMS + + +def test_strips_fragment(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com/page#section") == "https://example.com/page" + + +def test_prefers_https(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("http://example.com/page") == "https://example.com/page" + + +def test_lowercases_hostname(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://EXAMPLE.COM/page") == "https://example.com/page" + + +def test_preserves_path_case(monkeypatch): + """Paths are case-sensitive and should not be lowercased.""" + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com/Foo/Bar") == "https://example.com/Foo/Bar" + + +def test_strips_default_https_port(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com:443/page") == "https://example.com/page" + + +@pytest.mark.xfail(reason="clean_url upgrades http->https before the port-default check, " + "so port 80 is not stripped. Minor dedup bug — harmless but worth fixing.") +def test_strips_http_port_80(monkeypatch): + """Expected: http://foo:80 → https://foo (both scheme-upgrade and port-strip). + + Currently fails because scheme is upgraded to https *before* the port check, + so `scheme == "http" and port == 80` is never true by the time the check runs. + """ + patch_dns_ok(monkeypatch) + assert clean_url("http://example.com:80/page") == "https://example.com/page" + + +def test_preserves_non_default_port(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com:8443/page") == "https://example.com:8443/page" + + +def test_strips_trailing_slash(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com/page/") == "https://example.com/page" + + +def test_root_slash_preserved(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com/") == "https://example.com/" + + +@pytest.mark.parametrize("param", sorted(TRACKING_PARAMS)) +def test_tracking_params_stripped(monkeypatch, param): + patch_dns_ok(monkeypatch) + result = clean_url(f"https://example.com/page?{param}=value&keep=yes") + assert param not in result + assert "keep=yes" in result + + +def test_strips_www_when_nonwww_resolves(monkeypatch): + """Standard case: strip `www.` prefix to canonicalize.""" + patch_dns_ok(monkeypatch) + assert clean_url("https://www.example.com/page") == "https://example.com/page" + + +def test_preserves_www_when_nonwww_does_not_resolve(monkeypatch): + """Regression for 6ffd38d. + + Some sites only serve their content at `www.domain.tld`; the bare domain + doesn't resolve. Stripping `www.` in that case produced a URL that we could + never actually fetch or dedupe against the real one. + """ + patch_dns_fail(monkeypatch) + assert clean_url("https://www.example.com/page") == "https://www.example.com/page" + + +def test_query_params_sorted_for_stable_ordering(monkeypatch): + """Same URL with different param orderings should produce the same clean URL.""" + patch_dns_ok(monkeypatch) + a = clean_url("https://example.com/page?b=2&a=1") + b = clean_url("https://example.com/page?a=1&b=2") + assert a == b + + +def test_path_and_query_preserved_through_cleanup(monkeypatch): + patch_dns_ok(monkeypatch) + result = clean_url("https://example.com/path/to/page?id=42&utm_source=twitter") + assert result == "https://example.com/path/to/page?id=42" diff --git a/themes/junimo.html b/themes/junimo.html new file mode 100644 index 0000000..f85e315 --- /dev/null +++ b/themes/junimo.html @@ -0,0 +1,1626 @@ + + + + + + + + + + + + + + +