{" | ".join(parts)}
' - - -# --- Tag helpers --- - - -def _get_page_tags(page_id, db=None): - close = False - if db is None: - db = get_db() - close = True - rows = db.execute( - "SELECT t.name FROM tags t JOIN page_tags pt ON t.id = pt.tag_id " - "WHERE pt.page_id = ? ORDER BY t.name", (page_id,) - ).fetchall() - if close: - return_db(db) - return [r["name"] for r in rows] - - -def _set_page_tags(page_id, tag_string, db=None): - close = False - if db is None: - db = get_db() - close = True - db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,)) - for name in (t.strip().lower() for t in tag_string.split(",") if t.strip()): - db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,)) - tag_id = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()["id"] - db.execute("INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", (page_id, tag_id)) - if close: - db.commit() - return_db(db) - - -# --- Route handlers --- - - -def handle_search(query): - q = query.get("q", [""])[0].strip() - page = _paginate(query) - offset = (page - 1) * PER_PAGE - db = get_db() - try: - count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - name = get_site_name() - - result_html = "" - trusted_html = "" - if q: - try: - total_results = db.execute( - "SELECT count(*) FROM pages_fts WHERE pages_fts MATCH ?", - (_sanitize_fts_query(q),), - ).fetchone()[0] - rows = db.execute( - "SELECT p.id, p.url, p.title, p.body, p.note " - "FROM pages_fts f JOIN pages p ON f.rowid = p.id " - "WHERE pages_fts MATCH ? ORDER BY rank LIMIT ? OFFSET ?", - (_sanitize_fts_query(q), PER_PAGE, offset), - ).fetchall() - except Exception: - rows = [] - total_results = 0 - if rows: - for r in rows: - note_html = "" - if r["note"]: - note_html = f'No results in your index.
" - - # search all linked pages from trusted sites - words = q.lower().split() - all_links = db.execute( - "SELECT l.url, l.label, p.title AS source_title " - "FROM links l JOIN pages p ON l.page_id = p.id", - ).fetchall() - indexed_urls = set(r["url"] for r in rows) if rows else set() - seen = set() - trusted = [] - for l in all_links: - if l["url"] in indexed_urls or l["url"] in seen: - continue - if any(w in l["label"].lower() for w in words): - seen.add(l["url"]) - trusted.append(l) - if len(trusted) >= 20: - break - - if trusted: - items = "" - for l in trusted: - items += ( - f'{msg}
" - f'back' - ) - - -def handle_add_submit(body): - url = clean_url(body.get("url", [""])[0].strip()) - note = body.get("note", [""])[0].strip() - tags = body.get("tags", [""])[0].strip() - if not url: - return handle_add_form("URL is required.") - if not url.startswith(("http://", "https://")): - return handle_add_form("URL must start with http:// or https://") - try: - title = index_url(url, note) - if tags: - db = get_db() - try: - row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() - if row: - _set_page_tags(row["id"], tags, db) - db.commit() - finally: - return_db(db) - return handle_add_form(f'Indexed: {esc(title)}') - except ValueError as e: - return handle_add_form(f"Error: {esc(str(e))}") - except Exception: - return handle_add_form("Error: could not fetch or index that URL.") - - -def handle_pages(query=None): - page = _paginate(query or {}) - offset = (page - 1) * PER_PAGE - db = get_db() - try: - total = db.execute("SELECT count(*) FROM pages").fetchone()[0] - rows = db.execute( - "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?", - (PER_PAGE, offset), - ).fetchall() - items = "" - for r in rows: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - tags = _get_page_tags(r["id"], db) - tags_html = "" - if tags: - tag_links = " ".join(f'[{esc(t)}]' for t in tags) - tags_html = f' {tag_links}' - items += ( - f'{esc(row['title'])}
"
- f"{esc(row['url'])}
{msg}
" - f'back' - ) - - -def handle_edit_submit(page_id, body): - note = body.get("note", [""])[0].strip() - tags = body.get("tags", [""])[0].strip() - db = get_db() - try: - db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) - _set_page_tags(page_id, tags, db) - db.commit() - finally: - return_db(db) - return _redirect("/pages") - - -def handle_delete_confirm(page_id): - db = get_db() - try: - row = db.execute("SELECT id, url, title FROM pages WHERE id = ?", (page_id,)).fetchone() - finally: - return_db(db) - if not row: - return _error(404) - return _respond( - f"Remove {esc(row['title'])}
"
- f"{esc(row['url'])}
Paste the contents of a tinyweb export file (JSON).
" - f'" - f"{msg}
" - f'back' - ) - - -def handle_import_submit(body): - raw = body.get("data", [""])[0].strip() - if not raw: - return handle_import_form("Paste JSON data.") - try: - data = json.loads(raw) - except json.JSONDecodeError: - return handle_import_form("Invalid JSON.") - if not isinstance(data, list): - return handle_import_form("Expected a JSON array.") - - MAX_IMPORT = 100 - if len(data) > MAX_IMPORT: - return handle_import_form(f"Too many entries. Maximum is {MAX_IMPORT}.") - - imported = 0 - errors = 0 - for entry in data: - url = entry.get("url", "").strip() - note = entry.get("note", "").strip() - if not url: - continue - try: - index_url(url, note) - imported += 1 - except Exception: - errors += 1 - - return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") - - -def handle_style_form(msg=""): - template = get_setting("custom_template") or DEFAULT_TEMPLATE - name = get_site_name() - sharing = get_setting("sharing_enabled", "0") - checked = " checked" if sharing == "1" else "" - return _respond( - f"Drag this link to your bookmarks bar. Click it on any page to index it instantly.
" - f'' - f"{msg}
" - f'back', - use_default=True, - ) - - -def handle_style_submit(body): - template = body.get("template", [""])[0] - name = body.get("site_name", ["tinyweb"])[0].strip() - sharing = "1" if body.get("sharing_enabled") else "0" - set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "") - set_setting("site_name", name or "tinyweb") - set_setting("sharing_enabled", sharing) - return handle_style_form("Saved.") - - -def handle_about(): - name = get_site_name() - dest_hash = get_setting("dest_hash") - sharing = get_setting("sharing_enabled", "0") == "1" - db = get_db() - try: - page_count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - tag_count = db.execute("SELECT count(DISTINCT tag_id) FROM page_tags").fetchone()[0] - sub_count = db.execute("SELECT count(*) FROM subscriptions").fetchone()[0] - finally: - return_db(db) - - sharing_html = ( - 'This instance shares its index publicly. Subscribe to join the network.
' - if sharing else - 'This instance is private.
' - ) - - hash_html = "" - if dest_hash: - hash_html = ( - f'To subscribe to this instance, add this destination hash in your TinyWeb:
' - f'{esc(dest_hash)}'
- )
-
- return _respond(
- f'A personal search engine, built for the slow web.
' - f'TinyWeb is about taking back the internet. No algorithms, no ads, no tracking. ' - f'Just human-curated pages shared freely across a mesh network.
' - f'The slow web is a movement for intentionality over speed, ' - f'human curation over algorithmic feeds, privacy over surveillance, ' - f'and community over corporations. Every page in this index was saved by a person ' - f'because they found it valuable — not because an algorithm told them to click.
' - f'No tags yet. Add tags when saving or editing pages.
" - f'back' - ) - - -def handle_tag_browse(tag_name, query=None): - page = _paginate(query or {}) - offset = (page - 1) * PER_PAGE - db = get_db() - try: - total = db.execute( - "SELECT count(*) FROM page_tags pt JOIN tags t ON t.id = pt.tag_id WHERE t.name = ?", - (tag_name,), - ).fetchone()[0] - rows = db.execute( - "SELECT p.id, p.url, p.title, p.note FROM pages p " - "JOIN page_tags pt ON p.id = pt.page_id " - "JOIN tags t ON t.id = pt.tag_id " - "WHERE t.name = ? ORDER BY p.id DESC LIMIT ? OFFSET ?", - (tag_name, PER_PAGE, offset), - ).fetchall() - items = "" - for r in rows: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - tags = _get_page_tags(r["id"], db) - tag_links = " ".join(f'[{esc(t)}]' for t in tags) - items += ( - f'{total} page(s)
' - f'| instance | last sync | auto-sync | actions |
|---|
{msg}
' - f'{len(sites)} site(s) available, {new_count} new
' - f'' - f'Invalid or missing CSRF token.
", status=403) - if path == "/add": - return handle_add_submit(body) - elif path.startswith("/edit/"): - pid = extract_id("/edit/") - return handle_edit_submit(pid, body) if pid is not None else _error(400) - elif path.startswith("/delete/"): - pid = extract_id("/delete/") - return handle_delete(pid) if pid is not None else _error(400) - elif path == "/style": - return handle_style_submit(body) - elif path == "/style/reset": - set_setting("custom_template", "") - return handle_style_form("Template reset to default.") - elif path == "/import": - return handle_import_submit(body) - elif path == "/subscriptions/add": - return handle_subscription_add(body) - elif path == "/subscriptions/pick": - return handle_subscription_pick(body) - elif path.startswith("/subscriptions/sync/"): - sid = extract_id("/subscriptions/sync/") - return handle_subscription_sync(sid) if sid is not None else _error(400) - elif path.startswith("/subscriptions/autosync/"): - sid = extract_id("/subscriptions/autosync/") - return handle_subscription_autosync(sid) if sid is not None else _error(400) - elif path.startswith("/subscriptions/delete/"): - sid = extract_id("/subscriptions/delete/") - return handle_subscription_delete(sid) if sid is not None else _error(400) - elif path == "/subscriptions/syncall": - return handle_subscription_syncall() - - return _error(404) - - -def dispatch_request(data): - cookies = data.get("cookies", {}) - csrf_token = cookies.get("_csrf", "") - if not csrf_token: - csrf_token = secrets.token_hex(32) - _request_local.csrf_token = csrf_token - - resp = _dispatch_inner(data) - - resp.setdefault("headers", {}) - resp["headers"]["Set-Cookie"] = f"_csrf={csrf_token}; SameSite=Strict; HttpOnly; Path=/" - resp["headers"]["X-Frame-Options"] = "DENY" - resp["headers"]["X-Content-Type-Options"] = "nosniff" - if resp.get("content_type", "").startswith("text/html"): - resp["headers"]["Content-Security-Policy"] = ( - "default-src 'self'; " - "script-src 'self' 'unsafe-inline'; " - "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " - "font-src 'self' https://fonts.gstatic.com; " - "img-src * data:; " - "frame-ancestors 'none'; " - "form-action 'self'; " - "base-uri 'self'" - ) - return resp diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6dfbe36 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "tinyweb" +version = "0.1.0" +description = "Personal decentralized search engine" +requires-python = ">=3.10" + +[tool.setuptools.packages.find] +where = ["src"] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.backends._legacy:_Backend" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..16d6cc5 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +testpaths = tests +python_files = test_*.py +filterwarnings = + ignore::DeprecationWarning diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..26b77f6 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest diff --git a/requirements.txt b/requirements.txt index f63da5d..29da454 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,9 @@ +# forum: pip install tinyweb-forum (optional, adds URL discussion board) requests beautifulsoup4 rns +onnxruntime +tokenizers +hnswlib +numpy +huggingface_hub diff --git a/rns_client.py b/rns_client.py deleted file mode 100644 index dbc0af5..0000000 --- a/rns_client.py +++ /dev/null @@ -1,79 +0,0 @@ -import time -import RNS - -APP_NAME = "tinyweb" -ASPECTS = ["server"] -REQUEST_TIMEOUT = 30 - - -def fetch_remote_sites(dest_hash_hex, since=""): - """ - Connect to a remote TinyWeb instance over Reticulum and fetch its - shared sites. Returns the response dict from /api/sites, or raises - an exception on failure. Pass `since` as ISO timestamp for delta sync. - """ - dest_hash = bytes.fromhex(dest_hash_hex) - - # Resolve path if needed - if not RNS.Transport.has_path(dest_hash): - RNS.Transport.request_path(dest_hash) - elapsed = 0 - while not RNS.Transport.has_path(dest_hash) and elapsed < 15: - time.sleep(0.5) - elapsed += 0.5 - if not RNS.Transport.has_path(dest_hash): - raise ConnectionError(f"Could not find path to {dest_hash_hex}") - - server_identity = RNS.Identity.recall(dest_hash) - if server_identity is None: - raise ConnectionError(f"Could not recall identity for {dest_hash_hex}") - - destination = RNS.Destination( - server_identity, - RNS.Destination.OUT, - RNS.Destination.SINGLE, - APP_NAME, - *ASPECTS, - ) - - # Establish link - link = RNS.Link(destination) - elapsed = 0 - while link.status == RNS.Link.PENDING and elapsed < 15: - time.sleep(0.25) - elapsed += 0.25 - - if link.status != RNS.Link.ACTIVE: - raise ConnectionError(f"Could not establish link to {dest_hash_hex}") - - try: - # Request /api/sites - query = {"since": [since]} if since else {} - request_data = { - "method": "GET", - "path": "/api/sites", - "query": query, - "body": {}, - "gateway_host": "", - } - - receipt = link.request("/tinyweb", data=request_data, timeout=REQUEST_TIMEOUT) - - elapsed = 0 - done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) - while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT: - time.sleep(0.5) - elapsed += 0.5 - - if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): - resp = receipt.get_response() - if resp["status"] == 403: - raise PermissionError("That instance has sharing disabled.") - if resp["status"] != 200: - raise ConnectionError(f"Remote returned status {resp['status']}") - import json - return json.loads(resp["body"]) - else: - raise ConnectionError(f"Request failed or timed out") - finally: - link.teardown() diff --git a/src/tinyweb/__init__.py b/src/tinyweb/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/tinyweb/__init__.py @@ -0,0 +1 @@ + diff --git a/src/tinyweb/app.py b/src/tinyweb/app.py new file mode 100644 index 0000000..10b756f --- /dev/null +++ b/src/tinyweb/app.py @@ -0,0 +1,323 @@ +import os +import sys +import time +import threading +import argparse +import RNS +from http.server import HTTPServer, ThreadingHTTPServer + +from tinyweb.db import init_db, get_setting, set_setting +from tinyweb.handlers import dispatch_request +import tinyweb.handlers as handlers_mod +import tinyweb.templates as templates_mod +import tinyweb.gateway as gateway +from tinyweb.gateway import GatewayState, GatewayHandler + +IDENTITY_FILE = "tinyweb_identity" +DEFAULT_TRANSPORT_HOST = "rnode.bre.land" +DEFAULT_TRANSPORT_PORT = 4242 +DATA_DIR = os.environ.get("TINYWEB_DATA_DIR") or os.path.expanduser("~/.tinyweb") + + +def get_transport_config(): + host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) + port = get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT)) + return host, int(port) + + +def find_available_port(start=8080, max_attempts=20, host="127.0.0.1"): + """Find an available port starting from start.""" + import socket + for port in range(start, start + max_attempts): + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + return port + except OSError: + continue + return start + + +def get_version(): + """Get version from git tag or VERSION file.""" + try: + import subprocess + tag = subprocess.check_output( + ["git", "describe", "--tags", "--abbrev=0"], + stderr=subprocess.DEVNULL, + text=True + ).strip() + if tag.startswith("v"): + return tag[1:] + return tag + except Exception: + version_file = os.path.join(os.path.dirname(__file__), "VERSION") + if os.path.exists(version_file): + with open(version_file) as f: + return f.read().strip() + return "0.0.0" + + +def load_or_create_identity(): + os.makedirs(DATA_DIR, exist_ok=True) + identity_path = os.path.join(DATA_DIR, IDENTITY_FILE) + if os.path.isfile(identity_path): + current = os.stat(identity_path).st_mode & 0o777 + if current != 0o600: + os.chmod(identity_path, 0o600) + return RNS.Identity.from_file(identity_path) + identity = RNS.Identity() + identity.to_file(identity_path) + os.chmod(identity_path, 0o600) + return identity + + +# Remote peers on the Reticulum mesh can reach read-only public pages. +# Only GET is allowed; POST is blocked because CSRF cannot authenticate +# mesh callers (the attacker controls both the "cookie" and the "form" side). +_RNS_ALLOWED_GET = { + "/", "/about", "/api/sites", "/share/preview", +} + +_RNS_ALLOWED_PREFIXES = ("/pages", "/tags", "/api/sites", "/rns") + + +def _rns_is_allowed(method, path): + if method != "GET": + return False + if path in _RNS_ALLOWED_GET: + return True + return any(path.startswith(p) for p in _RNS_ALLOWED_PREFIXES) + + +def rns_request_handler(path, data, request_id, link_id, remote_identity, requested_at): + if data is None: + data = {"method": "GET", "path": "/", "query": {}, "body": {}, "gateway_host": ""} + method = data.get("method", "GET") + req_path = data.get("path", "/") + if not _rns_is_allowed(method, req_path): + return { + "status": 403, + "content_type": "text/plain; charset=utf-8", + "body": "Forbidden: this endpoint is not available over Reticulum.", + "headers": {}, + } + return dispatch_request(data) + + +def start_gateway(reticulum, bind_host="127.0.0.1"): + GatewayState.reticulum = reticulum + GatewayState.local_dispatch = dispatch_request + HTTPServer.allow_reuse_address = True + server = ThreadingHTTPServer((bind_host, gateway.GATEWAY_PORT), GatewayHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + +def _config_settings_match(config_file, desired_host, desired_port): + """Check if existing config transport and LoRa settings match desired values.""" + import configparser + try: + config = configparser.ConfigParser() + config.read(config_file) + # Check TCP transport + tcp_enabled = get_setting("tcp_enabled", "1") == "1" + has_tcp = config.has_section("TCP Transport") + if tcp_enabled != has_tcp: + return False + if tcp_enabled and has_tcp: + if (config.get("TCP Transport", "target_host") != desired_host or + config.get("TCP Transport", "target_port") != str(desired_port)): + return False + # Check LoRa + lora_enabled = get_setting("lora_enabled", "0") == "1" + has_lora = config.has_section("RNode LoRa") + if lora_enabled != has_lora: + return False + if lora_enabled and has_lora: + if config.get("RNode LoRa", "port", fallback="") != get_setting("lora_port", ""): + return False + if config.get("RNode LoRa", "frequency", fallback="") != get_setting("lora_frequency", "867200000"): + return False + return True + except Exception: + pass + return False + + +def ensure_rns_config(config_dir, transport_host=None, transport_port=None): + """Generate a default Reticulum config with internet transport if none exists.""" + if config_dir is None: + config_dir = os.path.expanduser("~/.reticulum") + config_file = os.path.join(config_dir, "config") + if transport_host is None: + transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) + if transport_port is None: + transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))) + + managed_sentinel = "# managed by tinyweb" + if os.path.exists(config_file): + try: + with open(config_file) as f: + existing = f.read() + except OSError: + existing = "" + if managed_sentinel not in existing: + # User-authored config — don't clobber it. + if not _config_settings_match(config_file, transport_host, transport_port): + print( + f"Warning: {config_file} was not created by tinyweb; " + "leaving it alone. Edit it manually to change transport/LoRa settings." + ) + return + if _config_settings_match(config_file, transport_host, transport_port): + return + + # Build optional interface blocks + tcp_block = "" + if get_setting("tcp_enabled", "1") == "1": + tcp_block = f""" + [[TCP Transport]] + type = TCPClientInterface + enabled = yes + target_host = {transport_host} + target_port = {transport_port} +""" + + lora_block = "" + if get_setting("lora_enabled", "0") == "1": + lora_port = get_setting("lora_port", "") + if lora_port: + lora_frequency = get_setting("lora_frequency", "867200000") + lora_bandwidth = get_setting("lora_bandwidth", "125000") + lora_txpower = get_setting("lora_txpower", "7") + lora_sf = get_setting("lora_sf", "8") + lora_cr = get_setting("lora_cr", "5") + lora_block = f""" + [[RNode LoRa]] + type = RNodeInterface + enabled = yes + port = {lora_port} + frequency = {lora_frequency} + bandwidth = {lora_bandwidth} + txpower = {lora_txpower} + spreadingfactor = {lora_sf} + codingrate = {lora_cr} +""" + + os.makedirs(config_dir, exist_ok=True) + with open(config_file, "w") as f: + f.write(f"""{managed_sentinel} +[reticulum] + enable_transport = False + share_instance = No + +[logging] + loglevel = 4 + +[interfaces] + [[Default Interface]] + type = AutoInterface + enabled = Yes +{tcp_block}{lora_block}""") + print(f"Created Reticulum config at {config_file}") + + +def _preload_embeddings(): + """Pre-load the embedding model and build the HNSW index in background.""" + if get_setting("semantic_search", "0") != "1": + print("Semantic search disabled.") + return + try: + from tinyweb.embeddings import _get_session, _get_reranker, build_index + _get_session() + build_index() + if get_setting("use_reranker", "0") == "1": + _get_reranker() + print("Semantic search ready (with reranker).") + else: + print("Semantic search ready.") + except Exception as e: + print(f"Semantic search unavailable: {e}") + + +def main(): + parser = argparse.ArgumentParser(prog="tinyweb", description="Personal decentralized search engine") + parser.add_argument("--version", "-v", action="store_true", help="Show version") + parser.add_argument("--port", "-p", type=int, default=None, help="HTTP gateway port (default: 8080)") + parser.add_argument( + "--bind", "-b", default="127.0.0.1", + help="Address to bind the HTTP gateway to (default: 127.0.0.1). " + "Use 0.0.0.0 to expose to the LAN; note that the web UI has no authentication.", + ) + args = parser.parse_args() + + if args.version: + print(f"TinyWeb {get_version()}") + return + + bind_host = args.bind + port = args.port or 8080 + gateway.GATEWAY_PORT = find_available_port(port, host=bind_host) + + init_db() + transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) + transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))) + threading.Thread(target=_preload_embeddings, daemon=True).start() + config_dir = os.environ.get("RNS_CONFIG_DIR") + ensure_rns_config(config_dir, transport_host, transport_port) + reticulum = RNS.Reticulum(configdir=config_dir) + identity = load_or_create_identity() + + destination = RNS.Destination( + identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + gateway.APP_NAME, + *gateway.ASPECTS, + ) + + destination.register_request_handler( + "/tinyweb", + response_generator=rns_request_handler, + allow=RNS.Destination.ALLOW_ALL, + ) + + # Initialize forum plugin if available + forum = None + try: + from tinyweb_forum import ForumPlugin + from tinyweb.db import get_site_name + forum = ForumPlugin(DATA_DIR, identity, reticulum, site_name=get_site_name()) + if get_setting("forum_enabled", "0") == "1": + forum.enable() + templates_mod.FORUM_ENABLED = True + handlers_mod.forum_plugin = forum + print(f"Forum plugin: {'enabled' if forum.is_enabled() else 'available (enable in settings)'}") + except ImportError: + print("Forum plugin not installed (pip install tinyweb[forum])") + except Exception as e: + print(f"Forum plugin error: {e}") + + # Brief delay to ensure all interfaces (especially TCP) are fully ready + time.sleep(2) + destination.announce() + set_setting("dest_hash", destination.hash.hex()) + start_gateway(reticulum, bind_host=bind_host) + + print(f"TinyWeb running!") + if bind_host in ("0.0.0.0", "::"): + print(f"Open http://localhost:{gateway.GATEWAY_PORT} in your browser") + print(f"WARNING: listening on {bind_host} — the web UI has no authentication. " + "Anyone on your network can control this instance.") + else: + print(f"Open http://{bind_host}:{gateway.GATEWAY_PORT} in your browser") + print(f"Destination hash: {RNS.prettyhexrep(destination.hash)} (share this so friends can subscribe)") + + while True: + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/db.py b/src/tinyweb/db.py similarity index 65% rename from db.py rename to src/tinyweb/db.py index 0d32b3a..b794b7c 100644 --- a/db.py +++ b/src/tinyweb/db.py @@ -2,10 +2,12 @@ import socket import ipaddress import sqlite3 import requests +import os from urllib.parse import urlparse, urljoin, parse_qs, urlencode, urlunparse, quote from bs4 import BeautifulSoup -DATABASE = "index.db" +DATA_DIR = os.environ.get("TINYWEB_DATA_DIR") or os.path.expanduser("~/.tinyweb") +DATABASE = os.path.join(DATA_DIR, "index.db") BLOCKED_NETWORKS = [ ipaddress.ip_network("127.0.0.0/8"), @@ -20,6 +22,22 @@ BLOCKED_NETWORKS = [ ] +def _is_blocked_response(html, status_code): + """Check if response is a CDN challenge/block page.""" + if status_code == 403: + return True + html_lower = html.lower() + if "just a moment" in html_lower or "cloudflare" in html_lower: + return True + if "enable javascript and cookies" in html_lower: + return True + if "request rejected" in html_lower: + return True + if "access denied" in html_lower: + return True + return False + + def _validate_url_target(url): """Resolve hostname and block private/internal IPs to prevent SSRF.""" parsed = urlparse(url) @@ -52,10 +70,16 @@ def clean_url(url): # Prefer https scheme = "https" if parsed.scheme in ("http", "https") else parsed.scheme - # Normalize hostname: lowercase, strip www. + # Normalize hostname: lowercase, strip www (only if non-www resolves) hostname = (parsed.hostname or "").lower() + original_hostname = hostname if hostname.startswith("www."): hostname = hostname[4:] + port = parsed.port or (443 if scheme == "https" else 80) + try: + socket.getaddrinfo(hostname, port, proto=socket.IPPROTO_TCP) + except socket.gaierror: + hostname = original_hostname # Preserve explicit non-default ports port = parsed.port @@ -79,7 +103,7 @@ def clean_url(url): _pool = [] _pool_lock = __import__("threading").Lock() -_POOL_SIZE = 4 +_POOL_SIZE = 16 def get_db(): @@ -99,6 +123,14 @@ def get_db(): def return_db(db): + try: + db.rollback() + except Exception: + try: + db.close() + except Exception: + pass + return with _pool_lock: if len(_pool) < _POOL_SIZE: _pool.append(db) @@ -107,6 +139,7 @@ def return_db(db): def init_db(): + os.makedirs(DATA_DIR, exist_ok=True) db = sqlite3.connect(DATABASE) db.execute( "CREATE TABLE IF NOT EXISTS pages (" @@ -115,7 +148,8 @@ def init_db(): " title TEXT," " body TEXT," " note TEXT DEFAULT ''," - " last_modified TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now'))" + " last_modified TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now'))," + " reticulum_dest TEXT DEFAULT ''" ")" ) db.execute( @@ -207,26 +241,66 @@ def init_db(): VALUES (new.id, new.title, new.url, new.note); END; """) - # Migrate old subscriptions table if needed - cols = [row[1] for row in db.execute("PRAGMA table_info(subscriptions)").fetchall()] - if "url" in cols and "dest_hash" not in cols: - db.execute("ALTER TABLE subscriptions RENAME COLUMN url TO dest_hash") + db.execute( + "CREATE TABLE IF NOT EXISTS mesh_sites (" + " hash TEXT PRIMARY KEY," + " name TEXT DEFAULT ''," + " added_at TEXT DEFAULT (strftime('%Y-%m-%dT%H:%M:%S','now'))" + ")" + ) + # Migrate pages: add reticulum_dest column if missing + if "reticulum_dest" not in page_cols: + db.execute("ALTER TABLE pages ADD COLUMN reticulum_dest TEXT DEFAULT ''") db.commit() - # Migrate remote_pages: add tags column if missing - rp_cols = [row[1] for row in db.execute("PRAGMA table_info(remote_pages)").fetchall()] - if "tags" not in rp_cols: - db.execute("ALTER TABLE remote_pages ADD COLUMN tags TEXT DEFAULT ''") + # Migrate subscriptions: add forum_enabled column + sub_cols = [row[1] for row in db.execute("PRAGMA table_info(subscriptions)").fetchall()] + if "forum_enabled" not in sub_cols: + db.execute("ALTER TABLE subscriptions ADD COLUMN forum_enabled INTEGER DEFAULT 0") db.commit() - # Migrate pages: add last_modified column if missing - page_cols = [row[1] for row in db.execute("PRAGMA table_info(pages)").fetchall()] - if "last_modified" not in page_cols: - db.execute("ALTER TABLE pages ADD COLUMN last_modified TEXT DEFAULT ''") - db.execute("UPDATE pages SET last_modified = strftime('%Y-%m-%dT%H:%M:%S','now') WHERE last_modified = ''") - db.commit() + # Chunks table for semantic search embeddings + db.execute( + "CREATE TABLE IF NOT EXISTS chunks (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " page_id INTEGER," + " remote_page_id INTEGER," + " chunk_index INTEGER NOT NULL," + " chunk_text TEXT NOT NULL," + " embedding BLOB NOT NULL," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE," + " FOREIGN KEY (remote_page_id) REFERENCES remote_pages(id) ON DELETE CASCADE" + ")" + ) + db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_page ON chunks(page_id)") + db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_remote ON chunks(remote_page_id)") + db.execute("CREATE INDEX IF NOT EXISTS idx_chunks_page_idx ON chunks(page_id, chunk_index)") + db.execute("CREATE INDEX IF NOT EXISTS idx_pages_url ON pages(url)") + db.execute("CREATE INDEX IF NOT EXISTS idx_pages_modified ON pages(last_modified)") + db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_page ON page_tags(page_id)") + db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_tag ON page_tags(tag_id)") + + # Migrate custom_template: replace hardcoded forum link with {{forum_link}} placeholder + cur = db.execute("SELECT value FROM settings WHERE key='custom_template'") + row = cur.fetchone() + if row: + updated = row[0].replace('forum', "{{forum_link}}") + if updated != row[0]: + db.execute("UPDATE settings SET value=? WHERE key='custom_template'", (updated,)) + db.commit() + + # Migrate custom_template: replace hardcoded site name with {{site_name}} placeholder + cur = db.execute("SELECT value FROM settings WHERE key='custom_template'") + row = cur.fetchone() + if row and '{{site_name}}' not in row[0]: + updated = row[0].replace('href="/">tinyweb', 'href="/">{{site_name}}') + if updated != row[0]: + db.execute("UPDATE settings SET value=? WHERE key='custom_template'", (updated,)) + db.commit() db.execute("PRAGMA journal_mode=WAL") + db.execute("PRAGMA synchronous=NORMAL") + db.execute("PRAGMA cache_size=-64000") db.commit() db.close() @@ -240,6 +314,16 @@ def get_setting(key, default=""): return_db(db) +def vacuum_db(): + """Run VACUUM and WAL checkpoint to reclaim space after deletions.""" + db = get_db() + try: + db.execute("PRAGMA wal_checkpoint(TRUNCATE)") + db.execute("VACUUM") + finally: + return_db(db) + + def set_setting(key, value): db = get_db() try: @@ -260,6 +344,10 @@ def get_site_name(): def fetch_page(url): _validate_url_target(url) resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False) + + if _is_blocked_response(resp.text, resp.status_code): + raise Exception(f"Site blocks automated access: {resp.status_code}") + # Follow redirects manually, re-validating each target max_redirects = 5 while resp.is_redirect and max_redirects > 0: @@ -296,24 +384,40 @@ def fetch_page(url): label = a.get_text(strip=True) or href links.append((href, label[:200])) - for tag in soup(["script", "style", "nav", "footer", "header"]): + # Extract meta description before stripping tags (case-insensitive) + meta_desc = "" + for m in soup.find_all("meta"): + name = (m.get("name") or "").lower() + prop = (m.get("property") or "").lower() + content = (m.get("content") or "").strip() + if not content: + continue + if name == "description" and len(content) > len(meta_desc): + meta_desc = content + elif prop == "og:description" and not meta_desc: + meta_desc = content + + for tag in soup(["script", "style", "nav", "footer", "header", "noscript", "aside"]): tag.decompose() title = soup.title.string.strip() if soup.title and soup.title.string else url + body = soup.get_text(separator=" ", strip=True) - return title, body, links + return title, body, links, meta_desc -def index_url(url, note=""): + +def index_url(url, note="", reticulum_dest=""): url = clean_url(url) - title, body, links = fetch_page(url) + title, body, links, meta_desc = fetch_page(url) + summary = meta_desc if meta_desc and len(meta_desc) > 20 else "" db = get_db() try: now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") db.execute( - "INSERT INTO pages (url, title, body, note, last_modified) VALUES (?, ?, ?, ?, ?) " + "INSERT INTO pages (url, title, body, note, last_modified, summary, reticulum_dest) VALUES (?, ?, ?, ?, ?, ?, ?) " "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, " - "note=excluded.note, last_modified=excluded.last_modified", - (url, title, body, note, now), + "note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary, reticulum_dest=excluded.reticulum_dest", + (url, title, body, note, now, summary, reticulum_dest), ) page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0] db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) @@ -323,6 +427,12 @@ def index_url(url, note=""): (page_id, href, label), ) db.commit() + if get_setting("semantic_search", "0") == "1": + try: + from tinyweb.embeddings import store_embeddings + store_embeddings(page_id, title, body, db) + except Exception: + pass # embedding generation is best-effort finally: return_db(db) return title diff --git a/src/tinyweb/embeddings.py b/src/tinyweb/embeddings.py new file mode 100644 index 0000000..2aecee4 --- /dev/null +++ b/src/tinyweb/embeddings.py @@ -0,0 +1,586 @@ +"""Semantic search using Snowflake arctic-embed-s via ONNX Runtime + hnswlib.""" + +import os +import re +import threading +import numpy as np + +DATA_DIR = os.path.expanduser("~/.tinyweb") +MODEL_ID = "Snowflake/snowflake-arctic-embed-s" +MODEL_DIR = os.path.join(DATA_DIR, "models", "snowflake-arctic-embed-s") +RERANKER_DIR = os.path.join(DATA_DIR, "models", "cross-encoder") +HNSW_PATH = os.path.join(DATA_DIR, "index.hnsw") +DIMS = 384 +MAX_TOKENS = 512 +QUERY_PREFIX = "Represent this sentence for searching relevant passages: " + +_session = None +_tokenizer = None +_lock = threading.Lock() + +_reranker_session = None +_reranker_tokenizer = None +_reranker_lock = threading.Lock() + +# Live HNSW index and chunk-id mapping +_hnsw_index = None +_hnsw_ids = [] # maps internal HNSW label -> chunks.id +_hnsw_lock = threading.Lock() + + +# --------------------------------------------------------------------------- +# Model download & loading +# --------------------------------------------------------------------------- + +def _ensure_model(): + """Download the ONNX model and tokenizer from HuggingFace if not present.""" + os.makedirs(MODEL_DIR, exist_ok=True) + model_path = os.path.join(MODEL_DIR, "model.onnx") + tokenizer_path = os.path.join(MODEL_DIR, "tokenizer.json") + if os.path.exists(model_path) and os.path.exists(tokenizer_path): + return + from huggingface_hub import hf_hub_download + os.makedirs(MODEL_DIR, exist_ok=True) + files = { + "onnx/model_quantized.onnx": "model.onnx", + "tokenizer.json": "tokenizer.json", + "tokenizer_config.json": "tokenizer_config.json", + } + for remote, local in files.items(): + target = os.path.join(MODEL_DIR, local) + if os.path.exists(target): + continue + cached = hf_hub_download(repo_id=MODEL_ID, filename=remote) + # hf_hub_download returns the cached file path; copy to our model dir + import shutil + shutil.copy2(cached, target) + + +def _get_session(): + """Return (onnxruntime.InferenceSession, tokenizers.Tokenizer) singleton.""" + global _session, _tokenizer + if _session is not None: + return _session, _tokenizer + with _lock: + if _session is not None: + return _session, _tokenizer + _ensure_model() + import onnxruntime as ort + from tokenizers import Tokenizer + _session = ort.InferenceSession( + os.path.join(MODEL_DIR, "model.onnx"), + providers=["CPUExecutionProvider"], + ) + _tokenizer = Tokenizer.from_file(os.path.join(MODEL_DIR, "tokenizer.json")) + _tokenizer.enable_truncation(max_length=MAX_TOKENS) + _tokenizer.enable_padding(pad_id=0, pad_token="[PAD]", length=None) + return _session, _tokenizer + + +def _get_reranker(): + """Return (onnxruntime.InferenceSession, tokenizers.Tokenizer) for the cross-encoder reranker.""" + global _reranker_session, _reranker_tokenizer + if _reranker_session is not None: + return _reranker_session, _reranker_tokenizer + with _reranker_lock: + if _reranker_session is not None: + return _reranker_session, _reranker_tokenizer + model_path = os.path.join(RERANKER_DIR, "model.onnx") + tok_path = os.path.join(RERANKER_DIR, "tokenizer.json") + if not os.path.exists(model_path) or not os.path.exists(tok_path): + return None, None + import onnxruntime as ort + from tokenizers import Tokenizer + _reranker_session = ort.InferenceSession( + model_path, providers=["CPUExecutionProvider"], + ) + _reranker_tokenizer = Tokenizer.from_file(tok_path) + _reranker_tokenizer.enable_truncation(max_length=512) + _reranker_tokenizer.enable_padding(pad_id=0, pad_token="[PAD]", length=None) + return _reranker_session, _reranker_tokenizer + + +def rerank(query, documents, limit=10): + """Score query-document pairs with the cross-encoder and return reranked indices. + + Args: + query: search query string + documents: list of document texts to score against the query + limit: max results to return + + Returns: list of (original_index, score) sorted by score descending. + """ + session, tokenizer = _get_reranker() + if session is None: + return [(i, 0.0) for i in range(min(limit, len(documents)))] + + # Cross-encoder takes (query, document) pairs — encode as pair sequences + pairs = [[query, doc] for doc in documents] + encodings = tokenizer.encode_batch(pairs) + + input_ids = np.array([e.ids for e in encodings], dtype=np.int64) + attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64) + token_type_ids = np.array([e.type_ids for e in encodings], dtype=np.int64) + + outputs = session.run( + None, + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids, + }, + ) + # Output is logits — higher = more relevant + scores = outputs[0].flatten() + ranked = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True) + return [(i, float(scores[i])) for i in ranked[:limit]] + + +# --------------------------------------------------------------------------- +# Text chunking +# --------------------------------------------------------------------------- + +_SENTENCE_RE = re.compile(r'(?<=[.!?])\s+') + + +def chunk_text(title, body): + """Split body into chunks, each prefixed with title for context. + + Strategy: split on double newlines (paragraphs). If a paragraph exceeds + MAX_TOKENS words, split at sentence boundaries. Each chunk is prefixed + with the page title. + """ + if not body or not body.strip(): + return [f"{title}"] if title else [] + + prefix = f"{title}: " if title else "" + # Rough word budget for chunk body (leave room for prefix) + prefix_words = len(prefix.split()) + max_words = MAX_TOKENS - prefix_words # approximate; tokenizer may differ + + paragraphs = re.split(r'\n\s*\n', body.strip()) + chunks = [] + + for para in paragraphs: + para = para.strip() + if len(para) < 20: + continue + words = para.split() + if len(words) <= max_words: + chunks.append(prefix + para) + else: + # Split paragraph into sentences + sentences = _SENTENCE_RE.split(para) + current = [] + current_len = 0 + for sent in sentences: + sent_words = len(sent.split()) + if current_len + sent_words > max_words and current: + chunks.append(prefix + " ".join(current)) + current = [] + current_len = 0 + if sent_words > max_words: + # Sentence too long — use sliding window + s_words = sent.split() + for i in range(0, len(s_words), max_words - 50): + window = s_words[i:i + max_words] + chunks.append(prefix + " ".join(window)) + else: + current.append(sent) + current_len += sent_words + if current: + chunks.append(prefix + " ".join(current)) + + if not chunks and title: + chunks = [title] + + return chunks + + +# --------------------------------------------------------------------------- +# Embedding +# --------------------------------------------------------------------------- + +def embed(texts, is_query=False): + """Encode texts into L2-normalized float32 embeddings (N, 384). + + For queries, prepend the model's query prefix. + Processes in batches of 32 to limit memory usage. + """ + if not texts: + return np.empty((0, DIMS), dtype=np.float32) + + session, tokenizer = _get_session() + + if is_query: + texts = [QUERY_PREFIX + t for t in texts] + + batch_size = 32 + all_embeddings = [] + + for start in range(0, len(texts), batch_size): + batch = texts[start:start + batch_size] + encodings = tokenizer.encode_batch(batch) + input_ids = np.array([e.ids for e in encodings], dtype=np.int64) + attention_mask = np.array([e.attention_mask for e in encodings], dtype=np.int64) + token_type_ids = np.zeros_like(input_ids) + + outputs = session.run( + None, + { + "input_ids": input_ids, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids, + }, + ) + emb = outputs[0][:, 0, :] + all_embeddings.append(emb) + + embeddings = np.concatenate(all_embeddings, axis=0) + norms = np.linalg.norm(embeddings, axis=1, keepdims=True) + norms = np.maximum(norms, 1e-12) + embeddings = embeddings / norms + return _maybe_compress(embeddings.astype(np.float32)) + + +def _maybe_compress(embeddings): + """Compress embeddings to float16 if compression is enabled.""" + try: + from tinyweb.db import get_setting + if get_setting("compress_embeddings", "0") == "1": + return embeddings.astype(np.float16) + except Exception: + pass + return embeddings + + +def _decompress(embeddings): + """Decompress float16 embeddings to float32 if needed.""" + if embeddings.dtype == np.float16: + return embeddings.astype(np.float32) + return embeddings + + +def _blob_to_vec(buf): + """Decode a stored embedding blob to a float32 vector, inferring dtype from length.""" + if len(buf) == DIMS * 2: + return np.frombuffer(buf, dtype=np.float16).astype(np.float32) + return np.frombuffer(buf, dtype=np.float32) + + +# --------------------------------------------------------------------------- +# HNSW index management +# --------------------------------------------------------------------------- + +BATCH_SIZE = 50000 + +def build_index(db=None): + """Load all embeddings from chunks table and build HNSW index in batches.""" + import hnswlib + global _hnsw_index, _hnsw_ids + + from tinyweb.db import get_db, return_db + own_db = db is None + if own_db: + db = get_db() + + try: + total = db.execute("SELECT COUNT(*) FROM chunks").fetchone()[0] + if total == 0: + with _hnsw_lock: + _hnsw_index = None + _hnsw_ids = [] + return + + all_ids = [] + all_embeddings = [] + + for offset in range(0, total, BATCH_SIZE): + rows = db.execute( + "SELECT id, embedding FROM chunks ORDER BY id LIMIT ? OFFSET ?", + (BATCH_SIZE, offset), + ).fetchall() + for r in rows: + emb = _blob_to_vec(r["embedding"]) + all_ids.append(r["id"]) + all_embeddings.append(emb) + finally: + if own_db: + return_db(db) + + if not all_ids: + with _hnsw_lock: + _hnsw_index = None + _hnsw_ids = [] + return + + matrix = np.stack(all_embeddings) + n = len(all_ids) + ids = all_ids + + index = hnswlib.Index(space="cosine", dim=DIMS) + index.init_index(max_elements=max(n, 1024), ef_construction=200, M=16) + index.add_items(matrix, list(range(n))) + index.set_ef(50) + + with _hnsw_lock: + _hnsw_index = index + _hnsw_ids = ids + + +def _add_to_index(chunk_ids, embeddings_matrix): + """Add new embeddings to the live HNSW index.""" + import hnswlib + global _hnsw_index, _hnsw_ids + + with _hnsw_lock: + if _hnsw_index is None: + index = hnswlib.Index(space="cosine", dim=DIMS) + index.init_index(max_elements=1024, ef_construction=200, M=16) + index.set_ef(50) + _hnsw_index = index + _hnsw_ids = [] + + current_max = _hnsw_index.get_max_elements() + needed = len(_hnsw_ids) + len(chunk_ids) + if needed > current_max: + _hnsw_index.resize_index(max(needed * 2, current_max * 2)) + + labels = list(range(len(_hnsw_ids), len(_hnsw_ids) + len(chunk_ids))) + _hnsw_index.add_items(embeddings_matrix, labels) + _hnsw_ids.extend(chunk_ids) + + +# --------------------------------------------------------------------------- +# Store embeddings for pages +# --------------------------------------------------------------------------- + +def store_embeddings(page_id, title, body, db): + """Chunk, embed, and store embeddings for a page. Adds to HNSW index.""" + chunks = chunk_text(title, body) + if not chunks: + return + + embeddings_matrix = embed(chunks) + embeddings_matrix = _decompress(embeddings_matrix) + + db.execute("DELETE FROM chunks WHERE page_id = ?", (page_id,)) + + new_ids = [] + for i, (text, emb) in enumerate(zip(chunks, embeddings_matrix)): + cursor = db.execute( + "INSERT INTO chunks (page_id, remote_page_id, chunk_index, chunk_text, embedding) " + "VALUES (?, NULL, ?, ?, ?)", + (page_id, i, text, emb.tobytes()), + ) + new_ids.append(cursor.lastrowid) + db.commit() + + _add_to_index(new_ids, embeddings_matrix) + + +def store_remote_embeddings(remote_page_id, title, note, db): + """Store a single embedding for a remote page (title + note).""" + text = f"{title}: {note}" if note else (title or "") + if not text.strip(): + return + + embeddings_matrix = embed([text]) + embeddings_matrix = _decompress(embeddings_matrix) + + db.execute("DELETE FROM chunks WHERE remote_page_id = ?", (remote_page_id,)) + cursor = db.execute( + "INSERT INTO chunks (page_id, remote_page_id, chunk_index, chunk_text, embedding) " + "VALUES (NULL, ?, 0, ?, ?)", + (remote_page_id, text, embeddings_matrix[0].tobytes()), + ) + db.commit() + + _add_to_index([cursor.lastrowid], embeddings_matrix) + + +# --------------------------------------------------------------------------- +# Search +# --------------------------------------------------------------------------- + +def semantic_search(query_text, limit=100, db=None): + """Search for pages by semantic similarity. + + Returns: [(page_id, score, best_chunk_text), ...] sorted by score desc. + Groups by page_id, taking the max chunk score per page. + """ + if _hnsw_index is None or not _hnsw_ids: + return [] + + query_emb = embed([query_text], is_query=True) + + with _hnsw_lock: + if _hnsw_index is None or not _hnsw_ids: + return [] + k = min(limit * 3, len(_hnsw_ids)) # oversample to account for grouping + if k == 0: + return [] + labels, distances = _hnsw_index.knn_query(query_emb, k=k) + + # Map HNSW labels back to chunk IDs + chunk_ids = [_hnsw_ids[int(lbl)] for lbl in labels[0]] + # cosine distance -> similarity: hnswlib returns 1-cosine for "cosine" space + scores = [1.0 - float(d) for d in distances[0]] + + # Fetch chunk details from DB + from tinyweb.db import get_db, return_db + own_db = db is None + if own_db: + db = get_db() + try: + placeholders = ",".join("?" * len(chunk_ids)) + rows = db.execute( + f"SELECT id, page_id, chunk_text FROM chunks WHERE id IN ({placeholders})", + chunk_ids, + ).fetchall() + finally: + if own_db: + return_db(db) + + chunk_map = {r["id"]: r for r in rows} + + # Group by page_id, keep best score and chunk text per page + page_best = {} # page_id -> (score, chunk_text) + for cid, score in zip(chunk_ids, scores): + chunk = chunk_map.get(cid) + if not chunk or chunk["page_id"] is None: + continue + pid = chunk["page_id"] + if pid not in page_best or score > page_best[pid][0]: + page_best[pid] = (score, chunk["chunk_text"]) + + results = [(pid, score, text) for pid, (score, text) in page_best.items()] + results.sort(key=lambda x: x[1], reverse=True) + return results[:limit] + + +def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=False): + """Merge BM25 and semantic results via RRF, optionally rerank with cross-encoder. + + Default (two-stage): BM25 + semantic fused via RRF. + With use_reranker=True (three-stage): rerank top 20 with cross-encoder. + + Returns: [(page_id, best_chunk_text), ...] in ranked order. + """ + k = 60 # RRF constant + + sem_results = semantic_search(query_text, limit=100, db=db) + + best_chunks = {} # page_id -> chunk_text + for _rank, (pid, _score, chunk_text) in enumerate(sem_results): + if pid not in best_chunks: + best_chunks[pid] = chunk_text + + # When BM25 has no hits, use raw semantic similarity scores directly + # (RRF rank positions distort nearly-equal scores) + if not bm25_ranked_ids: + fused_ids = [(pid, score) for pid, score, _ in sem_results] + else: + rrf_scores = {} + for rank, pid in enumerate(bm25_ranked_ids): + rrf_scores[pid] = rrf_scores.get(pid, 0) + 1.0 / (k + rank + 1) + for rank, (pid, _score, chunk_text) in enumerate(sem_results): + rrf_scores[pid] = rrf_scores.get(pid, 0) + 1.0 / (k + rank + 1) + fused_ids = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True) + + fused = fused_ids + all_ids = [pid for pid, _ in fused] + + if not all_ids: + return [] + + if not use_reranker: + return [(pid, best_chunks.get(pid, "")) for pid in all_ids[:limit]] + + # --- Rerank top 20, append next 10 from RRF order --- + rerank_ids = all_ids[:20] + tail_ids = all_ids[20:30] + + from tinyweb.db import get_db, return_db + own_db = db is None + if own_db: + db = get_db() + try: + placeholders = ",".join("?" * len(rerank_ids)) + rows = db.execute( + f"SELECT id, title, body FROM pages WHERE id IN ({placeholders})", + rerank_ids, + ).fetchall() + finally: + if own_db: + return_db(db) + + page_map = {r["id"]: r for r in rows} + + doc_texts = [] + ordered_ids = [] + for pid in rerank_ids: + page = page_map.get(pid) + if not page: + continue + chunk = best_chunks.get(pid, "") + body_preview = chunk[:200] if chunk else page["body"][:200] + doc = f"{page['title']}. {body_preview}" + doc_texts.append(doc) + ordered_ids.append(pid) + + if not doc_texts: + return [] + + try: + reranked = rerank(query_text, doc_texts, limit=20) + results = [(ordered_ids[idx], best_chunks.get(ordered_ids[idx], "")) for idx, _score in reranked] + except Exception: + results = [(pid, best_chunks.get(pid, "")) for pid in ordered_ids[:20]] + + # Append next 10 from RRF order (no reranking) + reranked_set = {pid for pid, _ in results} + for pid in tail_ids: + if pid not in reranked_set: + results.append((pid, best_chunks.get(pid, ""))) + + return results[:30] + + +# --------------------------------------------------------------------------- +# Reindex +# --------------------------------------------------------------------------- + +def reindex_all(db=None, progress_callback=None): + """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index.""" + from tinyweb.db import get_db, return_db + own_db = db is None + if own_db: + db = get_db() + try: + # Clear existing chunks so everything is regenerated + db.execute("DELETE FROM chunks") + db.commit() + + rows = db.execute( + "SELECT p.id, p.title, p.body, p.summary FROM pages p" + ).fetchall() + + total = len(rows) + for i, row in enumerate(rows): + store_embeddings(row["id"], row["title"], row["body"], db) + if progress_callback: + progress_callback(i + 1, total) + + # Also handle remote pages + remote_rows = db.execute( + "SELECT rp.id, rp.title, rp.note FROM remote_pages rp" + ).fetchall() + + for rp in remote_rows: + store_remote_embeddings(rp["id"], rp["title"], rp["note"], db) + finally: + if own_db: + return_db(db) + + build_index(db) diff --git a/gateway.py b/src/tinyweb/gateway.py similarity index 64% rename from gateway.py rename to src/tinyweb/gateway.py index a13816f..5b292ff 100644 --- a/gateway.py +++ b/src/tinyweb/gateway.py @@ -1,14 +1,21 @@ +import re import sys import time import threading +import collections import RNS -from http.server import HTTPServer, BaseHTTPRequestHandler +from http.server import HTTPServer, ThreadingHTTPServer, BaseHTTPRequestHandler from urllib.parse import parse_qs, urlparse APP_NAME = "tinyweb" ASPECTS = ["server"] GATEWAY_PORT = 8080 REQUEST_TIMEOUT = 60 +MAX_BODY_SIZE = 16 * 1024 * 1024 # 16 MiB — covers /import and every other form +RATE_LIMIT_WINDOW = 60 +RATE_LIMIT_MAX = 30 +_rate_tracker = collections.defaultdict(list) +_rate_lock = threading.Lock() class GatewayState: @@ -65,14 +72,40 @@ def ensure_link(): class GatewayHandler(BaseHTTPRequestHandler): + def _check_rate_limit(self): + client = self.client_address[0] + now = time.time() + with _rate_lock: + times = _rate_tracker[client] + cutoff = now - RATE_LIMIT_WINDOW + while times and times[0] < cutoff: + times.pop(0) + if len(times) >= RATE_LIMIT_MAX: + return False + times.append(now) + return True + def _forward(self, method): parsed = urlparse(self.path) query = parse_qs(parsed.query) body = {} if method == "POST": - length = int(self.headers.get("Content-Length", 0)) - raw = self.rfile.read(length).decode() + if not self._check_rate_limit(): + self.send_error(429, "Too many requests — slow down.") + return + try: + length = int(self.headers.get("Content-Length", 0)) + except ValueError: + self.send_error(400, "Invalid Content-Length") + return + if length < 0: + self.send_error(400, "Invalid Content-Length") + return + if length > MAX_BODY_SIZE: + self.send_error(413, "Request body too large") + return + raw = self.rfile.read(length).decode("utf-8", errors="replace") body = parse_qs(raw) # Parse cookies @@ -92,6 +125,7 @@ class GatewayHandler(BaseHTTPRequestHandler): "body": body, "cookies": cookies, "gateway_host": self.headers.get("Host", f"localhost:{GATEWAY_PORT}"), + "scheme": self.headers.get("X-Forwarded-Proto", "http"), } try: @@ -123,12 +157,23 @@ class GatewayHandler(BaseHTTPRequestHandler): self.send_response(resp["status"]) self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8")) + self.send_header("Referrer-Policy", "no-referrer") + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("X-Frame-Options", "DENY") + self.send_header("Content-Security-Policy", + "default-src 'self'; " + "style-src 'self' 'unsafe-inline'; " + "script-src 'self' 'unsafe-inline'; " + "img-src 'self' data:") + resp_body = resp.get("body", "") + encoded = resp_body.encode() if isinstance(resp_body, str) else resp_body + if encoded: + self.send_header("Content-Length", str(len(encoded))) for k, v in resp.get("headers", {}).items(): self.send_header(k, v) self.end_headers() - resp_body = resp.get("body", "") - if resp_body: - self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body) + if encoded: + self.wfile.write(encoded) except ConnectionError as e: GatewayState.link = None @@ -144,12 +189,19 @@ class GatewayHandler(BaseHTTPRequestHandler): self._forward("POST") def log_message(self, format, *args): - print(f"[Gateway] {args[0]}") + try: + msg = format % args + except TypeError: + msg = format + # /bookmark carries a long-lived token and the URL being indexed — + # redact the query so it doesn't end up in stdout, journald, docker logs, etc. + msg = re.sub(r'(/bookmark)\?\S*', r'\1?[redacted]', msg) + print(f"[Gateway] {msg}") def main(): if len(sys.argv) < 2: - print(f"Usage: python gateway.pyInvalid or missing CSRF token.
", status=403) + if path == "/add": + return handle_add_submit(body) + elif path == "/pages/bulk": + return handle_bulk_action(body) + elif path == "/add/manual": + return handle_add_manual_submit(body) + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_submit(pid, body) if pid is not None else _error(400) + elif path.startswith("/delete/"): + pid = extract_id("/delete/") + return handle_delete(pid) if pid is not None else _error(400) + elif path == "/style": + return handle_style_submit(body, gateway_host=gateway_host, scheme=scheme) + elif path == "/style/template": + return handle_style_template_submit(body, gateway_host=gateway_host, scheme=scheme) + elif path == "/style/field": + return handle_field_save(body) + elif path == "/style/reset": + set_setting("custom_template", "") + _set_flash("Template reset to default.") + return _redirect("/style") + elif path == "/style/vacuum": + from tinyweb.db import vacuum_db + vacuum_db() + _set_flash("Database vacuumed.") + return _redirect("/style") + elif path == "/import": + return handle_import_submit(body) + elif path == "/reindex": + return handle_reindex_submit(body) + elif path == "/subscriptions/add": + return handle_subscription_add(body) + elif path == "/subscriptions/pick": + return handle_subscription_pick(body) + elif path.startswith("/subscriptions/sync/"): + sid = extract_id("/subscriptions/sync/") + return handle_subscription_sync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/autosync/"): + sid = extract_id("/subscriptions/autosync/") + return handle_subscription_autosync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/delete/"): + sid = extract_id("/subscriptions/delete/") + return handle_subscription_delete(sid) if sid is not None else _error(400) + elif path == "/subscriptions/syncall": + return handle_subscription_syncall() + elif path == "/rns/delete": + return handle_rns_delete_hash(body) + + return _error(404) + + +def dispatch_request(data): + path = data.get("path", "/") + cookies = data.get("cookies", {}) + + if path.startswith("/forum") and forum_plugin and forum_plugin.is_enabled(): + resp = _dispatch_inner(data) + resp.setdefault("headers", {}) + resp["headers"]["X-Frame-Options"] = "DENY" + resp["headers"]["X-Content-Type-Options"] = "nosniff" + if resp.get("content_type", "").startswith("text/html"): + resp["body"] = wrap_page(resp.get("body", "")) + resp["headers"]["Content-Security-Policy"] = ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline'; " + "style-src 'self' 'unsafe-inline'; " + "img-src * data:; " + "frame-ancestors 'none'; " + "form-action 'self'; " + "base-uri 'self'" + ) + return resp + + csrf_token = cookies.get("_csrf", "") + if not csrf_token: + csrf_token = secrets.token_hex(32) + _request_local.csrf_token = csrf_token + + resp = _dispatch_inner(data) + + resp.setdefault("headers", {}) + resp["headers"]["Set-Cookie"] = f"_csrf={csrf_token}; SameSite=Strict; HttpOnly; Path=/" + resp["headers"]["X-Frame-Options"] = "DENY" + resp["headers"]["X-Content-Type-Options"] = "nosniff" + if resp.get("content_type", "").startswith("text/html"): + resp["headers"]["Content-Security-Policy"] = ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline'; " + "style-src 'self' 'unsafe-inline'; " + "img-src * data:; " + "frame-ancestors 'none'; " + "form-action 'self'; " + "base-uri 'self'" + ) + return resp diff --git a/src/tinyweb/handlers/_helpers.py b/src/tinyweb/handlers/_helpers.py new file mode 100644 index 0000000..c7e8d83 --- /dev/null +++ b/src/tinyweb/handlers/_helpers.py @@ -0,0 +1,167 @@ +import json +import re +import secrets +import threading + +from tinyweb.db import get_db, return_db, get_setting, set_setting +from tinyweb.templates import wrap_page + + +_request_local = threading.local() + + +def _get_csrf_token(): + return getattr(_request_local, 'csrf_token', '') + + +def _csrf_field(): + return f'' + + +def _check_csrf(body): + token = body.get("_csrf", [""])[0] + expected = _get_csrf_token() + if not expected or not token: + return False + return secrets.compare_digest(token, expected) + + +_STOPWORDS = frozenset({ + "a", "an", "the", "and", "or", "but", "is", "are", "was", "were", + "in", "on", "at", "to", "for", "of", "with", "by", "from", "as", + "into", "about", "how", "what", "which", "who", "where", "when", + "do", "does", "did", "be", "been", "being", "have", "has", "had", + "it", "its", "this", "that", "not", "no", "so", "if", "can", "will", + "my", "your", "i", "me", "we", "you", "he", "she", "they", +}) + + +def _sanitize_fts_query(query): + words = query.split() + if not words: + return '""' + tokens = [] + last_idx = len(words) - 1 + for i, w in enumerate(words): + cleaned = re.sub(r'["\'\(\)\*\+\-\^~:]', '', w).strip() + if not cleaned: + continue + if cleaned.lower() in _STOPWORDS: + continue + if cleaned.upper() in ("AND", "OR", "NOT", "NEAR"): + continue + if i == last_idx: + tokens.append(f"{cleaned}*") + else: + tokens.append(f'"{cleaned}"') + return " ".join(tokens) if tokens else '""' + + +def _get_bookmark_token(): + token = get_setting("bookmark_token") + if not token: + token = secrets.token_hex(16) + set_setting("bookmark_token", token) + return token + + +def _respond(body_html, status=200, use_default=False, head_html=""): + return { + "status": status, + "content_type": "text/html; charset=utf-8", + "body": wrap_page(body_html, use_default=use_default, head_html=head_html), + "headers": {}, + } + + +def _redirect(location): + if not location.startswith("/") or location.startswith("//"): + location = "/" + return { + "status": 302, + "content_type": "text/html; charset=utf-8", + "body": "", + "headers": {"Location": location}, + } + + +def _json_response(data, status=200, headers=None): + return { + "status": status, + "content_type": "application/json", + "body": json.dumps(data, indent=2), + "headers": headers or {}, + } + + +def _text_response(text, status=200, headers=None): + return { + "status": status, + "content_type": "text/plain", + "body": text, + "headers": headers or {}, + } + + +def _error(status): + return _respond(f"{" | ".join(parts)}
' + + +def _get_page_tags(page_id, db=None): + close = False + if db is None: + db = get_db() + close = True + rows = db.execute( + "SELECT t.name FROM tags t JOIN page_tags pt ON t.id = pt.tag_id " + "WHERE pt.page_id = ? ORDER BY t.name", (page_id,) + ).fetchall() + if close: + return_db(db) + return [r["name"] for r in rows] + + +def _set_page_tags(page_id, tag_string, db=None): + close = False + if db is None: + db = get_db() + close = True + db.execute("DELETE FROM page_tags WHERE page_id = ?", (page_id,)) + for name in (t.strip().lower() for t in tag_string.split(",") if t.strip()): + db.execute("INSERT OR IGNORE INTO tags (name) VALUES (?)", (name,)) + tag_id = db.execute("SELECT id FROM tags WHERE name = ?", (name,)).fetchone()["id"] + db.execute("INSERT OR IGNORE INTO page_tags (page_id, tag_id) VALUES (?, ?)", (page_id, tag_id)) + if close: + db.commit() + return_db(db) + + +def _cleanup_orphaned_tags(db): + db.execute("DELETE FROM tags WHERE id NOT IN (SELECT DISTINCT tag_id FROM page_tags)") diff --git a/src/tinyweb/handlers/customize.py b/src/tinyweb/handlers/customize.py new file mode 100644 index 0000000..959476d --- /dev/null +++ b/src/tinyweb/handlers/customize.py @@ -0,0 +1,367 @@ +from tinyweb.db import get_db, return_db, get_setting, set_setting, get_site_name +import tinyweb.templates as templates_mod +from tinyweb.templates import esc, DEFAULT_TEMPLATE +from ._helpers import _respond, _redirect, _json_response, _csrf_field, _get_bookmark_token, _request_local +from .subscriptions import _count_shared_pages + +_flash = {} + + +def _set_flash(msg): + _flash[_request_local.csrf_token] = msg + + +def _get_flash(): + return _flash.pop(_request_local.csrf_token, "") + + +def handle_style_form(msg="", gateway_host="", scheme="http"): + template = get_setting("custom_template") or DEFAULT_TEMPLATE + name = get_site_name() + sharing = get_setting("sharing_enabled", "0") + checked = " checked" if sharing == "1" else "" + sharing_mode = get_setting("sharing_mode", "exclude_private") + forum = get_setting("forum_enabled", "0") + forum_checked = " checked" if forum == "1" else "" + exclude_checked = " checked" if sharing_mode != "require_public" else "" + require_checked = " checked" if sharing_mode == "require_public" else "" + shared_count = _count_shared_pages() + semantic = get_setting("semantic_search", "0") + semantic_checked = " checked" if semantic == "1" else "" + reranker = get_setting("use_reranker", "0") + reranker_checked = " checked" if reranker == "1" else "" + disabled = "" if semantic == "1" else " disabled" + tcp_enabled = get_setting("tcp_enabled", "1") + tcp_checked = " checked" if tcp_enabled == "1" else "" + tcp_disabled = "" if tcp_enabled == "1" else " disabled" + transport_host = get_setting("transport_host", "rnode.bre.land") + transport_port = get_setting("transport_port", "4242") + compress = get_setting("compress_embeddings", "0") + compress_checked = " checked" if compress == "1" else "" + lora_enabled = get_setting("lora_enabled", "0") + lora_checked = " checked" if lora_enabled == "1" else "" + lora_disabled = "" if lora_enabled == "1" else " disabled" + lora_port = get_setting("lora_port", "") + lora_frequency = get_setting("lora_frequency", "867200000") + lora_bandwidth = get_setting("lora_bandwidth", "125000") + lora_txpower = get_setting("lora_txpower", "7") + lora_sf = get_setting("lora_sf", "8") + lora_cr = get_setting("lora_cr", "5") + csrf = _csrf_field() + from tinyweb.handlers import forum_plugin as _fp + if _fp is not None: + forum_body = ( + f"{msg}
'.format(msg=esc(msg)) + + return _respond( + f"Edit the full page template.
" + f'" + f"Drag this link to your bookmarks bar.
" + f'r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}
' + f"This instance shares its index publicly. Subscribe to join the network.
' + if sharing else + 'This instance is private.
' + ) + + hash_html = "" + if dest_hash: + hash_html = ( + f'To subscribe to this instance, add this destination hash in your TinyWeb:
' + f'{esc(dest_hash)}'
+ )
+
+ return _respond(
+ f'A personal, decentralized search engine.
' + f'You save pages you find. They are stored locally and shared over a mesh network ' + f'so other people can find them too.
' + f'Search results come from your index and the indexes of people you are connected to.
' + f'Everything is stored locally under ~/.tinyweb/:
tinyweb_identity — your permanent mesh identity. '
+ f'If you lose this file, your destination hash changes and subscribers '
+ f'have to re-subscribe to the new one.index.db — your full reading history: every page, '
+ f'note, tag, and synced remote page.models/ — the semantic search model if you enabled it '
+ f'(redownloadable, safe to delete).Back up ~/.tinyweb/ periodically. '
+ f'Copying the whole directory to another device preserves your identity and index together. '
+ f'The export page gives you a JSON dump of pages only — '
+ f'it does not preserve your identity or subscription state, so it is a migration aid, '
+ f'not a substitute for a full backup.
Paste the contents of a tinyweb export file (JSON).
" + f'" + f"{msg}
" + f'back' + ) + + +def handle_import_submit(body): + raw = body.get("data", [""])[0].strip() + if not raw: + return handle_import_form("Paste JSON data.") + try: + data = json.loads(raw) + except json.JSONDecodeError: + return handle_import_form("Invalid JSON.") + if not isinstance(data, list): + return handle_import_form("Expected a JSON array.") + + MAX_IMPORT = 100 + if len(data) > MAX_IMPORT: + return handle_import_form(f"Too many entries. Maximum is {MAX_IMPORT}.") + + imported = 0 + errors = 0 + for entry in data: + url = entry.get("url", "").strip() + note = entry.get("note", "").strip() + if not url: + continue + try: + index_url(url, note) + imported += 1 + except Exception: + errors += 1 + + return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") + + +def handle_reindex_form(): + if get_setting("semantic_search", "0") != "1": + return _respond( + f"Semantic search is disabled. Enable it in settings to use embeddings.
" + f'' + ) + db = get_db() + try: + total_pages = db.execute("SELECT count(*) FROM pages").fetchone()[0] + pages_with_chunks = db.execute( + "SELECT count(DISTINCT page_id) FROM chunks WHERE page_id IS NOT NULL" + ).fetchone()[0] + finally: + return_db(db) + progress = get_setting("reindex_progress", "") + status_html = "" + if progress: + status_html = f'' + elif _reindex_thread and _reindex_thread.is_alive(): + status_html = '' + return _respond( + f"{pages_with_chunks} of {total_pages} pages have embeddings.
" + f'{status_html}' + f'' + f'' + ) + + +def handle_reindex_submit(body): + global _reindex_thread + if _reindex_thread and _reindex_thread.is_alive(): + return handle_reindex_form() + + def _run(): + try: + from tinyweb.embeddings import reindex_all + def progress(current, total): + set_setting("reindex_progress", f"{current}/{total}") + reindex_all(progress_callback=progress) + except Exception: + pass + finally: + set_setting("reindex_progress", "") + + _reindex_thread = threading.Thread(target=_run, daemon=True) + _reindex_thread.start() + return _redirect("/reindex") diff --git a/src/tinyweb/handlers/pages.py b/src/tinyweb/handlers/pages.py new file mode 100644 index 0000000..05d885a --- /dev/null +++ b/src/tinyweb/handlers/pages.py @@ -0,0 +1,408 @@ +from pathlib import Path +import json +import secrets +from urllib.parse import unquote + +from tinyweb.db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url +from tinyweb.templates import esc +from ._helpers import ( + _csrf_field, _respond, _redirect, _error, + _paginate, _page_nav, _get_page_tags, _set_page_tags, _cleanup_orphaned_tags, + _get_bookmark_token, _text_response, + BROWSE_PER_PAGE, +) + + +def handle_add_form(msg="", action_type="index", prefill_url=""): + if action_type == "subscribe": + return _respond( + f"Subscribe to a friend's TinyWeb instance to sync their index
" + f'" + f"" + f"{msg}
" + f'back' + ) + url_value = f'value="{esc(prefill_url)}" ' if prefill_url else "" + return _respond( + f"Add a site to your index — URL or RNS destination hash
" + f'" + f"{msg}
" + f'back' + ) + + +def handle_add_submit(body): + raw = body.get("url", [""])[0].strip().replace("<", "").replace(">", "") + note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + + if not raw: + return handle_add_form("URL or RNS hash is required.") + + is_rns = ( + len(raw) == 32 + and all(c in "0123456789abcdefABCDEF" for c in raw) + ) + if raw.startswith("rns:") or raw.startswith("RNS:"): + raw = raw[4:] + is_rns = ( + len(raw) == 32 + and all(c in "0123456789abcdefABCDEF" for c in raw) + ) + + if is_rns: + from .rns import handle_rns_add_hash + errs = handle_rns_add_hash(raw) + if errs: + return handle_add_form(f"Hash saved but indexing failed: {'; '.join(errs)}") + return _redirect("/") + + url = clean_url(raw) + if not url.startswith(("http://", "https://")): + return handle_add_form("Enter a URL (http:// or https://) or a 32-char RNS destination hash.") + + try: + title = index_url(url, note) + if tags: + db = get_db() + try: + row = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone() + if row: + _set_page_tags(row["id"], tags, db) + db.commit() + finally: + return_db(db) + return handle_add_form(f'Indexed: {esc(url)}') + except ValueError as e: + return handle_add_form(f"Error: {esc(str(e))}") + except Exception as e: + error_msg = str(e).lower() + if any(x in error_msg for x in ("block", "cloudflare", "403", "429", "ssl", "handshake", "max retries", "timeout", "connection")): + return _respond( + f"{esc(url)} blocks automated access. " + f"You can still save it manually:
" + f'" + f'back' + ) + return handle_add_form(f"Error: could not fetch or index that URL. {esc(str(e)[:100])}") + + +def handle_add_manual_submit(body): + url = clean_url(body.get("url", [""])[0].strip()) + note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + manual_title = body.get("manual_title", [""])[0].strip() + manual_desc = body.get("manual_description", [""])[0].strip() + + if not url: + return handle_add_form("URL is required.") + + if not manual_title: + return handle_add_form("Title is required for manual entry.") + + db = get_db() + try: + now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + + db.execute( + "INSERT INTO pages (url, title, body, note, last_modified, summary) VALUES (?, ?, ?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, " + "note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary", + (url, manual_title, manual_desc, note, now, manual_desc[:200]), + ) + + page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0] + + if tags: + _set_page_tags(page_id, tags, db) + + db.commit() + + if get_setting("semantic_search", "0") == "1": + try: + from tinyweb.embeddings import store_embeddings + store_embeddings(page_id, manual_title, manual_desc, db) + db.commit() + except Exception as e: + print(f"Error generating embeddings: {e}") + + return handle_add_form(f'Added manually: {esc(manual_title)}') + finally: + return_db(db) + + +def handle_pages(query=None): + msg = query.get("msg", [""])[0] if query else "" + msg_html = f'{esc(msg)}
' if msg else "" + page = _paginate(query or {}) + offset = (page - 1) * BROWSE_PER_PAGE + db = get_db() + try: + total = db.execute("SELECT count(*) FROM pages").fetchone()[0] + rows = db.execute( + "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?", + (BROWSE_PER_PAGE, offset), + ).fetchall() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tags_html = "" + if tags: + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + tags_html = f' {tag_links}' + url = r["url"] + if url.startswith("rns:"): + display_url = url + link_url = f"/rns/{esc(url[4:])}/" + else: + display_url = url + link_url = url + items += ( + f'Remove the following {n} page{'' if n == 1 else 's'}?
" + f"{esc(row['title'])}
"
+ f"{esc(row['url'])}
{msg}
" + f'back' + ) + + +def handle_edit_submit(page_id, body): + title = body.get("title", [""])[0].strip() + summary = body.get("summary", [""])[0].strip() + note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + + db = get_db() + try: + db.execute( + "UPDATE pages SET title = ?, summary = ?, note = ? WHERE id = ?", + (title, summary, note, page_id) + ) + + _set_page_tags(page_id, tags, db) + _cleanup_orphaned_tags(db) + + db.commit() + + finally: + return_db(db) + + page = body.get("_page", [""])[0].strip() + target = "/pages" + if page: + target += "?p=" + page + return _redirect(target) + + +def handle_delete_confirm(page_id): + db = get_db() + try: + row = db.execute("SELECT id, url, title FROM pages WHERE id = ?", (page_id,)).fetchone() + finally: + return_db(db) + if not row: + return _error(404) + return _respond( + f"Remove {esc(row['title'])}
"
+ f"{esc(row['url'])}
{esc(str(e))}
", + "headers": {}, + } + except PermissionError: + return { + "status": 200, + "content_type": "text/html; charset=utf-8", + "body": "the remote instance blocked this request.
", + "headers": {}, + } + + if resp.get("status") != 200: + return { + "status": 200, + "content_type": "text/html; charset=utf-8", + "body": f"remote returned status {resp['status']}
", + "headers": {}, + } + + body = resp.get("body", "") + + if resp.get("content_type", "").startswith("application/json"): + try: + data = json.loads(body) + body = f"{esc(json.dumps(data, indent=2))}"
+ except (json.JSONDecodeError, TypeError):
+ body = f"{esc(body[:2000])}"
+ else:
+ body = _inject_base_tag(body, dest_hash)
+
+ _page_cache.put(cache_key, body)
+
+ return {
+ "status": 200,
+ "content_type": "text/html; charset=utf-8",
+ "body": body,
+ "headers": {},
+ }
diff --git a/src/tinyweb/handlers/search.py b/src/tinyweb/handlers/search.py
new file mode 100644
index 0000000..10b87fd
--- /dev/null
+++ b/src/tinyweb/handlers/search.py
@@ -0,0 +1,196 @@
+from tinyweb.db import get_db, return_db, get_setting, get_site_name, clean_url
+from tinyweb.templates import esc
+from ._helpers import _sanitize_fts_query, _paginate, _get_page_tags, _respond, _page_nav, PER_PAGE
+
+
+def handle_search(query):
+ q = query.get("q", [""])[0].strip()
+ page = _paginate(query)
+ offset = (page - 1) * PER_PAGE
+ db = get_db()
+ try:
+ count = db.execute("SELECT count(*) FROM pages").fetchone()[0]
+ name = get_site_name()
+
+ result_html = ""
+ trusted_html = ""
+ if q:
+ try:
+ fts_q = _sanitize_fts_query(q)
+ bm25_rows = db.execute(
+ "SELECT p.id, p.url, p.title, p.body, p.note "
+ "FROM pages_fts f JOIN pages p ON f.rowid = p.id "
+ "WHERE pages_fts MATCH ? "
+ "ORDER BY bm25(pages_fts, 10.0, 1.0, 5.0, 3.0) LIMIT 100",
+ (fts_q,),
+ ).fetchall()
+ except Exception:
+ bm25_rows = []
+
+ bm25_ids = [r["id"] for r in bm25_rows]
+ chunk_snippets = {}
+ if get_setting("semantic_search", "0") == "1":
+ try:
+ from tinyweb.embeddings import hybrid_search
+ use_reranker = get_setting("use_reranker", "1") == "1"
+ fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker)
+ fused_ids = [pid for pid, _ in fused]
+ chunk_snippets = {pid: text for pid, text in fused if text}
+ except Exception:
+ fused_ids = bm25_ids
+ else:
+ fused_ids = bm25_ids
+
+ # Also match by tag
+ search_terms = [w.lower() for w in q.split() if w]
+ if search_terms:
+ placeholders = ",".join("?" * len(search_terms))
+ tag_rows = db.execute(
+ f"SELECT DISTINCT pt.page_id FROM page_tags pt "
+ f"JOIN tags t ON t.id = pt.tag_id "
+ f"WHERE LOWER(t.name) IN ({placeholders})",
+ search_terms,
+ ).fetchall()
+ tag_ids = {r["page_id"] for r in tag_rows}
+ seen = set(fused_ids)
+ for pid in tag_ids:
+ if pid not in seen:
+ fused_ids.append(pid)
+ seen.add(pid)
+
+ total_results = len(fused_ids)
+ page_ids = fused_ids[offset:offset + PER_PAGE]
+
+ if page_ids:
+ placeholders = ",".join("?" * len(page_ids))
+ all_rows = db.execute(
+ f"SELECT id, url, title, body, note, summary FROM pages WHERE id IN ({placeholders})",
+ page_ids,
+ ).fetchall()
+ row_map = {r["id"]: r for r in all_rows}
+ rows = [row_map[pid] for pid in page_ids if pid in row_map]
+ else:
+ rows = []
+
+ if rows:
+ for r in rows:
+ note_html = ""
+ if r["note"]:
+ note_html = f'No results in your index.
" + + words = q.lower().split() + all_links = db.execute( + "SELECT l.url, l.label, p.title AS source_title " + "FROM links l JOIN pages p ON l.page_id = p.id", + ).fetchall() + indexed_urls = set(r["url"] for r in rows) if rows else set() + seen = set() + trusted = [] + for l in all_links: + if l["url"] in indexed_urls or l["url"] in seen: + continue + if any(w in l["label"].lower() for w in words): + seen.add(l["url"]) + trusted.append(l) + if len(trusted) >= 20: + break + + if trusted: + items = "" + for l in trusted: + items += ( + f'Your index is empty.
' + 'tinyweb is a personal search engine for pages you save. ' + 'The index stays on your machine; so does every search.
' + 'From here: add a page, ' + 'get the bookmarklet, or ' + 'subscribe to another instance.
' + 'public"
+ if mode == "require_public"
+ else "all pages except those tagged private"
+ )
+ sharing_on = get_setting("sharing_enabled", "0") == "1"
+ status = (
+ 'Sharing is enabled. Subscribers see the pages listed below.
' + if sharing_on else + 'Sharing is disabled. Nothing is actually being shared right now; ' + 'this is the list that would be exposed if you enabled it.
' + ) + db = get_db() + try: + sites = _shared_sites(db) + finally: + return_db(db) + if not sites: + body = ( + "Rule: {mode_label}.
" + f"{status}" + "No pages match the current rule.
" + '' + ) + return _respond(body) + rows = "" + for s in sites: + tags_html = "" + if s["tags"]: + tags_html = " " + " ".join(f"[{esc(t)}]" for t in s["tags"]) + note_html = f' — {esc(s["note"])}' if s["note"] else "" + rows += ( + f'Rule: {mode_label}.
" + f"{status}" + f"{len(sites)} page(s) visible to subscribers.
" + f"{msg}
' + f'{len(sites)} site(s) available, {new_count} new
' + f'' + f'No tags yet. Add tags when saving or editing pages.
" + f'back' + ) + + +def handle_tag_browse(tag_name, query=None): + page = _paginate(query or {}) + offset = (page - 1) * BROWSE_PER_PAGE + db = get_db() + try: + total = db.execute( + "SELECT count(*) FROM page_tags pt JOIN tags t ON t.id = pt.tag_id WHERE t.name = ?", + (tag_name,), + ).fetchone()[0] + rows = db.execute( + "SELECT p.id, p.url, p.title, p.note FROM pages p " + "JOIN page_tags pt ON p.id = pt.page_id " + "JOIN tags t ON t.id = pt.tag_id " + "WHERE t.name = ? ORDER BY p.id DESC LIMIT ? OFFSET ?", + (tag_name, BROWSE_PER_PAGE, offset), + ).fetchall() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + tags = _get_page_tags(r["id"], db) + tag_links = " ".join(f'[{esc(t)}]' for t in tags) + items += ( + f'{total} page(s)
' + f'{name}' + ' | search | browse' + ' | tags | subscriptions' + f'{forum_link}' + ' | customize | about
\n' + "{name}' - ' | search | browse' - ' | tags | subscriptions' - ' | customize | about
\n' - "body content
+ """ + title, body, links, meta = _fetch_with_html(monkeypatch, "https://example.com/", html) + assert meta == "the real description" + + +def test_og_description_fallback(monkeypatch): + """When there's no , og:description wins.""" + html = """ + + +body
+ """ + _, _, _, meta = _fetch_with_html(monkeypatch, "https://example.com/", html) + assert meta == "open graph fallback" diff --git a/tests/test_pagination.py b/tests/test_pagination.py new file mode 100644 index 0000000..6b6f727 --- /dev/null +++ b/tests/test_pagination.py @@ -0,0 +1,58 @@ +"""Tests for `_paginate` and `_page_nav`.""" +from tinyweb.handlers import _paginate, _page_nav, PER_PAGE + + +def test_paginate_default_is_one(): + assert _paginate({}) == 1 + + +def test_paginate_reads_query_string(): + assert _paginate({"p": ["3"]}) == 3 + + +def test_paginate_clamps_to_one(): + assert _paginate({"p": ["0"]}) == 1 + assert _paginate({"p": ["-5"]}) == 1 + + +def test_paginate_handles_bad_input(): + assert _paginate({"p": ["not-a-number"]}) == 1 + assert _paginate({"p": []}) == 1 + + +def test_paginate_custom_key(): + assert _paginate({"batch": ["7"]}, key="batch") == 7 + + +def test_page_nav_empty_when_single_page(): + assert _page_nav(1, PER_PAGE, "/?q=foo") == "" + assert _page_nav(1, 0, "/?q=foo") == "" + + +def test_page_nav_shows_next_on_first_page(): + out = _page_nav(1, PER_PAGE * 3, "/?q=foo") + assert "next" in out + assert "prev" not in out + assert "page 1 of 3" in out + + +def test_page_nav_shows_both_in_middle(): + out = _page_nav(2, PER_PAGE * 3, "/?q=foo") + assert "next" in out + assert "prev" in out + + +def test_page_nav_shows_prev_on_last_page(): + out = _page_nav(3, PER_PAGE * 3, "/?q=foo") + assert "next" not in out + assert "prev" in out + assert "page 3 of 3" in out + + +def test_page_nav_handles_query_string_separator(): + # when base_url already has ?, pagination links must use & + out = _page_nav(1, PER_PAGE * 2, "/?q=foo") + assert "&p=2" in out + # when base_url has no ?, pagination links use ? + out = _page_nav(1, PER_PAGE * 2, "/pages") + assert "?p=2" in out diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..eeab752 --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,107 @@ +"""Aggregator of regression tests tied to specific bug-fix commits. + +Each test here guards against a specific bug that was once shipped. Running +just this file gives a one-line-per-bug audit: + + pytest tests/test_regressions.py -v + +The test bodies are intentionally small; for the exhaustive behavior of each +module, see the topical test files (test_fts_sanitizer.py, test_url_cleanup.py, +etc.). This file's job is to make the bug catalog scannable. +""" +import socket +from unittest.mock import patch + +import pytest + +from tinyweb import app as app_module +import tinyweb.db as db_module +import tinyweb.handlers as handlers_module +from conftest import patch_dns_fail, patch_dns_ok +from tinyweb.db import clean_url +from tinyweb.handlers import _sanitize_fts_query, handle_bulk_action + + +def test_6ffd38d_clean_url_preserves_www_when_bare_domain_fails(monkeypatch): + """6ffd38d: `clean_url` used to strip `www.` unconditionally; for sites that + only serve at `www.`, this produced unreachable clean URLs.""" + patch_dns_fail(monkeypatch) + assert clean_url("https://www.example.com/page") == "https://www.example.com/page" + + +def test_1bc695f_fts_sanitizer_strips_colon(): + """1bc695f: FTS5 colon is a column filter — must not appear in sanitized output.""" + assert ":" not in _sanitize_fts_query("title:secret body:exposed") + + +@pytest.mark.parametrize("op", ["AND", "OR", "NOT", "NEAR"]) +def test_1bc695f_fts_sanitizer_drops_operator_words(op): + """1bc695f: operator words (AND/OR/NOT/NEAR) would be interpreted as FTS5 + operators if they landed on the unquoted last token.""" + out = _sanitize_fts_query(f"foo {op} bar") + # operator itself should not appear in the output + tokens = out.replace('"', '').split() + assert op not in [t.rstrip("*") for t in tokens] + + +def test_1bc695f_gateway_rejects_oversize_body(): + """1bc695f: 16 MiB body-size cap prevents memory-exhaustion DoS.""" + from tests.test_gateway_limits import FakeGatewayHandler + from tinyweb.gateway import MAX_BODY_SIZE + h = FakeGatewayHandler( + path="/add", method="POST", + headers={"Content-Length": str(MAX_BODY_SIZE + 1)}, + ) + h._forward("POST") + assert h._captured["error"] and h._captured["error"][0] == 413 + + +def test_1bc695f_mesh_rejects_non_whitelisted_paths(): + """1bc695f: Reticulum callers are limited to GET /api/sites; CSRF cannot + authenticate mesh callers.""" + resp = app_module.rns_request_handler( + path="/tinyweb", + data={"method": "POST", "path": "/add", "query": {}, "body": {}, "gateway_host": ""}, + request_id="x", link_id="y", remote_identity=None, requested_at=0, + ) + assert resp["status"] == 403 + + +def test_1bc695f_pool_returns_clean_connection(temp_db, monkeypatch): + """1bc695f: uncommitted transactions on a pooled connection used to leak + into the next consumer.""" + from tinyweb.db import get_db, return_db + db = get_db() + db.execute( + "INSERT INTO pages (url, title, body) VALUES (?, ?, ?)", + ("https://leak.example.com/", "should not persist", "body"), + ) + return_db(db) # no commit + db2 = get_db() + try: + urls = {r["url"] for r in db2.execute("SELECT url FROM pages").fetchall()} + finally: + return_db(db2) + assert "https://leak.example.com/" not in urls + + +def test_8dffd8c_bulk_delete_requires_confirmation(seeded_db, csrf_session): + """8dffd8c: bulk delete without confirmed=1 must render a confirm page + instead of deleting — the JS confirm on /pages is a first-line filter only.""" + from tinyweb.db import get_db, return_db + db = get_db() + try: + pid = db.execute("SELECT id FROM pages LIMIT 1").fetchone()["id"] + count_before = db.execute("SELECT count(*) FROM pages").fetchone()[0] + finally: + return_db(db) + + resp = handle_bulk_action({"ids": [str(pid)], "action": ["delete"]}) + assert "confirm delete" in resp["body"].lower() + + db = get_db() + try: + count_after = db.execute("SELECT count(*) FROM pages").fetchone()[0] + finally: + return_db(db) + assert count_before == count_after, "bulk delete ran without confirmation" diff --git a/tests/test_sharing_logic.py b/tests/test_sharing_logic.py new file mode 100644 index 0000000..36dca46 --- /dev/null +++ b/tests/test_sharing_logic.py @@ -0,0 +1,38 @@ +"""Tests for `_page_is_shared`. + +This function decides whether a page is exposed over Reticulum to +subscribers. Getting it wrong means either a privacy leak or silently +hiding pages the user meant to share — both are worth a regression net. +""" +import pytest + +from tinyweb.handlers import _page_is_shared + + +@pytest.mark.parametrize("mode", ["exclude_private", "require_public"]) +def test_private_tag_always_excludes(mode): + """`private` tag overrides every mode — the most important invariant.""" + assert _page_is_shared(["private"], mode) is False + assert _page_is_shared(["public", "private"], mode) is False + + +def test_exclude_private_defaults_to_shared(): + assert _page_is_shared([], "exclude_private") is True + assert _page_is_shared(["random-tag"], "exclude_private") is True + + +def test_require_public_needs_public_tag(): + assert _page_is_shared([], "require_public") is False + assert _page_is_shared(["rust"], "require_public") is False + assert _page_is_shared(["public"], "require_public") is True + + +def test_require_public_still_vetoes_private(): + # public AND private → private wins. + assert _page_is_shared(["public", "private"], "require_public") is False + + +def test_unknown_mode_treated_as_exclude_private(): + """The default mode is 'exclude_private'; unknown modes fall through to it.""" + assert _page_is_shared([], "totally-bogus-mode") is True + assert _page_is_shared(["private"], "totally-bogus-mode") is False diff --git a/tests/test_ssrf.py b/tests/test_ssrf.py new file mode 100644 index 0000000..31eb132 --- /dev/null +++ b/tests/test_ssrf.py @@ -0,0 +1,64 @@ +"""Tests for `_validate_url_target` — SSRF prevention. + +Any URL the app fetches must resolve to a public IP; private/internal/ +loopback addresses must be rejected so attacker-controlled URLs cannot +reach internal services via our HTTP client. +""" +import socket +from unittest.mock import patch + +import pytest + +from tinyweb.db import _validate_url_target + + +def _mock_getaddrinfo(address): + """Return a function suitable as a socket.getaddrinfo replacement.""" + def f(host, port, *args, **kwargs): + family = socket.AF_INET6 if ":" in address else socket.AF_INET + return [(family, socket.SOCK_STREAM, 0, "", (address, port or 80))] + return f + + +@pytest.mark.parametrize("blocked_ip", [ + "127.0.0.1", + "127.1.2.3", + "10.0.0.1", + "10.255.255.255", + "172.16.0.1", + "172.31.255.255", + "192.168.0.1", + "192.168.255.255", + "169.254.169.254", + "0.0.0.0", + "::1", + "fc00::1", + "fe80::1", +]) +def test_blocks_private_and_loopback(monkeypatch, blocked_ip): + monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo(blocked_ip)) + with pytest.raises(ValueError, match="blocked"): + _validate_url_target("https://evil.example.com/internal") + + +def test_allows_public_ipv4(monkeypatch): + monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo("8.8.8.8")) + _validate_url_target("https://dns.example.com/") # does not raise + + +def test_allows_public_ipv6(monkeypatch): + monkeypatch.setattr(socket, "getaddrinfo", _mock_getaddrinfo("2001:4860:4860::8888")) + _validate_url_target("https://v6.example.com/") # does not raise + + +def test_rejects_unresolvable_hostname(monkeypatch): + def boom(*args, **kwargs): + raise socket.gaierror("no such host") + monkeypatch.setattr(socket, "getaddrinfo", boom) + with pytest.raises(ValueError, match="Cannot resolve"): + _validate_url_target("https://does-not-exist.example.com/") + + +def test_rejects_missing_hostname(): + with pytest.raises(ValueError, match="No hostname"): + _validate_url_target("http:///path-only") diff --git a/tests/test_url_cleanup.py b/tests/test_url_cleanup.py new file mode 100644 index 0000000..8ade28b --- /dev/null +++ b/tests/test_url_cleanup.py @@ -0,0 +1,101 @@ +"""Tests for `clean_url` — URL normalization and tracking-param stripping. + +Clean URLs are the deduplication key in the pages table, so any change to +this function can silently cause duplicate rows or mask legitimate saves. +""" +import pytest + +from conftest import patch_dns_ok, patch_dns_fail +from tinyweb.db import clean_url, TRACKING_PARAMS + + +def test_strips_fragment(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com/page#section") == "https://example.com/page" + + +def test_prefers_https(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("http://example.com/page") == "https://example.com/page" + + +def test_lowercases_hostname(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://EXAMPLE.COM/page") == "https://example.com/page" + + +def test_preserves_path_case(monkeypatch): + """Paths are case-sensitive and should not be lowercased.""" + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com/Foo/Bar") == "https://example.com/Foo/Bar" + + +def test_strips_default_https_port(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com:443/page") == "https://example.com/page" + + +@pytest.mark.xfail(reason="clean_url upgrades http->https before the port-default check, " + "so port 80 is not stripped. Minor dedup bug — harmless but worth fixing.") +def test_strips_http_port_80(monkeypatch): + """Expected: http://foo:80 → https://foo (both scheme-upgrade and port-strip). + + Currently fails because scheme is upgraded to https *before* the port check, + so `scheme == "http" and port == 80` is never true by the time the check runs. + """ + patch_dns_ok(monkeypatch) + assert clean_url("http://example.com:80/page") == "https://example.com/page" + + +def test_preserves_non_default_port(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com:8443/page") == "https://example.com:8443/page" + + +def test_strips_trailing_slash(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com/page/") == "https://example.com/page" + + +def test_root_slash_preserved(monkeypatch): + patch_dns_ok(monkeypatch) + assert clean_url("https://example.com/") == "https://example.com/" + + +@pytest.mark.parametrize("param", sorted(TRACKING_PARAMS)) +def test_tracking_params_stripped(monkeypatch, param): + patch_dns_ok(monkeypatch) + result = clean_url(f"https://example.com/page?{param}=value&keep=yes") + assert param not in result + assert "keep=yes" in result + + +def test_strips_www_when_nonwww_resolves(monkeypatch): + """Standard case: strip `www.` prefix to canonicalize.""" + patch_dns_ok(monkeypatch) + assert clean_url("https://www.example.com/page") == "https://example.com/page" + + +def test_preserves_www_when_nonwww_does_not_resolve(monkeypatch): + """Regression for 6ffd38d. + + Some sites only serve their content at `www.domain.tld`; the bare domain + doesn't resolve. Stripping `www.` in that case produced a URL that we could + never actually fetch or dedupe against the real one. + """ + patch_dns_fail(monkeypatch) + assert clean_url("https://www.example.com/page") == "https://www.example.com/page" + + +def test_query_params_sorted_for_stable_ordering(monkeypatch): + """Same URL with different param orderings should produce the same clean URL.""" + patch_dns_ok(monkeypatch) + a = clean_url("https://example.com/page?b=2&a=1") + b = clean_url("https://example.com/page?a=1&b=2") + assert a == b + + +def test_path_and_query_preserved_through_cleanup(monkeypatch): + patch_dns_ok(monkeypatch) + result = clean_url("https://example.com/path/to/page?id=42&utm_source=twitter") + assert result == "https://example.com/path/to/page?id=42" diff --git a/themes/default.html b/themes/default.html new file mode 100644 index 0000000..bda39ec --- /dev/null +++ b/themes/default.html @@ -0,0 +1,307 @@ + + + + + + + + + + +