diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..799f1c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +tinyweb_identity +index.db diff --git a/app.py b/app.py index dc83ae4..890bf0a 100644 --- a/app.py +++ b/app.py @@ -1,524 +1,57 @@ -import json -import sqlite3 -import html -import requests -from http.server import HTTPServer, BaseHTTPRequestHandler -from urllib.parse import parse_qs, urlparse, urljoin -from bs4 import BeautifulSoup +import os +import time +import RNS -DATABASE = "index.db" +from db import init_db +from handlers import dispatch_request + +APP_NAME = "tinyweb" +ASPECTS = ["server"] +IDENTITY_FILE = "tinyweb_identity" -def get_db(): - db = sqlite3.connect(DATABASE) - db.row_factory = sqlite3.Row - return db +def load_or_create_identity(): + if os.path.isfile(IDENTITY_FILE): + return RNS.Identity.from_file(IDENTITY_FILE) + identity = RNS.Identity() + identity.to_file(IDENTITY_FILE) + return identity -def init_db(): - db = sqlite3.connect(DATABASE) - db.execute( - "CREATE TABLE IF NOT EXISTS pages (" - " id INTEGER PRIMARY KEY AUTOINCREMENT," - " url TEXT UNIQUE NOT NULL," - " title TEXT," - " body TEXT," - " note TEXT DEFAULT ''" - ")" +def rns_request_handler(path, data, request_id, link_id, remote_identity, requested_at): + if data is None: + data = {"method": "GET", "path": "/", "query": {}, "body": {}, "gateway_host": ""} + return dispatch_request(data) + + +def main(): + init_db() + reticulum = RNS.Reticulum() + identity = load_or_create_identity() + + destination = RNS.Destination( + identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, ) - db.execute( - "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " - "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + + destination.register_request_handler( + "/tinyweb", + response_generator=rns_request_handler, + allow=RNS.Destination.ALLOW_ALL, ) - db.execute( - "CREATE TABLE IF NOT EXISTS links (" - " id INTEGER PRIMARY KEY AUTOINCREMENT," - " page_id INTEGER NOT NULL," - " url TEXT NOT NULL," - " label TEXT," - " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" - ")" - ) - db.execute( - "CREATE TABLE IF NOT EXISTS settings (" - " key TEXT PRIMARY KEY," - " value TEXT" - ")" - ) - db.executescript(""" - CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN - INSERT INTO pages_fts(rowid, title, body, url, note) - VALUES (new.id, new.title, new.body, new.url, new.note); - END; - CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN - INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) - VALUES ('delete', old.id, old.title, old.body, old.url, old.note); - END; - CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN - INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) - VALUES ('delete', old.id, old.title, old.body, old.url, old.note); - INSERT INTO pages_fts(rowid, title, body, url, note) - VALUES (new.id, new.title, new.body, new.url, new.note); - END; - """) - db.commit() - db.close() + destination.announce() -SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + print(f"TinyWeb Reticulum server running") + print(f"Destination hash: {RNS.prettyhexrep(destination.hash)}") + print(f"Share this hash with clients to connect via gateway.py") - -def fetch_page(url): - resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) - resp.raise_for_status() - soup = BeautifulSoup(resp.text, "html.parser") - - # extract links before stripping tags - domain = urlparse(url).netloc - seen = set() - links = [] - for a in soup.find_all("a", href=True): - href = urljoin(url, a["href"]).split("#")[0] - parsed = urlparse(href) - if parsed.netloc != domain: - continue - if any(href.lower().endswith(ext) for ext in SKIP_EXT): - continue - if parsed.query or "action=" in href: - continue - path = parsed.path.lower() - if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): - continue - if href in seen or href == url: - continue - seen.add(href) - label = a.get_text(strip=True) or href - links.append((href, label[:200])) - - for tag in soup(["script", "style", "nav", "footer", "header"]): - tag.decompose() - title = soup.title.string.strip() if soup.title and soup.title.string else url - body = soup.get_text(separator=" ", strip=True) - return title, body, links - - -def snippet(text, query, ctx=80): - pos = text.lower().find(query.lower()) - if pos == -1: - return text[:200] - start = max(0, pos - ctx) - end = min(len(text), pos + len(query) + ctx) - return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") - - -def esc(s): - return html.escape(str(s)) - - -def get_setting(key, default=""): - db = get_db() - row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() - db.close() - return row["value"] if row else default - - -def set_setting(key, value): - db = get_db() - db.execute( - "INSERT INTO settings (key, value) VALUES (?, ?) " - "ON CONFLICT(key) DO UPDATE SET value=excluded.value", - (key, value), - ) - db.commit() - db.close() - - -def get_site_name(): - return get_setting("site_name", "tinyweb") - - -def wrap_page(body_html): - css = get_setting("custom_css") - style = f"" if css else "" - return f"{style}{body_html}" - - -class Handler(BaseHTTPRequestHandler): - - def respond(self, body, status=200): - self.send_response(status) - self.send_header("Content-Type", "text/html; charset=utf-8") - self.end_headers() - self.wfile.write(wrap_page(body).encode()) - - def do_GET(self): - parsed = urlparse(self.path) - path = parsed.path - params = parse_qs(parsed.query) - - if path == "/": - self.handle_search(params) - elif path == "/add": - self.handle_add_form() - elif path == "/pages": - self.handle_pages() - elif path.startswith("/delete/"): - self.handle_delete(path) - elif path.startswith("/edit/"): - self.handle_edit_form(path) - elif path == "/style": - self.handle_style_form() - elif path == "/bookmark": - self.handle_bookmark(params) - elif path == "/export": - self.handle_export() - elif path == "/import": - self.handle_import_form() - else: - self.respond("

404

", 404) - - def do_POST(self): - length = int(self.headers.get("Content-Length", 0)) - body = self.rfile.read(length).decode() - params = parse_qs(body) - - if self.path == "/add": - self.handle_add_submit(params) - elif self.path.startswith("/edit/"): - self.handle_edit_submit(self.path, params) - elif self.path == "/style": - self.handle_style_submit(params) - elif self.path == "/import": - self.handle_import_submit(params) - else: - self.respond("

404

", 404) - - def handle_search(self, params): - q = params.get("q", [""])[0].strip() - db = get_db() - count = db.execute("SELECT count(*) FROM pages").fetchone()[0] - name = get_site_name() - - result_html = "" - trusted_html = "" - if q: - rows = db.execute( - "SELECT p.id, p.url, p.title, p.body, p.note " - "FROM pages_fts f JOIN pages p ON f.rowid = p.id " - "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", - (q,), - ).fetchall() - if rows: - for r in rows: - note_html = "" - if r["note"]: - note_html = f'
{esc(r["note"])}
' - result_html += ( - f'
' - f'{esc(r["title"])}
' - f'{esc(r["url"])}
' - f'{esc(snippet(r["body"], q))}' - f'{note_html}' - f'
' - ) - else: - result_html = "

No results in your index.

" - - # search all linked pages from trusted sites - words = q.lower().split() - all_links = db.execute( - "SELECT l.url, l.label, p.title AS source_title " - "FROM links l JOIN pages p ON l.page_id = p.id", - ).fetchall() - indexed_urls = set(r["url"] for r in rows) if rows else set() - seen = set() - trusted = [] - for l in all_links: - if l["url"] in indexed_urls or l["url"] in seen: - continue - if any(w in l["label"].lower() for w in words): - seen.add(l["url"]) - trusted.append(l) - if len(trusted) >= 20: - break - - if trusted: - items = "" - for l in trusted: - items += ( - f'
  • {esc(l["label"])} ' - f'— from {esc(l["source_title"])}
    • ' - ) - trusted_html = ( - f'
    ' - f'from your trusted sites ({len(trusted)})' - f'' - f'
    ' - ) - - db.close() - self.respond( - f'

    {esc(name)}

    ' - f'
    ' - f'' - f' ' - f'
    ' - f'

    {count} page(s) indexed.' - f' + add url' - f' | browse' - f' | customize

    ' - f'
    {result_html}{trusted_html}' - ) - - def handle_add_form(self, msg=""): - self.respond( - f"

    add url

    " - f'
    ' - f'

    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_add_submit(self, params): - url = params.get("url", [""])[0].strip() - note = params.get("note", [""])[0].strip() - if not url: - return self.handle_add_form("URL is required.") - if not url.startswith(("http://", "https://")): - return self.handle_add_form("URL must start with http:// or https://") - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - self.handle_add_form(f'Indexed: {esc(title)}') - except Exception as e: - self.handle_add_form(f"Error: {esc(str(e))}") - - def handle_pages(self): - db = get_db() - rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() - db.close() - items = "" - for r in rows: - note_html = f' — {esc(r["note"])}' if r["note"] else "" - items += ( - f'
  • {esc(r["title"])}{note_html} ' - f'({esc(r["url"])}) ' - f'edit ' - f'remove
    • ' - ) - self.respond( - f"

    indexed pages ({len(rows)})

    " - f"" - f'

    export | import

    ' - f'back' - ) - - def handle_edit_form(self, path, msg=""): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - db = get_db() - row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() - db.close() - if not row: - return self.respond("

    404

    ", 404) - self.respond( - f"

    edit note

    " - f"

    {esc(row['title'])}
    " - f"{esc(row['url'])}

    " - f'
    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_edit_submit(self, path, params): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - note = params.get("note", [""])[0].strip() - db = get_db() - db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) - db.commit() - db.close() - self.send_response(302) - self.send_header("Location", "/pages") - self.end_headers() - - def handle_delete(self, path): - try: - page_id = int(path.split("/")[-1]) - except ValueError: - return self.respond("

    400

    ", 400) - db = get_db() - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) - db.commit() - db.close() - self.send_response(302) - self.send_header("Location", "/pages") - self.end_headers() - - def handle_bookmark(self, params): - url = params.get("url", [""])[0].strip() - if not url or not url.startswith(("http://", "https://")): - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Access-Control-Allow-Origin", "*") - self.end_headers() - self.wfile.write(b"error: invalid url") - return - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, '') " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body", - (url, title, body), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - msg = f"ok: {title}" - except Exception as e: - msg = f"error: {e}" - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Access-Control-Allow-Origin", "*") - self.end_headers() - self.wfile.write(msg.encode()) - - def handle_export(self): - db = get_db() - rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() - db.close() - data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Disposition", "attachment; filename=tinyweb-export.json") - self.end_headers() - self.wfile.write(json.dumps(data, indent=2).encode()) - - def handle_import_form(self, msg=""): - self.respond( - f"

    import

    " - f"

    Paste the contents of a tinyweb export file (JSON).

    " - f'
    ' - f'

    ' - f'' - f"
    " - f"

    {msg}

    " - f'back' - ) - - def handle_import_submit(self, params): - raw = params.get("data", [""])[0].strip() - if not raw: - return self.handle_import_form("Paste JSON data.") - try: - data = json.loads(raw) - except json.JSONDecodeError: - return self.handle_import_form("Invalid JSON.") - if not isinstance(data, list): - return self.handle_import_form("Expected a JSON array.") - - imported = 0 - errors = 0 - for entry in data: - url = entry.get("url", "").strip() - note = entry.get("note", "").strip() - if not url: - continue - try: - title, body, links = fetch_page(url) - db = get_db() - cur = db.execute( - "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", - (url, title, body, note), - ) - page_id = cur.lastrowid - db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) - for href, label in links: - db.execute( - "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", - (page_id, href, label), - ) - db.commit() - db.close() - imported += 1 - except Exception: - errors += 1 - - self.handle_import_form(f"Imported {imported} page(s). {errors} error(s).") - - def handle_style_form(self, msg=""): - css = get_setting("custom_css") - name = get_site_name() - self.respond( - f"

    customize

    " - f"

    name your search engine

    " - f'
    ' - f'

    ' - f"

    custom css

    " - f"

    Some classes you can target:

    " - f"
    "
-            f"body          - page background, font\n"
-            f"h1            - page titles\n"
-            f"input, button - search bar\n"
-            f"a             - links\n"
-            f".result       - each search result\n"
-            f".note         - your notes on results\n"
-            f".trusted      - trusted sites dropdown\n"
-            f"small         - url text\n"
-            f"ul, li        - browse page list"
-            f"
    " - f'

    ' - f'' - f"
    " - f"

    bookmarklet

    " - f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " - f'

    r.text()).then(t=>alert(t)).catch(()=>alert(\'tinyweb not running\')))">+ save to {esc(name)}

    ' - f"

    {msg}

    " - f'back' - ) - - def handle_style_submit(self, params): - css = params.get("css", [""])[0] - name = params.get("site_name", ["tinyweb"])[0].strip() - set_setting("custom_css", css) - set_setting("site_name", name or "tinyweb") - self.handle_style_form("Saved.") + while True: + time.sleep(1) if __name__ == "__main__": - init_db() - print("running on http://0.0.0.0:5001") - HTTPServer(("0.0.0.0", 5001), Handler).serve_forever() + main() diff --git a/db.py b/db.py new file mode 100644 index 0000000..903f824 --- /dev/null +++ b/db.py @@ -0,0 +1,149 @@ +import sqlite3 +import requests +from urllib.parse import urlparse, urljoin +from bs4 import BeautifulSoup + +DATABASE = "index.db" + +SKIP_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".pdf", ".zip", ".mp3", ".mp4", ".css", ".js", ".ico", ".xml", ".json") + + +def get_db(): + db = sqlite3.connect(DATABASE) + db.row_factory = sqlite3.Row + return db + + +def init_db(): + db = sqlite3.connect(DATABASE) + db.execute( + "CREATE TABLE IF NOT EXISTS pages (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " title TEXT," + " body TEXT," + " note TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts " + "USING fts5(title, body, url, note, content=pages, content_rowid=id)" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS links (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " page_id INTEGER NOT NULL," + " url TEXT NOT NULL," + " label TEXT," + " FOREIGN KEY (page_id) REFERENCES pages(id) ON DELETE CASCADE" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS settings (" + " key TEXT PRIMARY KEY," + " value TEXT" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS subscriptions (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " url TEXT UNIQUE NOT NULL," + " name TEXT DEFAULT ''," + " auto_sync INTEGER DEFAULT 0," + " last_sync TEXT DEFAULT ''" + ")" + ) + db.executescript(""" + CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages BEGIN + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + END; + CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages BEGIN + INSERT INTO pages_fts(pages_fts, rowid, title, body, url, note) + VALUES ('delete', old.id, old.title, old.body, old.url, old.note); + INSERT INTO pages_fts(rowid, title, body, url, note) + VALUES (new.id, new.title, new.body, new.url, new.note); + END; + """) + db.commit() + db.close() + + +def get_setting(key, default=""): + db = get_db() + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + db.close() + return row["value"] if row else default + + +def set_setting(key, value): + db = get_db() + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + db.close() + + +def get_site_name(): + return get_setting("site_name", "tinyweb") + + +def fetch_page(url): + resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, verify=False) + resp.raise_for_status() + soup = BeautifulSoup(resp.text, "html.parser") + + # extract links before stripping tags + domain = urlparse(url).netloc + seen = set() + links = [] + for a in soup.find_all("a", href=True): + href = urljoin(url, a["href"]).split("#")[0] + parsed = urlparse(href) + if parsed.netloc != domain: + continue + if any(href.lower().endswith(ext) for ext in SKIP_EXT): + continue + if parsed.query or "action=" in href: + continue + path = parsed.path.lower() + if any(s in path for s in ("/special:", "/talk:", "/user:", "/wikipedia:", "/help:", "/portal:", "/file:", "/category:")): + continue + if href in seen or href == url: + continue + seen.add(href) + label = a.get_text(strip=True) or href + links.append((href, label[:200])) + + for tag in soup(["script", "style", "nav", "footer", "header"]): + tag.decompose() + title = soup.title.string.strip() if soup.title and soup.title.string else url + body = soup.get_text(separator=" ", strip=True) + return title, body, links + + +def index_url(url, note=""): + title, body, links = fetch_page(url) + db = get_db() + cur = db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, note=excluded.note", + (url, title, body, note), + ) + page_id = cur.lastrowid + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + for href, label in links: + db.execute( + "INSERT INTO links (page_id, url, label) VALUES (?, ?, ?)", + (page_id, href, label), + ) + db.commit() + db.close() + return title diff --git a/gateway.py b/gateway.py new file mode 100644 index 0000000..0bde2d3 --- /dev/null +++ b/gateway.py @@ -0,0 +1,149 @@ +import sys +import time +import threading +import RNS +from http.server import HTTPServer, BaseHTTPRequestHandler +from urllib.parse import parse_qs, urlparse + +APP_NAME = "tinyweb" +ASPECTS = ["server"] +GATEWAY_PORT = 8080 +REQUEST_TIMEOUT = 60 + + +class GatewayState: + reticulum = None + destination = None + link = None + link_lock = threading.Lock() + + +def resolve_destination(dest_hash_hex): + dest_hash = bytes.fromhex(dest_hash_hex) + + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + print(f"Requesting path to {RNS.prettyhexrep(dest_hash)}...") + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + raise ConnectionError(f"Could not find path to {RNS.prettyhexrep(dest_hash)}") + + server_identity = RNS.Identity.recall(dest_hash) + GatewayState.destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + APP_NAME, + *ASPECTS, + ) + print(f"Resolved destination: {RNS.prettyhexrep(dest_hash)}") + + +def ensure_link(): + with GatewayState.link_lock: + if GatewayState.link and GatewayState.link.status == RNS.Link.ACTIVE: + return GatewayState.link + + print("Establishing link...") + link = RNS.Link(GatewayState.destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + + if link.status != RNS.Link.ACTIVE: + raise ConnectionError("Link establishment failed") + + GatewayState.link = link + print("Link established") + return link + + +class GatewayHandler(BaseHTTPRequestHandler): + + def _forward(self, method): + parsed = urlparse(self.path) + query = parse_qs(parsed.query) + + body = {} + if method == "POST": + length = int(self.headers.get("Content-Length", 0)) + raw = self.rfile.read(length).decode() + body = parse_qs(raw) + + request_data = { + "method": method, + "path": parsed.path, + "query": query, + "body": body, + "gateway_host": self.headers.get("Host", f"localhost:{GATEWAY_PORT}"), + } + + try: + link = ensure_link() + receipt = link.request( + "/tinyweb", + data=request_data, + timeout=REQUEST_TIMEOUT, + ) + + # Wait for the response + elapsed = 0 + done_statuses = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done_statuses and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + self.send_response(resp["status"]) + self.send_header("Content-Type", resp.get("content_type", "text/html; charset=utf-8")) + for k, v in resp.get("headers", {}).items(): + self.send_header(k, v) + self.end_headers() + resp_body = resp.get("body", "") + if resp_body: + self.wfile.write(resp_body.encode() if isinstance(resp_body, str) else resp_body) + elif receipt.get_status() == RNS.RequestReceipt.FAILED: + self.send_error(504, "Request to TinyWeb server failed") + else: + self.send_error(504, "Request to TinyWeb server timed out") + + except ConnectionError as e: + GatewayState.link = None + self.send_error(502, f"Gateway error: {e}") + except Exception as e: + GatewayState.link = None + self.send_error(502, f"Gateway error: {e}") + + def do_GET(self): + self._forward("GET") + + def do_POST(self): + self._forward("POST") + + def log_message(self, format, *args): + print(f"[Gateway] {args[0]}") + + +def main(): + if len(sys.argv) < 2: + print(f"Usage: python gateway.py ") + print(f" The destination hash is printed by app.py on startup.") + sys.exit(1) + + dest_hash = sys.argv[1].replace("<", "").replace(">", "") + + GatewayState.reticulum = RNS.Reticulum() + resolve_destination(dest_hash) + + print(f"Gateway listening on http://localhost:{GATEWAY_PORT}") + print(f"Open http://localhost:{GATEWAY_PORT} in your browser") + HTTPServer(("127.0.0.1", GATEWAY_PORT), GatewayHandler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/handlers.py b/handlers.py new file mode 100644 index 0000000..8f17ee8 --- /dev/null +++ b/handlers.py @@ -0,0 +1,660 @@ +import json +from datetime import datetime +import requests + +from db import get_db, get_setting, set_setting, get_site_name, index_url +from templates import esc, snippet, wrap_page + + +def _respond(body_html, status=200): + return { + "status": status, + "content_type": "text/html; charset=utf-8", + "body": wrap_page(body_html), + "headers": {}, + } + + +def _redirect(location): + return { + "status": 302, + "content_type": "text/html; charset=utf-8", + "body": "", + "headers": {"Location": location}, + } + + +def _json_response(data, status=200, headers=None): + return { + "status": status, + "content_type": "application/json", + "body": json.dumps(data, indent=2), + "headers": headers or {}, + } + + +def _text_response(text, status=200, headers=None): + return { + "status": status, + "content_type": "text/plain", + "body": text, + "headers": headers or {}, + } + + +def _error(status): + return _respond(f"

    {status}

    ", status) + + +# --- Route handlers --- + + +def handle_search(query): + q = query.get("q", [""])[0].strip() + db = get_db() + count = db.execute("SELECT count(*) FROM pages").fetchone()[0] + name = get_site_name() + + result_html = "" + trusted_html = "" + if q: + rows = db.execute( + "SELECT p.id, p.url, p.title, p.body, p.note " + "FROM pages_fts f JOIN pages p ON f.rowid = p.id " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT 50", + (q,), + ).fetchall() + if rows: + for r in rows: + note_html = "" + if r["note"]: + note_html = f'
    {esc(r["note"])}
    ' + result_html += ( + f'
    ' + f'{esc(r["title"])}
    ' + f'{esc(r["url"])}
    ' + f'{esc(snippet(r["body"], q))}' + f'{note_html}' + f'
    ' + ) + else: + result_html = "

    No results in your index.

    " + + # search all linked pages from trusted sites + words = q.lower().split() + all_links = db.execute( + "SELECT l.url, l.label, p.title AS source_title " + "FROM links l JOIN pages p ON l.page_id = p.id", + ).fetchall() + indexed_urls = set(r["url"] for r in rows) if rows else set() + seen = set() + trusted = [] + for l in all_links: + if l["url"] in indexed_urls or l["url"] in seen: + continue + if any(w in l["label"].lower() for w in words): + seen.add(l["url"]) + trusted.append(l) + if len(trusted) >= 20: + break + + if trusted: + items = "" + for l in trusted: + items += ( + f'
  • {esc(l["label"])} ' + f'— from {esc(l["source_title"])}
    • ' + ) + trusted_html = ( + f'
    ' + f'from your trusted sites ({len(trusted)})' + f'' + f'
    ' + ) + + db.close() + return _respond( + f'

    {esc(name)}

    ' + f'
    ' + f'' + f' ' + f'
    ' + f'

    {count} page(s) indexed.' + f' + add url' + f' | browse' + f' | subscriptions' + f' | customize

    ' + f'
    {result_html}{trusted_html}' + ) + + +def handle_add_form(msg=""): + return _respond( + f"

    add url

    " + f'
    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_add_submit(body): + url = body.get("url", [""])[0].strip() + note = body.get("note", [""])[0].strip() + if not url: + return handle_add_form("URL is required.") + if not url.startswith(("http://", "https://")): + return handle_add_form("URL must start with http:// or https://") + try: + title = index_url(url, note) + return handle_add_form(f'Indexed: {esc(title)}') + except Exception as e: + return handle_add_form(f"Error: {esc(str(e))}") + + +def handle_pages(): + db = get_db() + rows = db.execute("SELECT id, url, title, note FROM pages ORDER BY id DESC").fetchall() + db.close() + items = "" + for r in rows: + note_html = f' — {esc(r["note"])}' if r["note"] else "" + items += ( + f'
  • {esc(r["title"])}{note_html} ' + f'({esc(r["url"])}) ' + f'edit ' + f'remove
    • ' + ) + return _respond( + f"

    indexed pages ({len(rows)})

    " + f"" + f'

    export | import

    ' + f'back' + ) + + +def handle_edit_form(page_id, msg=""): + db = get_db() + row = db.execute("SELECT id, url, title, note FROM pages WHERE id = ?", (page_id,)).fetchone() + db.close() + if not row: + return _error(404) + return _respond( + f"

    edit note

    " + f"

    {esc(row['title'])}
    " + f"{esc(row['url'])}

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_edit_submit(page_id, body): + note = body.get("note", [""])[0].strip() + db = get_db() + db.execute("UPDATE pages SET note = ? WHERE id = ?", (note, page_id)) + db.commit() + db.close() + return _redirect("/pages") + + +def handle_delete(page_id): + db = get_db() + db.execute("DELETE FROM links WHERE page_id = ?", (page_id,)) + db.execute("DELETE FROM pages WHERE id = ?", (page_id,)) + db.commit() + db.close() + return _redirect("/pages") + + +def handle_bookmark(query): + url = query.get("url", [""])[0].strip() + if not url or not url.startswith(("http://", "https://")): + return _text_response("error: invalid url", headers={"Access-Control-Allow-Origin": "*"}) + try: + title = index_url(url) + msg = f"ok: {title}" + except Exception as e: + msg = f"error: {e}" + return _text_response(msg, headers={"Access-Control-Allow-Origin": "*"}) + + +def handle_export(): + db = get_db() + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id").fetchall() + db.close() + data = [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows] + return _json_response(data, headers={"Content-Disposition": "attachment; filename=tinyweb-export.json"}) + + +def handle_import_form(msg=""): + return _respond( + f"

    import

    " + f"

    Paste the contents of a tinyweb export file (JSON).

    " + f'
    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + +def handle_import_submit(body): + raw = body.get("data", [""])[0].strip() + if not raw: + return handle_import_form("Paste JSON data.") + try: + data = json.loads(raw) + except json.JSONDecodeError: + return handle_import_form("Invalid JSON.") + if not isinstance(data, list): + return handle_import_form("Expected a JSON array.") + + imported = 0 + errors = 0 + for entry in data: + url = entry.get("url", "").strip() + note = entry.get("note", "").strip() + if not url: + continue + try: + index_url(url, note) + imported += 1 + except Exception: + errors += 1 + + return handle_import_form(f"Imported {imported} page(s). {errors} error(s).") + + +def handle_style_form(msg="", gateway_host=""): + css = get_setting("custom_css") + name = get_site_name() + sharing = get_setting("sharing_enabled", "0") + checked = " checked" if sharing == "1" else "" + host = gateway_host or "localhost:8080" + return _respond( + f"

    customize

    " + f"

    name your search engine

    " + f'
    ' + f'

    ' + f"

    sharing

    " + f'

    " + f"

    custom css

    " + f"

    Some classes you can target:

    " + f"
    "
+        f"body          - page background, font\n"
+        f"h1            - page titles\n"
+        f"input, button - search bar\n"
+        f"a             - links\n"
+        f".result       - each search result\n"
+        f".note         - your notes on results\n"
+        f".trusted      - trusted sites dropdown\n"
+        f"small         - url text\n"
+        f"ul, li        - browse page list"
+        f"
    " + f'

    ' + f'' + f"
    " + f"

    bookmarklet

    " + f"

    Drag this link to your bookmarks bar. Click it on any page to index it instantly.

    " + f'

    + save to {esc(name)}

    ' + f"

    {msg}

    " + f'back' + ) + + +def handle_style_submit(body): + css = body.get("css", [""])[0] + name = body.get("site_name", ["tinyweb"])[0].strip() + sharing = "1" if body.get("sharing_enabled") else "0" + set_setting("custom_css", css) + set_setting("site_name", name or "tinyweb") + set_setting("sharing_enabled", sharing) + return handle_style_form("Saved.") + + +def handle_api_sites(): + if get_setting("sharing_enabled", "0") != "1": + return _json_response( + {"error": "sharing disabled"}, + status=403, + headers={"Access-Control-Allow-Origin": "*"}, + ) + db = get_db() + rows = db.execute("SELECT url, title, note FROM pages ORDER BY id DESC").fetchall() + db.close() + data = { + "name": get_site_name(), + "sites": [{"url": r["url"], "title": r["title"], "note": r["note"]} for r in rows], + } + return _json_response(data, headers={"Access-Control-Allow-Origin": "*"}) + + +def handle_subscriptions(msg=""): + db = get_db() + subs = db.execute("SELECT * FROM subscriptions ORDER BY id DESC").fetchall() + db.close() + items = "" + for s in subs: + auto_label = "on" if s["auto_sync"] else "off" + last = s["last_sync"] or "never" + items += ( + f'' + f'{esc(s["name"] or "unknown")}
    {esc(s["url"])}' + f'{esc(last)}' + f'' + f'
    ' + f'
    ' + f'' + f'' + f'browse ' + f'
    ' + f'
    ' + f'
    ' + f'
    ' + f'' + f'' + ) + table = "" + if subs: + table = ( + f'' + f'{items}
    instancelast syncauto-syncactions
    ' + f'
    ' + f'
    ' + ) + return _respond( + f"

    subscriptions

    " + f'
    ' + f' ' + f'' + f'
    ' + f'

    {msg}

    ' + f'
    {table}' + f'
    back' + ) + + +def handle_subscription_add(body): + url = body.get("url", [""])[0].strip().rstrip("/") + if not url or not url.startswith(("http://", "https://")): + return handle_subscriptions("URL must start with http:// or https://") + try: + resp = requests.get(f"{url}/api/sites", timeout=5) + if resp.status_code == 403: + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + data = resp.json() + name = data.get("name", "") + except Exception as e: + return handle_subscriptions(f"Could not reach that instance: {esc(str(e))}") + db = get_db() + try: + db.execute( + "INSERT INTO subscriptions (url, name) VALUES (?, ?) " + "ON CONFLICT(url) DO UPDATE SET name=excluded.name", + (url, name), + ) + db.commit() + finally: + db.close() + return handle_subscriptions(f"Subscribed to {esc(name or url)}.") + + +def handle_subscription_browse(sub_id): + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + if not sub: + db.close() + return _error(404) + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + db.close() + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code == 403: + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + sites = resp.json().get("sites", []) + except Exception as e: + return handle_subscriptions(f"Could not fetch sites: {esc(str(e))}") + + new_items = "" + existing_items = "" + new_count = 0 + for s in sites: + if s["url"] in local_urls: + existing_items += ( + f'
  • {esc(s["title"])} ' + f'({esc(s["url"])}) — already indexed
    • ' + ) + else: + new_count += 1 + note_html = f' — {esc(s["note"])}' if s.get("note") else "" + new_items += ( + f'
    • ' + ) + + buttons = "" + if new_count: + buttons = ' ' + return _respond( + f'

    browsing: {esc(sub["name"] or sub["url"])}

    ' + f'

    {len(sites)} site(s) available, {new_count} new

    ' + f'
    ' + f'' + f'' + f'{buttons}' + f'
    ' + f'

    already indexed

    ' + f'back' + ) + + +def handle_subscription_pick(body): + sub_id = body.get("sub_id", [""])[0] + import_all = body.get("import_all", [""])[0] + + if import_all: + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + db.close() + if not sub: + return handle_subscriptions("Subscription not found.") + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + resp.raise_for_status() + sites = resp.json().get("sites", []) + except Exception as e: + return handle_subscriptions(f"Error: {esc(str(e))}") + urls = [s["url"] for s in sites if s["url"] not in local_urls] + else: + urls = body.get("urls", []) + + if not urls: + return handle_subscriptions("No sites selected.") + + imported = 0 + errors = 0 + for url in urls: + try: + index_url(url) + imported += 1 + except Exception: + errors += 1 + return handle_subscriptions(f"Imported {imported} page(s). {errors} error(s).") + + +def handle_subscription_sync(sub_id): + db = get_db() + sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() + if not sub: + db.close() + return handle_subscriptions("Subscription not found.") + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code == 403: + db.close() + return handle_subscriptions("That instance has sharing disabled.") + resp.raise_for_status() + data = resp.json() + sites = data.get("sites", []) + remote_name = data.get("name", sub["name"]) + except Exception as e: + db.close() + return handle_subscriptions(f"Could not sync: {esc(str(e))}") + + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + synced = 0 + for s in sites: + if s["url"] in local_urls: + continue + try: + db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", + (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + ) + synced += 1 + except Exception: + pass + now = datetime.now().strftime("%Y-%m-%d %H:%M") + db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id)) + db.commit() + db.close() + return handle_subscriptions(f"Synced {synced} new site(s) from {esc(remote_name)}.") + + +def handle_subscription_autosync(sub_id): + db = get_db() + db.execute("UPDATE subscriptions SET auto_sync = 1 - auto_sync WHERE id = ?", (sub_id,)) + db.commit() + db.close() + return _redirect("/subscriptions") + + +def handle_subscription_delete(sub_id): + db = get_db() + db.execute("DELETE FROM subscriptions WHERE id = ?", (sub_id,)) + db.commit() + db.close() + return _redirect("/subscriptions") + + +def handle_subscription_syncall(): + db = get_db() + subs = db.execute("SELECT * FROM subscriptions WHERE auto_sync = 1").fetchall() + db.close() + if not subs: + return handle_subscriptions("No subscriptions have auto-sync enabled.") + total = 0 + for sub in subs: + try: + resp = requests.get(f"{sub['url']}/api/sites", timeout=5) + if resp.status_code != 200: + continue + data = resp.json() + sites = data.get("sites", []) + remote_name = data.get("name", sub["name"]) + db = get_db() + local_urls = set(r["url"] for r in db.execute("SELECT url FROM pages").fetchall()) + for s in sites: + if s["url"] in local_urls: + continue + try: + db.execute( + "INSERT INTO pages (url, title, body, note) VALUES (?, ?, ?, ?)", + (s["url"], s["title"], f"[synced from {remote_name}]", s.get("note", "")), + ) + except Exception: + pass + now = datetime.now().strftime("%Y-%m-%d %H:%M") + db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub["id"])) + db.commit() + db.close() + total += 1 + except Exception: + pass + return handle_subscriptions(f"Synced {total} subscription(s).") + + +# --- Dispatcher --- + + +def dispatch_request(data): + method = data.get("method", "GET") + path = data.get("path", "/") + query = data.get("query", {}) + body = data.get("body", {}) + gateway_host = data.get("gateway_host", "") + + def extract_id(prefix): + try: + return int(path[len(prefix):]) + except (ValueError, IndexError): + return None + + if method == "GET": + if path == "/": + return handle_search(query) + elif path == "/add": + return handle_add_form() + elif path == "/pages": + return handle_pages() + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_form(pid) if pid is not None else _error(400) + elif path.startswith("/delete/"): + pid = extract_id("/delete/") + return handle_delete(pid) if pid is not None else _error(400) + elif path == "/bookmark": + return handle_bookmark(query) + elif path == "/style": + return handle_style_form(gateway_host=gateway_host) + elif path == "/export": + return handle_export() + elif path == "/import": + return handle_import_form() + elif path == "/api/sites": + return handle_api_sites() + elif path == "/subscriptions": + return handle_subscriptions() + elif path.startswith("/subscriptions/browse/"): + sid = extract_id("/subscriptions/browse/") + return handle_subscription_browse(sid) if sid is not None else _error(400) + elif method == "POST": + if path == "/add": + return handle_add_submit(body) + elif path.startswith("/edit/"): + pid = extract_id("/edit/") + return handle_edit_submit(pid, body) if pid is not None else _error(400) + elif path == "/style": + return handle_style_submit(body) + elif path == "/import": + return handle_import_submit(body) + elif path == "/subscriptions/add": + return handle_subscription_add(body) + elif path == "/subscriptions/pick": + return handle_subscription_pick(body) + elif path.startswith("/subscriptions/sync/"): + sid = extract_id("/subscriptions/sync/") + return handle_subscription_sync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/autosync/"): + sid = extract_id("/subscriptions/autosync/") + return handle_subscription_autosync(sid) if sid is not None else _error(400) + elif path.startswith("/subscriptions/delete/"): + sid = extract_id("/subscriptions/delete/") + return handle_subscription_delete(sid) if sid is not None else _error(400) + elif path == "/subscriptions/syncall": + return handle_subscription_syncall() + + return _error(404) diff --git a/requirements.txt b/requirements.txt index 1190bd8..f63da5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests beautifulsoup4 +rns diff --git a/templates.py b/templates.py new file mode 100644 index 0000000..735a38e --- /dev/null +++ b/templates.py @@ -0,0 +1,21 @@ +import html +from db import get_setting + + +def esc(s): + return html.escape(str(s)) + + +def snippet(text, query, ctx=80): + pos = text.lower().find(query.lower()) + if pos == -1: + return text[:200] + start = max(0, pos - ctx) + end = min(len(text), pos + len(query) + ctx) + return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") + + +def wrap_page(body_html): + css = get_setting("custom_css") + style = f"" if css else "" + return f"{style}{body_html}"