diff --git a/.runner b/.runner new file mode 100644 index 0000000..f7d4203 --- /dev/null +++ b/.runner @@ -0,0 +1,11 @@ +{ + "WARNING": "This file is automatically generated by forgejo-runner. Do not edit it manually unless you know what you are doing. Removing this file will cause act runner to re-register as a new runner.", + "id": 14, + "uuid": "2f3252bd-5f69-4d3c-8f77-8b54caf784af", + "name": "nixos-runner", + "token": "993ebcbb295c7c58ac35fd5c2ec07cc752c99152", + "address": "https://git.derickphan.com", + "labels": [ + "ubuntu-latest:docker" + ] +} diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 98ade52..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,43 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## What is TinyWeb - -A personal, decentralized search engine built on the Reticulum mesh network. Users curate and search their own index of web pages, share collections over an encrypted mesh, and subscribe to friends' indexes. No algorithms, no tracking. - -## Running - -```bash -pip install -r requirements.txt -python app.py # Starts RNS server + HTTP gateway on 0.0.0.0:8080 -python gateway.py # Run as HTTP gateway to a remote TinyWeb instance -``` - -There are no tests, linter, or build step. - -## Architecture - -Three entry points form a pipeline: - -- **app.py** — Boots Reticulum, loads/creates identity from `tinyweb_identity`, announces on mesh, starts HTTP gateway as a daemon thread, then loops handling RNS requests. -- **gateway.py** — `BaseHTTPRequestHandler` that translates HTTP GET/POST into a request dict and dispatches it. When `local_dispatch` is set (the default when launched from app.py), it calls handlers directly; otherwise it sends requests over a Reticulum link. -- **handlers.py** — Central router (`handle_request`) that pattern-matches the path and calls the appropriate handler. Every handler returns `{"status", "content_type", "body", "headers"}`. - -## Database (db.py → index.db) - -SQLite with FTS5. Schema is initialized and migrated in `init_db()` on every startup. - -Key tables: `pages` (indexed URLs), `links` (extracted same-domain links), `tags`/`page_tags` (many-to-many tagging), `pages_fts` (full-text search via triggers), `subscriptions` (remote instances), `remote_pages`/`remote_pages_fts` (synced content). - -`get_db()` opens a fresh connection each call — no connection pooling. - -## Patterns to follow - -- All HTML output is built as inline strings in handlers.py; there is no template engine. Use `templates.wrap_page(title, body_html)` to wrap content with boilerplate and custom CSS. -- Use `esc()` (html.escape) for all user-supplied content rendered in HTML. -- Handlers receive `(path_segment, ...)` args extracted by the router and return a response dict. -- Tags are stored in a join table; orphaned rows in `tags` can accumulate — always query through `page_tags` for accurate counts. -- Link extraction (`extract_links`) only follows same-domain URLs and skips binary file extensions and Wikipedia special pages. -- URL cleanup: fragments are stripped, tracking params (utm_*, fbclid, gclid, etc.) are removed before storing. -- Settings are stored as key-value pairs in the `settings` table; access via `get_setting(key, default)`. diff --git a/README.md b/README.md index e44c450..71015c0 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ A personal, decentralized search engine built on the [Reticulum](https://reticul - **Custom templates** — Full HTML/CSS/JS template editor to personalize your instance - **Import/export** — JSON-based backup and restore - **Mesh-native** — Works over Reticulum without the internet; encrypted and decentralized by default +- **Forum plugin** — Optional link-sharing discussion board over the mesh (see Forum section below) ## Performance & Scale @@ -50,34 +51,37 @@ Run the downloaded file — no installation required. ## Docker -Pull and run TinyWeb from the container registry: +TinyWeb is distributed as source. Clone the repo, then build and run with Docker Compose: ```bash -docker run -p 8080:8080 registry.derickphan.com/tinyweb:latest +git clone https://git.derickphan.com/lichenblankie/tinyweb.git +cd tinyweb +docker compose up -d ``` -Or with a specific version: - -```bash -docker run -p 8080:8080 registry.derickphan.com/tinyweb:v0.1.0 -``` - -### Docker Compose +The bundled `docker-compose.yml` builds the image from source and persists your data in a named volume: ```yaml services: tinyweb: - image: registry.derickphan.com/tinyweb:latest + build: . ports: - "8080:8080" volumes: - tinyweb-data:/data + restart: unless-stopped volumes: tinyweb-data: ``` -Run with `docker compose up -d`. +After the first build, the image is cached locally and subsequent `docker compose up -d` calls are instant. To update to the latest source: + +```bash +git pull && docker compose up -d --build +``` + +If you're on macOS or need to reach a Reticulum node over TCP, uncomment the `RNS_TCP_HOST` / `RNS_TCP_PORT` block in `docker-compose.yml` and point it at a host running Reticulum. On Linux with LAN auto-discovery, leave it as-is (or switch to `network_mode: host`). ### Storage Estimates @@ -104,6 +108,7 @@ Your data is stored in `~/.tinyweb/`: |------|-------------| | `index.db` | SQLite database with your indexed pages | | `tinyweb_identity` | Your Reticulum identity (keep safe!) | +| `forum.db` | Forum plugin database (only if forum is enabled) | | `models/` | Downloaded AI models for semantic search | | `index.hnsw` | Semantic search index | @@ -115,6 +120,7 @@ Back up the whole `~/.tinyweb/` directory periodically. The two files that matte - **`tinyweb_identity`** is your permanent mesh identity. If you lose it, your destination hash changes and every subscriber has to re-subscribe to the new one. Keep it somewhere you trust; the file is `0600` by default. - **`index.db`** is your full reading history — every page, note, tag, and synced remote page. Losing it loses everything you've curated. +- **`forum.db`** (if the forum plugin is enabled) — all threads, posts, upvotes, and moderation settings. Losing it loses your forum data. `models/` and `index.hnsw` are re-derivable (the model will re-download, and the HNSW index rebuilds from the database on next startup with semantic search enabled) so they don't need to be backed up. @@ -122,13 +128,18 @@ The `/export` page produces a JSON dump of your pages. It's a migration aid — ### Docker -Data is stored in the `/data` volume inside the container. Use a volume mount to persist data: +When you run via `docker compose up` (above), data is stored in the `tinyweb-data` named volume and persists across rebuilds. To inspect or back up: ```bash -docker run -p 8080:8080 -v tinyweb-data:/data registry.derickphan.com/tinyweb:latest +docker compose exec tinyweb ls -la /data +docker compose down # stop without removing the volume ``` -Or with docker-compose (see above) — data persists in the named volume. +To reset everything (destroys your index and identity — back up first): + +```bash +docker compose down -v +``` ### Command line options @@ -168,6 +179,31 @@ This connects over Reticulum and serves the remote instance at `http://localhost 3. **Subscribe** — Add a friend's destination hash on `/subscriptions` to sync their shared index 4. **Customize** — Edit your site name, HTML template, and sharing settings on `/style` +## Forum plugin + +TinyWeb ships with an optional [tinyweb-forum](https://git.derickphan.com/lichenblankie/tinyweb-forum) plugin — a decentralized link-sharing discussion board that runs in-process alongside TinyWeb. + +### Install + +```bash +pip install tinyweb-forum +``` + +Enable it on the `/style` page under "Forum". A "Forum" link will appear in the navigation bar. + +### How it works + +- Threads and posts are stored in `~/.tinyweb/forum.db` (separate from your search index) +- Instances are discovered automatically via mesh announces — no manual setup needed +- Sync is manual by default: click "sync now" on the forum page. Auto-sync every 5 minutes is optional (toggle on moderation page) +- At scale, sync uses epidemic gossip: 20 random peers per cycle, converging globally within ~O(log N) cycles +- Authors are identified by a short pseudonymous identity hash (no accounts, no sign-up) +- Auto-discovery can be disabled in the moderation page +- Threads are auto-pruned after 30 days (configurable, or set to 0 to keep everything) +- Moderation is local: block authors, mute threads, keyword filters, and gossip block lists with peers (auto-block after 3 peer reports) + +For full feature docs, see the [tinyweb-forum README](https://git.derickphan.com/lichenblankie/tinyweb-forum). + ## Project structure ``` @@ -193,6 +229,7 @@ Other hardening measures: - **XSS escaping** — All user-supplied content is HTML-escaped before rendering - **Bookmark authentication** — The bookmarklet endpoint requires a secret token - **Identity file protection** — The Reticulum identity key is restricted to owner-only permissions (0600) +- **Forum caveats** — See [tinyweb-forum Security](https://git.derickphan.com/lichenblankie/tinyweb-forum#security) for forum-specific risks (voluntary retractions, block gossip manipulation, no rate limiting) ## Maintenance diff --git a/app.py b/app.py index b1c4fe6..3830c85 100644 --- a/app.py +++ b/app.py @@ -8,6 +8,8 @@ from http.server import HTTPServer from db import init_db, get_setting, set_setting from handlers import dispatch_request +import handlers as handlers_mod +import templates as templates_mod import gateway from gateway import GatewayState, GatewayHandler @@ -31,6 +33,7 @@ def find_available_port(start=8080, max_attempts=20, host="127.0.0.1"): for port in range(start, start + max_attempts): try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.bind((host, port)) return port except OSError: @@ -97,6 +100,7 @@ def rns_request_handler(path, data, request_id, link_id, remote_identity, reques def start_gateway(reticulum, bind_host="127.0.0.1"): GatewayState.reticulum = reticulum GatewayState.local_dispatch = dispatch_request + HTTPServer.allow_reuse_address = True server = HTTPServer((bind_host, gateway.GATEWAY_PORT), GatewayHandler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() @@ -271,6 +275,22 @@ def main(): allow=RNS.Destination.ALLOW_ALL, ) + # Initialize forum plugin if available + forum = None + try: + from tinyweb_forum import ForumPlugin + from db import get_site_name + forum = ForumPlugin(DATA_DIR, identity, reticulum, site_name=get_site_name()) + if get_setting("forum_enabled", "0") == "1": + forum.enable() + templates_mod.FORUM_ENABLED = True + handlers_mod.forum_plugin = forum + print(f"Forum plugin: {'enabled' if forum.is_enabled() else 'available (enable in settings)'}") + except ImportError: + print("Forum plugin not installed (pip install tinyweb[forum])") + except Exception as e: + print(f"Forum plugin error: {e}") + # Brief delay to ensure all interfaces (especially TCP) are fully ready time.sleep(2) destination.announce() diff --git a/db.py b/db.py index 8a645ab..ec13254 100644 --- a/db.py +++ b/db.py @@ -291,6 +291,24 @@ def init_db(): db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_page ON page_tags(page_id)") db.execute("CREATE INDEX IF NOT EXISTS idx_page_tags_tag ON page_tags(tag_id)") + # Migrate custom_template: replace hardcoded forum link with {{forum_link}} placeholder + cur = db.execute("SELECT value FROM settings WHERE key='custom_template'") + row = cur.fetchone() + if row: + updated = row[0].replace('forum', "{{forum_link}}") + if updated != row[0]: + db.execute("UPDATE settings SET value=? WHERE key='custom_template'", (updated,)) + db.commit() + + # Migrate custom_template: replace hardcoded site name with {{site_name}} placeholder + cur = db.execute("SELECT value FROM settings WHERE key='custom_template'") + row = cur.fetchone() + if row and '{{site_name}}' not in row[0]: + updated = row[0].replace('href="/">tinyweb', 'href="/">{{site_name}}') + if updated != row[0]: + db.execute("UPDATE settings SET value=? WHERE key='custom_template'", (updated,)) + db.commit() + db.execute("PRAGMA journal_mode=WAL") db.execute("PRAGMA synchronous=NORMAL") db.execute("PRAGMA cache_size=-64000") diff --git a/handlers.py b/handlers.py index e47520b..523000f 100644 --- a/handlers.py +++ b/handlers.py @@ -6,9 +6,11 @@ from datetime import datetime from urllib.parse import unquote from db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url +import templates as templates_mod from templates import esc, wrap_page, DEFAULT_TEMPLATE from rns_client import fetch_remote_sites +forum_plugin = None _request_local = threading.local() @@ -360,7 +362,7 @@ def handle_search(query): ) -def handle_add_form(msg="", action_type="index"): +def handle_add_form(msg="", action_type="index", prefill_url=""): if action_type == "subscribe": return _respond( f"

subscribe

" @@ -374,12 +376,13 @@ def handle_add_form(msg="", action_type="index"): f"

{msg}

" f'back' ) + url_value = f'value="{esc(prefill_url)}" ' if prefill_url else "" return _respond( f"

add url

" f"

Add a site to your index

" f'
' f'{_csrf_field()}' - f'

' + f'

' f'

' f'
' f'tag: private to exclude from sharing

' @@ -814,6 +817,8 @@ def handle_style_form(msg=""): sharing = get_setting("sharing_enabled", "0") checked = " checked" if sharing == "1" else "" sharing_mode = get_setting("sharing_mode", "exclude_private") + forum = get_setting("forum_enabled", "0") + forum_checked = " checked" if forum == "1" else "" exclude_checked = " checked" if sharing_mode != "require_public" else "" require_checked = " checked" if sharing_mode == "require_public" else "" shared_count = _count_shared_pages() @@ -840,6 +845,17 @@ def handle_style_form(msg=""): lora_txpower = get_setting("lora_txpower", "7") lora_sf = get_setting("lora_sf", "8") lora_cr = get_setting("lora_cr", "5") + if forum_plugin is not None: + forum_section = ( + f"

forum

" + f'
" + f"Share URLs and discuss them with other TinyWeb instances. " + f"Requires tinyweb-forum — " + f'more info.

' + ) + else: + forum_section = "" return _respond( f"

customize

" f"

name your search engine

" @@ -915,6 +931,7 @@ def handle_style_form(msg=""): f"Saves ~50% on storage for embeddings. Slight quality reduction at large scale.

" f'manage semantic index

' f"" + f"{forum_section}" f"

custom html

" f"

Edit the full page template. Use {esc('{{content}}')} " f"where page content should appear.

" @@ -974,6 +991,27 @@ def handle_style_submit(body): set_setting("lora_txpower", body.get("lora_txpower", ["7"])[0].strip()) set_setting("lora_sf", body.get("lora_sf", ["8"])[0].strip()) set_setting("lora_cr", body.get("lora_cr", ["5"])[0].strip()) + forum_enabled = "1" if body.get("forum_enabled") else "0" + current_forum = get_setting("forum_enabled", "0") + if forum_enabled != current_forum: + if forum_enabled == "1" and forum_plugin is None: + return handle_style_form( + "Forum plugin not installed. Run: pip install tinyweb-forum" + ) + if forum_enabled == "1": + forum_plugin.enable() + try: + forum_plugin.fdb.set_setting("forum_enabled", "1") + except Exception: + pass + else: + forum_plugin.disable() + try: + forum_plugin.fdb.set_setting("forum_enabled", "0") + except Exception: + pass + set_setting("forum_enabled", forum_enabled) + templates_mod.FORUM_ENABLED = (forum_enabled == "1") return handle_style_form("Saved. Restart TinyWeb for mesh network changes to take effect.") @@ -1654,7 +1692,11 @@ def _dispatch_inner(data): return handle_search(query) elif path == "/add": action_type = query.get("type", ["index"])[0] - return handle_add_form(action_type=action_type if action_type == "subscribe" else "index") + prefill_url = query.get("url", [""])[0].strip() + return handle_add_form( + action_type=action_type if action_type == "subscribe" else "index", + prefill_url=prefill_url, + ) elif path == "/pages": return handle_pages(query) elif path.startswith("/edit/"): @@ -1689,7 +1731,15 @@ def _dispatch_inner(data): elif path.startswith("/subscriptions/browse/"): sid = extract_id("/subscriptions/browse/") return handle_subscription_browse(sid) if sid is not None else _error(400) + elif path.startswith("/forum"): + if forum_plugin and forum_plugin.is_enabled(): + return forum_plugin.handle(method, path, query, {}, data.get("cookies", {})) + return _error(404) elif method == "POST": + if path.startswith("/forum"): + if forum_plugin and forum_plugin.is_enabled(): + return forum_plugin.handle(method, path, query, body, data.get("cookies", {})) + return _error(404) if not _check_csrf(body): return _respond("

403 Forbidden

Invalid or missing CSRF token.

", status=403) if path == "/add": @@ -1737,7 +1787,28 @@ def _dispatch_inner(data): def dispatch_request(data): + path = data.get("path", "/") cookies = data.get("cookies", {}) + + # Forum handles its own CSRF — skip main CSRF to avoid cookie conflicts + if path.startswith("/forum") and forum_plugin and forum_plugin.is_enabled(): + resp = _dispatch_inner(data) + resp.setdefault("headers", {}) + resp["headers"]["X-Frame-Options"] = "DENY" + resp["headers"]["X-Content-Type-Options"] = "nosniff" + if resp.get("content_type", "").startswith("text/html"): + resp["body"] = wrap_page(resp.get("body", "")) + resp["headers"]["Content-Security-Policy"] = ( + "default-src 'self'; " + "script-src 'self' 'unsafe-inline'; " + "style-src 'self' 'unsafe-inline'; " + "img-src * data:; " + "frame-ancestors 'none'; " + "form-action 'self'; " + "base-uri 'self'" + ) + return resp + csrf_token = cookies.get("_csrf", "") if not csrf_token: csrf_token = secrets.token_hex(32) diff --git a/requirements.txt b/requirements.txt index 121fd1f..29da454 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +# forum: pip install tinyweb-forum (optional, adds URL discussion board) requests beautifulsoup4 rns diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..d8df32c --- /dev/null +++ b/start.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exec /nix/store/vhgmnrmvvfdiw0kc2xz8px7rvg60lszc-python3-3.13.12-env/bin/python /home/lichenblankie/apps/tinyweb/app.py --bind 0.0.0.0 \ No newline at end of file diff --git a/templates.py b/templates.py index 0dd9975..ff533cc 100644 --- a/templates.py +++ b/templates.py @@ -1,22 +1,24 @@ import html from db import get_setting +FORUM_ENABLED = False def esc(s): return html.escape(str(s)) - DEFAULT_TEMPLATE = "\n\n\n\n\n\n{{content}}\n\n" def _default_template(): name = esc(get_setting("site_name", "tinyweb")) + forum_link = ' | forum' if FORUM_ENABLED else "" return ( '\n\n\n\n\n\n' f'

{name}' ' | search | browse' ' | tags | subscriptions' + f'{forum_link}' ' | customize | about

\n' "
\n{{content}}\n\n" ) @@ -29,4 +31,7 @@ def wrap_page(body_html, use_default=False): template = get_setting("custom_template") or _default_template() if "{{content}}" not in template: template = _default_template() + forum_link = ' forum' if FORUM_ENABLED else "" + template = template.replace("{{forum_link}}", forum_link) + template = template.replace("{{site_name}}", esc(get_setting("site_name", "tinyweb"))) return template.replace("{{content}}", body_html) diff --git a/themes/kodama2.html b/themes/kodama2.html index 96eb963..31bd5fd 100644 --- a/themes/kodama2.html +++ b/themes/kodama2.html @@ -143,6 +143,7 @@ input[type="text"], input[type="url"], + input:not([type]), input[name="q"], input[name="url"], input[name="note"], @@ -328,6 +329,12 @@ label { color: #5a7880; } input[type="checkbox"] { accent-color: #3a6858; } + a.forum-action, a.forum-action-inline { + color: #5a7880; border: 1px solid rgba(40, 70, 65, 0.3); border-radius: 4px; padding: 6px 12px; + transition: background 0.2s, color 0.2s; font-size: 0.85rem; + } + a.forum-action:hover, a.forum-action-inline:hover { color: #90b4ac; background: rgba(10, 22, 25, 0.6); } + a.forum-action-inline { padding: 2px 6px; border: none; } hr { border: none; border-top: 1px solid rgba(30, 55, 50, 0.3); margin: 1rem 0; } small { @@ -368,11 +375,12 @@