From a89edde2de6f2229e00e4d39b4fe2961398a9c8e Mon Sep 17 00:00:00 2001 From: blankie Date: Thu, 4 Jun 2026 08:23:51 +0000 Subject: [PATCH 01/56] initial: decentralized link-sharing forum for TinyWeb --- .gitignore | 5 + README.md | 25 ++ pyproject.toml | 19 ++ tinyweb_forum/__init__.py | 34 +++ tinyweb_forum/db.py | 321 +++++++++++++++++++++++ tinyweb_forum/handlers.py | 538 ++++++++++++++++++++++++++++++++++++++ tinyweb_forum/sync.py | 146 +++++++++++ 7 files changed, 1088 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 tinyweb_forum/__init__.py create mode 100644 tinyweb_forum/db.py create mode 100644 tinyweb_forum/handlers.py create mode 100644 tinyweb_forum/sync.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a376320 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +*.pyo +dist/ +*.egg-info/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..4508429 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# tinyweb-forum + +A decentralized link-sharing forum for [TinyWeb](https://github.com/derickfay/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. + +## Install + +```bash +pip install tinyweb-forum +``` + +Enable the forum in TinyWeb's customize page (`/style`). + +## Development + +```bash +git clone https://github.com/derickfay/tinyweb-forum +pip install -e . +``` + +## How it works + +- Each TinyWeb instance stores forum threads and posts in its own `forum.db` +- Instances sync content with each other over RNS +- Moderation is per-instance: block instances, mute threads, keyword filters +- No global server, no algorithms, no tracking diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..19a0bb7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.backends._legacy:_Backend" + +[project] +name = "tinyweb-forum" +version = "0.1.0" +description = "Decentralized link-sharing forum for TinyWeb" +license = {text = "MIT"} +requires-python = ">=3.9" +dependencies = [ + "rns", +] + +[project.optional-dependencies] +dev = [] + +[tool.setuptools.packages.find] +include = ["tinyweb_forum*"] diff --git a/tinyweb_forum/__init__.py b/tinyweb_forum/__init__.py new file mode 100644 index 0000000..ffdbc9c --- /dev/null +++ b/tinyweb_forum/__init__.py @@ -0,0 +1,34 @@ +from tinyweb_forum.db import ForumDB +from tinyweb_forum.handlers import ForumHandlers +from tinyweb_forum.sync import ForumSync + +FORUM_ENABLED_KEY = "forum_enabled" + + +class ForumPlugin: + def __init__(self, data_dir, identity, reticulum, site_name="me"): + self.fdb = ForumDB(data_dir) + self.handlers = ForumHandlers( + self.fdb, None, identity, reticulum, site_name=site_name + ) + self.sync = ForumSync(self.fdb, identity, reticulum, lambda: self.handlers) + self.handlers.sync = self.sync + self.identity = identity + self.reticulum = reticulum + self._started = False + + def is_enabled(self): + return self.fdb.get_setting(FORUM_ENABLED_KEY, "0") == "1" + + def enable(self): + self.fdb.set_setting(FORUM_ENABLED_KEY, "1") + if not self._started: + self.sync.start() + self._started = True + + def disable(self): + self.fdb.set_setting(FORUM_ENABLED_KEY, "0") + # Keep sync running so we still receive content — disable just hides UI + + def handle(self, method, path, query, body, cookies=None): + return self.handlers.handle(method, path, query, body, cookies) diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py new file mode 100644 index 0000000..a38ad5c --- /dev/null +++ b/tinyweb_forum/db.py @@ -0,0 +1,321 @@ +import sqlite3 +import os +import threading + +FORUM_DB = "forum.db" + + +class ForumDB: + def __init__(self, data_dir): + self.path = os.path.join(data_dir, FORUM_DB) + self._pool = [] + self._pool_lock = threading.Lock() + self._POOL_SIZE = 8 + self.init_db() + + def init_db(self): + os.makedirs(os.path.dirname(self.path), exist_ok=True) + db = sqlite3.connect(self.path) + db.execute( + "CREATE TABLE IF NOT EXISTS threads (" + " id TEXT PRIMARY KEY," + " title TEXT NOT NULL," + " url TEXT DEFAULT ''," + " body TEXT DEFAULT ''," + " tags TEXT DEFAULT ''," + " author_instance TEXT NOT NULL," + " author_name TEXT DEFAULT ''," + " created_at TEXT NOT NULL," + " updated_at TEXT NOT NULL," + " score INTEGER DEFAULT 0" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS posts (" + " id TEXT PRIMARY KEY," + " thread_id TEXT NOT NULL," + " parent_id TEXT DEFAULT ''," + " body TEXT NOT NULL," + " author_instance TEXT NOT NULL," + " author_name TEXT DEFAULT ''," + " created_at TEXT NOT NULL," + " FOREIGN KEY (thread_id) REFERENCES threads(id)" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS upvotes (" + " thread_id TEXT NOT NULL," + " instance_hash TEXT NOT NULL," + " PRIMARY KEY (thread_id, instance_hash)" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS synced_instances (" + " instance_hash TEXT PRIMARY KEY," + " name TEXT DEFAULT ''," + " last_sync TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS settings (" + " key TEXT PRIMARY KEY," + " value TEXT" + ")" + ) + db.execute("CREATE INDEX IF NOT EXISTS idx_posts_thread ON posts(thread_id)") + db.execute("CREATE INDEX IF NOT EXISTS idx_posts_created ON posts(created_at)") + db.execute("CREATE INDEX IF NOT EXISTS idx_threads_updated ON threads(updated_at)") + db.execute("CREATE INDEX IF NOT EXISTS idx_threads_created ON threads(created_at)") + db.commit() + db.close() + + def get_db(self): + with self._pool_lock: + if self._pool: + db = self._pool.pop() + try: + db.execute("SELECT 1") + return db + except Exception: + pass + db = sqlite3.connect(self.path, timeout=10) + db.execute("PRAGMA journal_mode=WAL") + db.row_factory = sqlite3.Row + return db + + def return_db(self, db): + try: + db.rollback() + except Exception: + try: + db.close() + except Exception: + pass + return + with self._pool_lock: + if len(self._pool) < self._POOL_SIZE: + self._pool.append(db) + else: + db.close() + + def get_setting(self, key, default=""): + db = self.get_db() + try: + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + return row["value"] if row else default + finally: + self.return_db(db) + + def set_setting(self, key, value): + db = self.get_db() + try: + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + finally: + self.return_db(db) + + def get_thread(self, thread_id): + db = self.get_db() + try: + return db.execute( + "SELECT * FROM threads WHERE id = ?", (thread_id,) + ).fetchone() + finally: + self.return_db(db) + + def get_posts(self, thread_id): + db = self.get_db() + try: + return db.execute( + "SELECT * FROM posts WHERE thread_id = ? ORDER BY created_at ASC", + (thread_id,), + ).fetchall() + finally: + self.return_db(db) + + def get_threads(self, page=1, per_page=20, tag="", search=""): + db = self.get_db() + try: + offset = (page - 1) * per_page + params = [] + where = [] + if tag: + where.append("t.tags LIKE ?") + params.append(f"%{tag}%") + if search: + where.append("(t.title LIKE ? OR t.body LIKE ?)") + params.extend([f"%{search}%", f"%{search}%"]) + where_clause = (" WHERE " + " AND ".join(where)) if where else "" + total = db.execute( + f"SELECT count(*) FROM threads t{where_clause}", params + ).fetchone()[0] + rows = db.execute( + f"SELECT t.*, (SELECT count(*) FROM posts p WHERE p.thread_id = t.id) AS reply_count " + f"FROM threads t{where_clause} ORDER BY t.updated_at DESC LIMIT ? OFFSET ?", + params + [per_page, offset], + ).fetchall() + return rows, total + finally: + self.return_db(db) + + def create_thread(self, thread_id, title, url, body, tags, author_instance, author_name, now): + db = self.get_db() + try: + db.execute( + "INSERT INTO threads (id, title, url, body, tags, author_instance, author_name, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (thread_id, title, url, body, tags, author_instance, author_name, now, now), + ) + db.commit() + finally: + self.return_db(db) + + def create_post(self, post_id, thread_id, parent_id, body, author_instance, author_name, now): + db = self.get_db() + try: + db.execute( + "INSERT INTO posts (id, thread_id, parent_id, body, author_instance, author_name, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (post_id, thread_id, parent_id, body, author_instance, author_name, now), + ) + db.execute("UPDATE threads SET updated_at = ? WHERE id = ?", (now, thread_id)) + db.commit() + finally: + self.return_db(db) + + def toggle_upvote(self, thread_id, instance_hash): + db = self.get_db() + try: + row = db.execute( + "SELECT 1 FROM upvotes WHERE thread_id = ? AND instance_hash = ?", + (thread_id, instance_hash), + ).fetchone() + if row: + db.execute( + "DELETE FROM upvotes WHERE thread_id = ? AND instance_hash = ?", + (thread_id, instance_hash), + ) + db.execute("UPDATE threads SET score = score - 1 WHERE id = ?", (thread_id,)) + delta = -1 + else: + db.execute( + "INSERT INTO upvotes (thread_id, instance_hash) VALUES (?, ?)", + (thread_id, instance_hash), + ) + db.execute("UPDATE threads SET score = score + 1 WHERE id = ?", (thread_id,)) + delta = 1 + db.commit() + return delta + finally: + self.return_db(db) + + def get_synced_instances(self): + db = self.get_db() + try: + return db.execute("SELECT * FROM synced_instances").fetchall() + finally: + self.return_db(db) + + def upsert_synced_instance(self, instance_hash, name=""): + db = self.get_db() + try: + db.execute( + "INSERT INTO synced_instances (instance_hash, name) VALUES (?, ?) " + "ON CONFLICT(instance_hash) DO UPDATE SET name=excluded.name", + (instance_hash, name), + ) + db.commit() + finally: + self.return_db(db) + + def remove_synced_instance(self, instance_hash): + db = self.get_db() + try: + db.execute("DELETE FROM synced_instances WHERE instance_hash = ?", (instance_hash,)) + db.commit() + finally: + self.return_db(db) + + def update_last_sync(self, instance_hash, now): + db = self.get_db() + try: + db.execute( + "UPDATE synced_instances SET last_sync = ? WHERE instance_hash = ?", + (now, instance_hash), + ) + db.commit() + finally: + self.return_db(db) + + def set_last_sync(self, instance_hash, timestamp): + self.update_last_sync(instance_hash, timestamp) + + def get_new_content(self, since): + db = self.get_db() + try: + threads = db.execute( + "SELECT * FROM threads WHERE updated_at > ? ORDER BY updated_at ASC", + (since,), + ).fetchall() + posts = db.execute( + "SELECT * FROM posts WHERE created_at > ? ORDER BY created_at ASC", + (since,), + ).fetchall() + upvote_threads = db.execute( + "SELECT thread_id FROM upvotes u " + "WHERE NOT EXISTS (SELECT 1 FROM threads t WHERE t.id = u.thread_id AND t.updated_at > ?)", + (since,), + ).fetchall() + return threads, posts, [r["thread_id"] for r in upvote_threads] + finally: + self.return_db(db) + + def merge_thread(self, thread): + db = self.get_db() + try: + existing = db.execute("SELECT updated_at FROM threads WHERE id = ?", (thread["id"],)).fetchone() + if existing and existing["updated_at"] >= thread["updated_at"]: + return + db.execute( + "INSERT INTO threads (id, title, url, body, tags, author_instance, author_name, created_at, updated_at, score) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(id) DO UPDATE SET " + "title=excluded.title, url=excluded.url, body=excluded.body, " + "tags=excluded.tags, updated_at=excluded.updated_at", + (thread["id"], thread["title"], thread["url"], thread["body"], + thread["tags"], thread["author_instance"], thread["author_name"], + thread["created_at"], thread["updated_at"], thread["score"]), + ) + db.commit() + finally: + self.return_db(db) + + def merge_post(self, post): + db = self.get_db() + try: + db.execute( + "INSERT OR IGNORE INTO posts (id, thread_id, parent_id, body, author_instance, author_name, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (post["id"], post["thread_id"], post["parent_id"], post["body"], + post["author_instance"], post["author_name"], post["created_at"]), + ) + db.commit() + finally: + self.return_db(db) + + def merge_upvote(self, thread_id, instance_hash): + db = self.get_db() + try: + db.execute( + "INSERT OR IGNORE INTO upvotes (thread_id, instance_hash) VALUES (?, ?)", + (thread_id, instance_hash), + ) + if db.total_changes: + db.execute("UPDATE threads SET score = score + 1 WHERE id = ?", (thread_id,)) + db.commit() + finally: + self.return_db(db) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py new file mode 100644 index 0000000..c44bfa8 --- /dev/null +++ b/tinyweb_forum/handlers.py @@ -0,0 +1,538 @@ +import json +import secrets +import threading +from datetime import datetime +from urllib.parse import unquote + + +PER_PAGE = 20 +RECENT_SECONDS = 86400 * 7 # "new" = within last 7 days + + +def esc(s): + import html + return html.escape(str(s)) + + +class ForumHandlers: + def __init__(self, fdb, sync, identity, reticulum, site_name="me"): + self.fdb = fdb + self.sync = sync + self.identity = identity + self.reticulum = reticulum + self.site_name = site_name + self._request_local = threading.local() + + def _get_csrf(self): + return getattr(self._request_local, 'csrf_token', '') + + def _csrf_field(self): + token = self._get_csrf() + return f'' + + def _check_csrf(self, body): + token = body.get("_csrf", [""])[0] + expected = self._get_csrf() + if not expected or not token: + return False + return secrets.compare_digest(token, expected) + + def _respond(self, body_html, status=200): + return { + "status": status, + "content_type": "text/html; charset=utf-8", + "body": body_html, + "headers": {}, + } + + def _json(self, data, status=200): + return { + "status": status, + "content_type": "application/json", + "body": json.dumps(data), + "headers": {}, + } + + def _redirect(self, location): + return { + "status": 302, + "content_type": "text/html; charset=utf-8", + "body": "", + "headers": {"Location": location}, + } + + def _error(self, status): + return self._respond(f"

{status}

", status) + + def _paginate(self, query): + try: + p = int(query.get("p", ["1"])[0]) + except (ValueError, IndexError): + p = 1 + return max(1, p) + + def _page_nav(self, page, total, base_url): + if total <= PER_PAGE: + return "" + total_pages = (total + PER_PAGE - 1) // PER_PAGE + sep = "&" if "?" in base_url else "?" + parts = [] + if page > 1: + parts.append(f'« prev') + parts.append(f"page {page} of {total_pages}") + if page < total_pages: + parts.append(f'next »') + return f'

{" | ".join(parts)}

' + + def _now(self): + return datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + + def _time_ago(self, ts): + try: + dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S") + except (ValueError, TypeError): + return ts + delta = datetime.now() - dt + if delta.days > 365: + return f"{delta.days // 365}y ago" + if delta.days > 30: + return f"{delta.days // 30}mo ago" + if delta.days > 0: + return f"{delta.days}d ago" + if delta.seconds >= 3600: + return f"{delta.seconds // 3600}h ago" + if delta.seconds >= 60: + return f"{delta.seconds // 60}m ago" + return "just now" + + def _is_new(self, ts): + try: + dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S") + except (ValueError, TypeError): + return False + return (datetime.now() - dt).total_seconds() < RECENT_SECONDS + + def _blocked_instances(self): + raw = self.fdb.get_setting("blocked_instances", "") + return set(h.strip() for h in raw.split(",") if h.strip()) + + def _muted_threads(self): + raw = self.fdb.get_setting("muted_threads", "") + return set(h.strip() for h in raw.split(",") if h.strip()) + + def _keyword_filters(self): + raw = self.fdb.get_setting("keyword_filters", "") + return [k.strip().lower() for k in raw.split(",") if k.strip()] + + def _passes_filters(self, thread): + blocked = self._blocked_instances() + if thread["author_instance"] in blocked: + return False + keywords = self._keyword_filters() + if keywords: + text = (thread["title"] + " " + thread["body"]).lower() + if any(k in text for k in keywords): + return False + return True + + # --- Routes --- + + def handle_list(self, query): + page = self._paginate(query) + tag = unquote(query.get("tag", [""])[0]).strip() + search = query.get("q", [""])[0].strip() + rows, total = self.fdb.get_threads(page=page, per_page=PER_PAGE, tag=tag, search=search) + muted = self._muted_threads() + new_count = 0 + items = "" + for r in rows: + if r["id"] in muted: + continue + if self._is_new(r["created_at"]): + new_count += 1 + badge = "[share]" if r["url"] else "[request]" + tags_html = "" + if r["tags"]: + tag_links = " ".join( + f'[{esc(t.strip())}]' + for t in r["tags"].split(",") if t.strip() + ) + tags_html = f' {tag_links}' + reply_label = f"{r['reply_count']} replies" if r['reply_count'] else "no replies" + items += ( + f'
  • ' + f'{badge} ' + f'{esc(r["title"])}' + f'{tags_html}' + f'
    ' + f'{esc(r["author_name"] or r["author_instance"][:8])}' + f' · {self._time_ago(r["created_at"])}' + f' · {r["score"]} upvotes' + f' · {reply_label}' + f'
    • ' + ) + if not items: + items = "

    No threads yet.

    " + new_label = f" ({new_count} new)" if new_count else "" + search_form = ( + f'
    ' + f'' + f'
    ' + ) + tag_label = f' — tag: {esc(tag)}' if tag else "" + return self._respond( + f"

    forum{tag_label}

    " + f"

    {search_form}" + f' + new thread' + f' mod' + f"

    " + f"

    {total} threads{new_label}

    " + f"" + f"{self._page_nav(page, total, f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum')}" + ) + + def handle_new_form(self, msg=""): + return self._respond( + f"

    new thread

    " + f'
    ' + f'{self._csrf_field()}' + f'

    ' + f'

    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_new_submit(self, body): + title = body.get("title", [""])[0].strip() + url = body.get("url", [""])[0].strip() + body_text = body.get("body", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + if not title: + return self.handle_new_form("Title is required.") + thread_id = secrets.token_hex(16) + author_instance = self.identity.hash.hex() if self.identity else "local" + author_name = self.site_name + now = self._now() + self.fdb.create_thread(thread_id, title, url, body_text, tags, author_instance, author_name, now) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_thread(self, thread_id, query=None): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + posts = self.fdb.get_posts(thread_id) + muted = self._muted_threads() + is_muted = thread["id"] in muted + + badge = "[share]" if thread["url"] else "[request]" + url_html = "" + if thread["url"]: + url_html = ( + f'

    {esc(thread["url"])}' + f' (+ save to my index)

    ' + ) + tags_html = "" + if thread["tags"]: + tag_links = " ".join( + f'[{esc(t.strip())}]' + for t in thread["tags"].split(",") if t.strip() + ) + tags_html = f'

    {tag_links}

    ' + + body_html = f"

    {esc(thread['body'])}

    " if thread["body"] else "" + + mute_btn = ( + f'
    ' + f'{self._csrf_field()}
    ' + if is_muted else + f'
    ' + f'{self._csrf_field()}
    ' + ) + + posts_html = "" + for p in posts: + save_links = "" + for word in p["body"].split(): + w = word.strip().strip(",.!?;:") + if w.startswith(("http://", "https://")): + save_links += ( + f' + save' + ) + parent_ref = "" + if p["parent_id"]: + parent_ref = f' ↪ reply' + posts_html += ( + f'
    ' + f'{esc(p["author_name"] or p["author_instance"][:8])}' + f' · {self._time_ago(p["created_at"])}{parent_ref}' + f'

    {esc(p["body"])}

    ' + f'{save_links}' + f'
    ' + ) + + reply_form = ( + f'
    ' + f'{self._csrf_field()}' + f'
    ' + f'' + f"
    " + ) + + return self._respond( + f"

    {badge} {esc(thread['title'])}

    " + f'

    ' + f'by {esc(thread["author_name"] or thread["author_instance"][:8])}' + f' · {self._time_ago(thread["created_at"])}' + f' · {thread["score"]} upvotes' + f' · {mute_btn}' + f' ·

    ' + f'{self._csrf_field()}
    ' + f'

    ' + f'{url_html}' + f'{body_html}' + f'{tags_html}' + f"
    " + f"{posts_html}" + f"
    " + f"{reply_form}" + f'back to forum' + ) + + def handle_reply(self, thread_id, body): + body_text = body.get("body", [""])[0].strip() + if not body_text: + return self._redirect(f"/forum/t/{thread_id}") + parent_id = body.get("parent_id", [""])[0].strip() + author_instance = self.identity.hash.hex() if self.identity else "local" + author_name = self.site_name + post_id = secrets.token_hex(16) + now = self._now() + self.fdb.create_post(post_id, thread_id, parent_id, body_text, author_instance, author_name, now) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_upvote(self, thread_id, body): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + self.fdb.toggle_upvote(thread_id, instance_hash) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_mute(self, thread_id): + muted = self._muted_threads() + muted.add(thread_id) + self.fdb.set_setting("muted_threads", ",".join(muted)) + return self._redirect(f"/forum") + + def handle_unmute(self, thread_id): + muted = self._muted_threads() + muted.discard(thread_id) + self.fdb.set_setting("muted_threads", ",".join(muted)) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_moderation(self, msg=""): + blocked = self._blocked_instances() + blocked_items = "" + if blocked: + for h in sorted(blocked): + blocked_items += ( + f'
  • {esc(h[:16])}... ' + f'
    ' + f'{self._csrf_field()}' + f'' + f'
    ' + f'
    • ' + ) + blocked_items = f"" + else: + blocked_items = "

    No instances blocked.

    " + + filters = self._keyword_filters() + filters_str = ", ".join(filters) if filters else "" + + synced = self.fdb.get_synced_instances() + synced_items = "" + for s in synced: + synced_items += ( + f'
  • {esc(s["name"] or s["instance_hash"][:16])}... ' + f'
    ' + f'{self._csrf_field()}' + f'' + f'
    ' + f'
    • ' + ) + synced_items = f"" if synced_items else "

    No instances synced yet.

    " + + return self._respond( + f"

    forum moderation

    " + f"

    {msg}

    " + f"

    blocked instances

    " + f"{blocked_items}" + f'
    ' + f'{self._csrf_field()}' + f' ' + f'' + f"
    " + f"

    keyword filters

    " + f'
    ' + f'{self._csrf_field()}' + f'' + f'' + f"
    " + f"

    synced instances

    " + f"{synced_items}" + f'
    ' + f'{self._csrf_field()}' + f' ' + f' ' + f'' + f"
    " + f'
    ' + f'back to forum' + ) + + def handle_block(self, body): + instance = body.get("instance", [""])[0].strip().replace("<", "").replace(">", "") + if len(instance) != 32: + return self.handle_moderation("Invalid instance hash (must be 32 hex chars).") + blocked = self._blocked_instances() + blocked.add(instance) + self.fdb.set_setting("blocked_instances", ",".join(blocked)) + return self.handle_moderation(f"Blocked {instance[:16]}...") + + def handle_unblock(self, body): + instance = body.get("instance", [""])[0].strip() + blocked = self._blocked_instances() + blocked.discard(instance) + self.fdb.set_setting("blocked_instances", ",".join(blocked)) + return self.handle_moderation(f"Unblocked {instance[:16]}...") + + def handle_filters(self, body): + keywords = body.get("keywords", [""])[0].strip() + self.fdb.set_setting("keyword_filters", keywords) + return self.handle_moderation("Filters saved.") + + def handle_sync_add(self, body): + instance = body.get("instance", [""])[0].strip().replace("<", "").replace(">", "") + name = body.get("name", [""])[0].strip() + if len(instance) != 32: + return self.handle_moderation("Invalid instance hash (must be 32 hex chars).") + self.fdb.upsert_synced_instance(instance, name) + return self.handle_moderation(f"Added {name or instance[:16]}... to sync.") + + def handle_unsync(self, body): + instance = body.get("instance", [""])[0].strip() + self.fdb.remove_synced_instance(instance) + return self.handle_moderation("Removed.") + + # --- Sync endpoint (called over RNS) --- + + def handle_sync_request(self, data): + """Handle incoming sync request from another forum instance.""" + since = data.get("query", {}).get("since", [""])[0] if isinstance(data.get("query"), dict) else "" + incoming_threads = data.get("threads", []) + incoming_posts = data.get("posts", []) + incoming_upvotes = data.get("upvotes", []) + + if incoming_threads: + for t in incoming_threads: + self.fdb.merge_thread(t) + if incoming_posts: + for p in incoming_posts: + self.fdb.merge_post(p) + if incoming_upvotes: + for uv in incoming_upvotes: + self.fdb.merge_upvote(uv["thread_id"], uv["instance_hash"]) + + threads, posts, upvote_threads = [], [], [] + if since: + ts, posts_list, up_list = self.fdb.get_new_content(since) + threads = [dict(r) for r in ts] + posts = [dict(r) for r in posts_list] + upvote_threads = up_list + + return { + "status": 200, + "content_type": "application/json", + "body": json.dumps({ + "threads": threads, + "posts": posts, + "upvote_threads": upvote_threads, + }), + "headers": {}, + } + + def handle_sync_add_instance(self, body): + """Add instance for sync (from moderation page action).""" + return self.handle_sync_add(body) + + # --- Router --- + + def _with_csrf(self, resp, csrf_token): + resp.setdefault("headers", {}) + if resp.get("content_type", "").startswith("text/html"): + resp["headers"]["Set-Cookie"] = ( + f"_csrf={csrf_token}; SameSite=Strict; HttpOnly; Path=/forum" + ) + return resp + + def handle(self, method, path, query, body, cookies=None): + csrf_token = (cookies or {}).get("_csrf", "") + if not csrf_token: + csrf_token = secrets.token_hex(32) + self._request_local.csrf_token = csrf_token + + if not path.startswith("/forum"): + return self._with_csrf(self._error(404), csrf_token) + + sub = path[len("/forum"):] + + if method == "GET": + if sub == "" or sub == "/": + return self._with_csrf(self.handle_list(query), csrf_token) + elif sub == "/new": + return self._with_csrf(self.handle_new_form(), csrf_token) + elif sub == "/moderation": + return self._with_csrf(self.handle_moderation(), csrf_token) + elif sub.startswith("/t/"): + tid = sub[3:] + return self._with_csrf(self.handle_thread(tid, query), csrf_token) + elif method == "POST": + if not self._check_csrf(body): + return self._with_csrf( + self._respond("

    403 Forbidden

    ", status=403), csrf_token + ) + if sub == "/new": + return self._with_csrf(self.handle_new_submit(body), csrf_token) + elif sub.startswith("/t/"): + rest = sub[3:] + if "/reply" in rest: + tid = rest.split("/reply")[0] + return self._with_csrf(self.handle_reply(tid, body), csrf_token) + elif rest.endswith("/upvote"): + tid = rest[:-7] + return self._with_csrf(self.handle_upvote(tid, body), csrf_token) + elif sub.startswith("/mute/"): + return self._with_csrf(self.handle_mute(sub[6:]), csrf_token) + elif sub.startswith("/unmute/"): + return self._with_csrf(self.handle_unmute(sub[8:]), csrf_token) + elif sub == "/block": + return self._with_csrf(self.handle_block(body), csrf_token) + elif sub == "/unblock": + return self._with_csrf(self.handle_unblock(body), csrf_token) + elif sub == "/filters": + return self._with_csrf(self.handle_filters(body), csrf_token) + elif sub == "/sync/add": + return self._with_csrf(self.handle_sync_add(body), csrf_token) + elif sub == "/unsync": + return self._with_csrf(self.handle_unsync(body), csrf_token) + + return self._with_csrf(self._error(404), csrf_token) + + def handle_sync(self, data): + """Entry point for incoming RNS sync requests.""" + return self.handle_sync_request(data) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py new file mode 100644 index 0000000..32133b8 --- /dev/null +++ b/tinyweb_forum/sync.py @@ -0,0 +1,146 @@ +import json +import threading +import time +import RNS + +FORUM_APP = "tinyweb-forum" +SYNC_INTERVAL = 300 # 5 minutes +REQUEST_TIMEOUT = 60 + + +class ForumSync: + def __init__(self, fdb, identity, reticulum, handlers_ref): + self.fdb = fdb + self.identity = identity + self.reticulum = reticulum + self.handlers_ref = handlers_ref + self.destination = None + self._running = False + self._thread = None + + def start(self): + self.destination = RNS.Destination( + self.identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + FORUM_APP, + ) + self.destination.register_request_handler( + "/forum", + response_generator=self._rns_handler, + allow=RNS.Destination.ALLOW_ALL, + ) + self.destination.announce() + self._running = True + self._thread = threading.Thread(target=self._sync_loop, daemon=True) + self._thread.start() + + def stop(self): + self._running = False + + def _rns_handler(self, path, data, request_id, link_id, remote_identity, requested_at): + return self.handlers_ref().handle_sync(data) + + def _sync_loop(self): + while self._running: + try: + instances = self.fdb.get_synced_instances() + for inst in instances: + if not self._running: + break + try: + self._sync_with(inst["instance_hash"]) + except Exception as e: + print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") + except Exception: + pass + for _ in range(SYNC_INTERVAL): + if not self._running: + return + time.sleep(1) + + def _sync_with(self, instance_hash): + dest_hash = bytes.fromhex(instance_hash) + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + return + + server_identity = RNS.Identity.recall(dest_hash) + if server_identity is None: + return + + destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + FORUM_APP, + ) + + link = RNS.Link(destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + + if link.status != RNS.Link.ACTIVE: + return + + try: + inst = self.fdb.get_synced_instances() + last_sync = "" + for s in inst: + if s["instance_hash"] == instance_hash: + last_sync = s["last_sync"] or "" + break + + since = last_sync.replace(" ", "T") if last_sync else "" + + threads, posts = [], [] + upvotes = [] + if since: + ts, ps, uv = self.fdb.get_new_content(since) + threads = [dict(r) for r in ts] + posts = [dict(r) for r in ps] + upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv] + + request_data = { + "query": {"since": [since]} if since else {}, + "threads": threads, + "posts": posts, + "upvotes": upvotes, + } + + receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) + elapsed = 0 + done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + if isinstance(resp, dict) and resp.get("status") == 200: + try: + data = json.loads(resp["body"]) + except (json.JSONDecodeError, KeyError): + data = {} + for t in data.get("threads", []): + self.fdb.merge_thread(t) + for p in data.get("posts", []): + self.fdb.merge_post(p) + for tid in data.get("upvote_threads", []): + self.fdb.merge_upvote(tid, instance_hash) + now = time.strftime("%Y-%m-%dT%H:%M:%S") + self.fdb.set_last_sync(instance_hash, now) + else: + pass + finally: + link.teardown() + + def handle_sync(self, data): + return self.handlers_ref().handle_sync_request(data) From ff273669e731044ba7923a3cbdac72b40235a560 Mon Sep 17 00:00:00 2001 From: user Date: Thu, 4 Jun 2026 08:23:51 +0000 Subject: [PATCH 02/56] initial: decentralized link-sharing forum for TinyWeb --- .gitignore | 5 + README.md | 25 ++ pyproject.toml | 19 ++ tinyweb_forum/__init__.py | 34 +++ tinyweb_forum/db.py | 321 +++++++++++++++++++++++ tinyweb_forum/handlers.py | 538 ++++++++++++++++++++++++++++++++++++++ tinyweb_forum/sync.py | 146 +++++++++++ 7 files changed, 1088 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 tinyweb_forum/__init__.py create mode 100644 tinyweb_forum/db.py create mode 100644 tinyweb_forum/handlers.py create mode 100644 tinyweb_forum/sync.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a376320 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +*.pyo +dist/ +*.egg-info/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..4508429 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# tinyweb-forum + +A decentralized link-sharing forum for [TinyWeb](https://github.com/derickfay/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. + +## Install + +```bash +pip install tinyweb-forum +``` + +Enable the forum in TinyWeb's customize page (`/style`). + +## Development + +```bash +git clone https://github.com/derickfay/tinyweb-forum +pip install -e . +``` + +## How it works + +- Each TinyWeb instance stores forum threads and posts in its own `forum.db` +- Instances sync content with each other over RNS +- Moderation is per-instance: block instances, mute threads, keyword filters +- No global server, no algorithms, no tracking diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..19a0bb7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.backends._legacy:_Backend" + +[project] +name = "tinyweb-forum" +version = "0.1.0" +description = "Decentralized link-sharing forum for TinyWeb" +license = {text = "MIT"} +requires-python = ">=3.9" +dependencies = [ + "rns", +] + +[project.optional-dependencies] +dev = [] + +[tool.setuptools.packages.find] +include = ["tinyweb_forum*"] diff --git a/tinyweb_forum/__init__.py b/tinyweb_forum/__init__.py new file mode 100644 index 0000000..ffdbc9c --- /dev/null +++ b/tinyweb_forum/__init__.py @@ -0,0 +1,34 @@ +from tinyweb_forum.db import ForumDB +from tinyweb_forum.handlers import ForumHandlers +from tinyweb_forum.sync import ForumSync + +FORUM_ENABLED_KEY = "forum_enabled" + + +class ForumPlugin: + def __init__(self, data_dir, identity, reticulum, site_name="me"): + self.fdb = ForumDB(data_dir) + self.handlers = ForumHandlers( + self.fdb, None, identity, reticulum, site_name=site_name + ) + self.sync = ForumSync(self.fdb, identity, reticulum, lambda: self.handlers) + self.handlers.sync = self.sync + self.identity = identity + self.reticulum = reticulum + self._started = False + + def is_enabled(self): + return self.fdb.get_setting(FORUM_ENABLED_KEY, "0") == "1" + + def enable(self): + self.fdb.set_setting(FORUM_ENABLED_KEY, "1") + if not self._started: + self.sync.start() + self._started = True + + def disable(self): + self.fdb.set_setting(FORUM_ENABLED_KEY, "0") + # Keep sync running so we still receive content — disable just hides UI + + def handle(self, method, path, query, body, cookies=None): + return self.handlers.handle(method, path, query, body, cookies) diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py new file mode 100644 index 0000000..a38ad5c --- /dev/null +++ b/tinyweb_forum/db.py @@ -0,0 +1,321 @@ +import sqlite3 +import os +import threading + +FORUM_DB = "forum.db" + + +class ForumDB: + def __init__(self, data_dir): + self.path = os.path.join(data_dir, FORUM_DB) + self._pool = [] + self._pool_lock = threading.Lock() + self._POOL_SIZE = 8 + self.init_db() + + def init_db(self): + os.makedirs(os.path.dirname(self.path), exist_ok=True) + db = sqlite3.connect(self.path) + db.execute( + "CREATE TABLE IF NOT EXISTS threads (" + " id TEXT PRIMARY KEY," + " title TEXT NOT NULL," + " url TEXT DEFAULT ''," + " body TEXT DEFAULT ''," + " tags TEXT DEFAULT ''," + " author_instance TEXT NOT NULL," + " author_name TEXT DEFAULT ''," + " created_at TEXT NOT NULL," + " updated_at TEXT NOT NULL," + " score INTEGER DEFAULT 0" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS posts (" + " id TEXT PRIMARY KEY," + " thread_id TEXT NOT NULL," + " parent_id TEXT DEFAULT ''," + " body TEXT NOT NULL," + " author_instance TEXT NOT NULL," + " author_name TEXT DEFAULT ''," + " created_at TEXT NOT NULL," + " FOREIGN KEY (thread_id) REFERENCES threads(id)" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS upvotes (" + " thread_id TEXT NOT NULL," + " instance_hash TEXT NOT NULL," + " PRIMARY KEY (thread_id, instance_hash)" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS synced_instances (" + " instance_hash TEXT PRIMARY KEY," + " name TEXT DEFAULT ''," + " last_sync TEXT DEFAULT ''" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS settings (" + " key TEXT PRIMARY KEY," + " value TEXT" + ")" + ) + db.execute("CREATE INDEX IF NOT EXISTS idx_posts_thread ON posts(thread_id)") + db.execute("CREATE INDEX IF NOT EXISTS idx_posts_created ON posts(created_at)") + db.execute("CREATE INDEX IF NOT EXISTS idx_threads_updated ON threads(updated_at)") + db.execute("CREATE INDEX IF NOT EXISTS idx_threads_created ON threads(created_at)") + db.commit() + db.close() + + def get_db(self): + with self._pool_lock: + if self._pool: + db = self._pool.pop() + try: + db.execute("SELECT 1") + return db + except Exception: + pass + db = sqlite3.connect(self.path, timeout=10) + db.execute("PRAGMA journal_mode=WAL") + db.row_factory = sqlite3.Row + return db + + def return_db(self, db): + try: + db.rollback() + except Exception: + try: + db.close() + except Exception: + pass + return + with self._pool_lock: + if len(self._pool) < self._POOL_SIZE: + self._pool.append(db) + else: + db.close() + + def get_setting(self, key, default=""): + db = self.get_db() + try: + row = db.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + return row["value"] if row else default + finally: + self.return_db(db) + + def set_setting(self, key, value): + db = self.get_db() + try: + db.execute( + "INSERT INTO settings (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", + (key, value), + ) + db.commit() + finally: + self.return_db(db) + + def get_thread(self, thread_id): + db = self.get_db() + try: + return db.execute( + "SELECT * FROM threads WHERE id = ?", (thread_id,) + ).fetchone() + finally: + self.return_db(db) + + def get_posts(self, thread_id): + db = self.get_db() + try: + return db.execute( + "SELECT * FROM posts WHERE thread_id = ? ORDER BY created_at ASC", + (thread_id,), + ).fetchall() + finally: + self.return_db(db) + + def get_threads(self, page=1, per_page=20, tag="", search=""): + db = self.get_db() + try: + offset = (page - 1) * per_page + params = [] + where = [] + if tag: + where.append("t.tags LIKE ?") + params.append(f"%{tag}%") + if search: + where.append("(t.title LIKE ? OR t.body LIKE ?)") + params.extend([f"%{search}%", f"%{search}%"]) + where_clause = (" WHERE " + " AND ".join(where)) if where else "" + total = db.execute( + f"SELECT count(*) FROM threads t{where_clause}", params + ).fetchone()[0] + rows = db.execute( + f"SELECT t.*, (SELECT count(*) FROM posts p WHERE p.thread_id = t.id) AS reply_count " + f"FROM threads t{where_clause} ORDER BY t.updated_at DESC LIMIT ? OFFSET ?", + params + [per_page, offset], + ).fetchall() + return rows, total + finally: + self.return_db(db) + + def create_thread(self, thread_id, title, url, body, tags, author_instance, author_name, now): + db = self.get_db() + try: + db.execute( + "INSERT INTO threads (id, title, url, body, tags, author_instance, author_name, created_at, updated_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (thread_id, title, url, body, tags, author_instance, author_name, now, now), + ) + db.commit() + finally: + self.return_db(db) + + def create_post(self, post_id, thread_id, parent_id, body, author_instance, author_name, now): + db = self.get_db() + try: + db.execute( + "INSERT INTO posts (id, thread_id, parent_id, body, author_instance, author_name, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (post_id, thread_id, parent_id, body, author_instance, author_name, now), + ) + db.execute("UPDATE threads SET updated_at = ? WHERE id = ?", (now, thread_id)) + db.commit() + finally: + self.return_db(db) + + def toggle_upvote(self, thread_id, instance_hash): + db = self.get_db() + try: + row = db.execute( + "SELECT 1 FROM upvotes WHERE thread_id = ? AND instance_hash = ?", + (thread_id, instance_hash), + ).fetchone() + if row: + db.execute( + "DELETE FROM upvotes WHERE thread_id = ? AND instance_hash = ?", + (thread_id, instance_hash), + ) + db.execute("UPDATE threads SET score = score - 1 WHERE id = ?", (thread_id,)) + delta = -1 + else: + db.execute( + "INSERT INTO upvotes (thread_id, instance_hash) VALUES (?, ?)", + (thread_id, instance_hash), + ) + db.execute("UPDATE threads SET score = score + 1 WHERE id = ?", (thread_id,)) + delta = 1 + db.commit() + return delta + finally: + self.return_db(db) + + def get_synced_instances(self): + db = self.get_db() + try: + return db.execute("SELECT * FROM synced_instances").fetchall() + finally: + self.return_db(db) + + def upsert_synced_instance(self, instance_hash, name=""): + db = self.get_db() + try: + db.execute( + "INSERT INTO synced_instances (instance_hash, name) VALUES (?, ?) " + "ON CONFLICT(instance_hash) DO UPDATE SET name=excluded.name", + (instance_hash, name), + ) + db.commit() + finally: + self.return_db(db) + + def remove_synced_instance(self, instance_hash): + db = self.get_db() + try: + db.execute("DELETE FROM synced_instances WHERE instance_hash = ?", (instance_hash,)) + db.commit() + finally: + self.return_db(db) + + def update_last_sync(self, instance_hash, now): + db = self.get_db() + try: + db.execute( + "UPDATE synced_instances SET last_sync = ? WHERE instance_hash = ?", + (now, instance_hash), + ) + db.commit() + finally: + self.return_db(db) + + def set_last_sync(self, instance_hash, timestamp): + self.update_last_sync(instance_hash, timestamp) + + def get_new_content(self, since): + db = self.get_db() + try: + threads = db.execute( + "SELECT * FROM threads WHERE updated_at > ? ORDER BY updated_at ASC", + (since,), + ).fetchall() + posts = db.execute( + "SELECT * FROM posts WHERE created_at > ? ORDER BY created_at ASC", + (since,), + ).fetchall() + upvote_threads = db.execute( + "SELECT thread_id FROM upvotes u " + "WHERE NOT EXISTS (SELECT 1 FROM threads t WHERE t.id = u.thread_id AND t.updated_at > ?)", + (since,), + ).fetchall() + return threads, posts, [r["thread_id"] for r in upvote_threads] + finally: + self.return_db(db) + + def merge_thread(self, thread): + db = self.get_db() + try: + existing = db.execute("SELECT updated_at FROM threads WHERE id = ?", (thread["id"],)).fetchone() + if existing and existing["updated_at"] >= thread["updated_at"]: + return + db.execute( + "INSERT INTO threads (id, title, url, body, tags, author_instance, author_name, created_at, updated_at, score) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(id) DO UPDATE SET " + "title=excluded.title, url=excluded.url, body=excluded.body, " + "tags=excluded.tags, updated_at=excluded.updated_at", + (thread["id"], thread["title"], thread["url"], thread["body"], + thread["tags"], thread["author_instance"], thread["author_name"], + thread["created_at"], thread["updated_at"], thread["score"]), + ) + db.commit() + finally: + self.return_db(db) + + def merge_post(self, post): + db = self.get_db() + try: + db.execute( + "INSERT OR IGNORE INTO posts (id, thread_id, parent_id, body, author_instance, author_name, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (post["id"], post["thread_id"], post["parent_id"], post["body"], + post["author_instance"], post["author_name"], post["created_at"]), + ) + db.commit() + finally: + self.return_db(db) + + def merge_upvote(self, thread_id, instance_hash): + db = self.get_db() + try: + db.execute( + "INSERT OR IGNORE INTO upvotes (thread_id, instance_hash) VALUES (?, ?)", + (thread_id, instance_hash), + ) + if db.total_changes: + db.execute("UPDATE threads SET score = score + 1 WHERE id = ?", (thread_id,)) + db.commit() + finally: + self.return_db(db) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py new file mode 100644 index 0000000..c44bfa8 --- /dev/null +++ b/tinyweb_forum/handlers.py @@ -0,0 +1,538 @@ +import json +import secrets +import threading +from datetime import datetime +from urllib.parse import unquote + + +PER_PAGE = 20 +RECENT_SECONDS = 86400 * 7 # "new" = within last 7 days + + +def esc(s): + import html + return html.escape(str(s)) + + +class ForumHandlers: + def __init__(self, fdb, sync, identity, reticulum, site_name="me"): + self.fdb = fdb + self.sync = sync + self.identity = identity + self.reticulum = reticulum + self.site_name = site_name + self._request_local = threading.local() + + def _get_csrf(self): + return getattr(self._request_local, 'csrf_token', '') + + def _csrf_field(self): + token = self._get_csrf() + return f'' + + def _check_csrf(self, body): + token = body.get("_csrf", [""])[0] + expected = self._get_csrf() + if not expected or not token: + return False + return secrets.compare_digest(token, expected) + + def _respond(self, body_html, status=200): + return { + "status": status, + "content_type": "text/html; charset=utf-8", + "body": body_html, + "headers": {}, + } + + def _json(self, data, status=200): + return { + "status": status, + "content_type": "application/json", + "body": json.dumps(data), + "headers": {}, + } + + def _redirect(self, location): + return { + "status": 302, + "content_type": "text/html; charset=utf-8", + "body": "", + "headers": {"Location": location}, + } + + def _error(self, status): + return self._respond(f"

    {status}

    ", status) + + def _paginate(self, query): + try: + p = int(query.get("p", ["1"])[0]) + except (ValueError, IndexError): + p = 1 + return max(1, p) + + def _page_nav(self, page, total, base_url): + if total <= PER_PAGE: + return "" + total_pages = (total + PER_PAGE - 1) // PER_PAGE + sep = "&" if "?" in base_url else "?" + parts = [] + if page > 1: + parts.append(f'« prev') + parts.append(f"page {page} of {total_pages}") + if page < total_pages: + parts.append(f'next »') + return f'

    {" | ".join(parts)}

    ' + + def _now(self): + return datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + + def _time_ago(self, ts): + try: + dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S") + except (ValueError, TypeError): + return ts + delta = datetime.now() - dt + if delta.days > 365: + return f"{delta.days // 365}y ago" + if delta.days > 30: + return f"{delta.days // 30}mo ago" + if delta.days > 0: + return f"{delta.days}d ago" + if delta.seconds >= 3600: + return f"{delta.seconds // 3600}h ago" + if delta.seconds >= 60: + return f"{delta.seconds // 60}m ago" + return "just now" + + def _is_new(self, ts): + try: + dt = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%S") + except (ValueError, TypeError): + return False + return (datetime.now() - dt).total_seconds() < RECENT_SECONDS + + def _blocked_instances(self): + raw = self.fdb.get_setting("blocked_instances", "") + return set(h.strip() for h in raw.split(",") if h.strip()) + + def _muted_threads(self): + raw = self.fdb.get_setting("muted_threads", "") + return set(h.strip() for h in raw.split(",") if h.strip()) + + def _keyword_filters(self): + raw = self.fdb.get_setting("keyword_filters", "") + return [k.strip().lower() for k in raw.split(",") if k.strip()] + + def _passes_filters(self, thread): + blocked = self._blocked_instances() + if thread["author_instance"] in blocked: + return False + keywords = self._keyword_filters() + if keywords: + text = (thread["title"] + " " + thread["body"]).lower() + if any(k in text for k in keywords): + return False + return True + + # --- Routes --- + + def handle_list(self, query): + page = self._paginate(query) + tag = unquote(query.get("tag", [""])[0]).strip() + search = query.get("q", [""])[0].strip() + rows, total = self.fdb.get_threads(page=page, per_page=PER_PAGE, tag=tag, search=search) + muted = self._muted_threads() + new_count = 0 + items = "" + for r in rows: + if r["id"] in muted: + continue + if self._is_new(r["created_at"]): + new_count += 1 + badge = "[share]" if r["url"] else "[request]" + tags_html = "" + if r["tags"]: + tag_links = " ".join( + f'[{esc(t.strip())}]' + for t in r["tags"].split(",") if t.strip() + ) + tags_html = f' {tag_links}' + reply_label = f"{r['reply_count']} replies" if r['reply_count'] else "no replies" + items += ( + f'
  • ' + f'{badge} ' + f'{esc(r["title"])}' + f'{tags_html}' + f'
    ' + f'{esc(r["author_name"] or r["author_instance"][:8])}' + f' · {self._time_ago(r["created_at"])}' + f' · {r["score"]} upvotes' + f' · {reply_label}' + f'
    • ' + ) + if not items: + items = "

    No threads yet.

    " + new_label = f" ({new_count} new)" if new_count else "" + search_form = ( + f'
    ' + f'' + f'
    ' + ) + tag_label = f' — tag: {esc(tag)}' if tag else "" + return self._respond( + f"

    forum{tag_label}

    " + f"

    {search_form}" + f' + new thread' + f' mod' + f"

    " + f"

    {total} threads{new_label}

    " + f"" + f"{self._page_nav(page, total, f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum')}" + ) + + def handle_new_form(self, msg=""): + return self._respond( + f"

    new thread

    " + f'
    ' + f'{self._csrf_field()}' + f'

    ' + f'

    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_new_submit(self, body): + title = body.get("title", [""])[0].strip() + url = body.get("url", [""])[0].strip() + body_text = body.get("body", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + if not title: + return self.handle_new_form("Title is required.") + thread_id = secrets.token_hex(16) + author_instance = self.identity.hash.hex() if self.identity else "local" + author_name = self.site_name + now = self._now() + self.fdb.create_thread(thread_id, title, url, body_text, tags, author_instance, author_name, now) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_thread(self, thread_id, query=None): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + posts = self.fdb.get_posts(thread_id) + muted = self._muted_threads() + is_muted = thread["id"] in muted + + badge = "[share]" if thread["url"] else "[request]" + url_html = "" + if thread["url"]: + url_html = ( + f'

    {esc(thread["url"])}' + f' (+ save to my index)

    ' + ) + tags_html = "" + if thread["tags"]: + tag_links = " ".join( + f'[{esc(t.strip())}]' + for t in thread["tags"].split(",") if t.strip() + ) + tags_html = f'

    {tag_links}

    ' + + body_html = f"

    {esc(thread['body'])}

    " if thread["body"] else "" + + mute_btn = ( + f'
    ' + f'{self._csrf_field()}
    ' + if is_muted else + f'
    ' + f'{self._csrf_field()}
    ' + ) + + posts_html = "" + for p in posts: + save_links = "" + for word in p["body"].split(): + w = word.strip().strip(",.!?;:") + if w.startswith(("http://", "https://")): + save_links += ( + f' + save' + ) + parent_ref = "" + if p["parent_id"]: + parent_ref = f' ↪ reply' + posts_html += ( + f'
    ' + f'{esc(p["author_name"] or p["author_instance"][:8])}' + f' · {self._time_ago(p["created_at"])}{parent_ref}' + f'

    {esc(p["body"])}

    ' + f'{save_links}' + f'
    ' + ) + + reply_form = ( + f'
    ' + f'{self._csrf_field()}' + f'
    ' + f'' + f"
    " + ) + + return self._respond( + f"

    {badge} {esc(thread['title'])}

    " + f'

    ' + f'by {esc(thread["author_name"] or thread["author_instance"][:8])}' + f' · {self._time_ago(thread["created_at"])}' + f' · {thread["score"]} upvotes' + f' · {mute_btn}' + f' ·

    ' + f'{self._csrf_field()}
    ' + f'

    ' + f'{url_html}' + f'{body_html}' + f'{tags_html}' + f"
    " + f"{posts_html}" + f"
    " + f"{reply_form}" + f'back to forum' + ) + + def handle_reply(self, thread_id, body): + body_text = body.get("body", [""])[0].strip() + if not body_text: + return self._redirect(f"/forum/t/{thread_id}") + parent_id = body.get("parent_id", [""])[0].strip() + author_instance = self.identity.hash.hex() if self.identity else "local" + author_name = self.site_name + post_id = secrets.token_hex(16) + now = self._now() + self.fdb.create_post(post_id, thread_id, parent_id, body_text, author_instance, author_name, now) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_upvote(self, thread_id, body): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + self.fdb.toggle_upvote(thread_id, instance_hash) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_mute(self, thread_id): + muted = self._muted_threads() + muted.add(thread_id) + self.fdb.set_setting("muted_threads", ",".join(muted)) + return self._redirect(f"/forum") + + def handle_unmute(self, thread_id): + muted = self._muted_threads() + muted.discard(thread_id) + self.fdb.set_setting("muted_threads", ",".join(muted)) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_moderation(self, msg=""): + blocked = self._blocked_instances() + blocked_items = "" + if blocked: + for h in sorted(blocked): + blocked_items += ( + f'
  • {esc(h[:16])}... ' + f'
    ' + f'{self._csrf_field()}' + f'' + f'
    ' + f'
    • ' + ) + blocked_items = f"" + else: + blocked_items = "

    No instances blocked.

    " + + filters = self._keyword_filters() + filters_str = ", ".join(filters) if filters else "" + + synced = self.fdb.get_synced_instances() + synced_items = "" + for s in synced: + synced_items += ( + f'
  • {esc(s["name"] or s["instance_hash"][:16])}... ' + f'
    ' + f'{self._csrf_field()}' + f'' + f'
    ' + f'
    • ' + ) + synced_items = f"" if synced_items else "

    No instances synced yet.

    " + + return self._respond( + f"

    forum moderation

    " + f"

    {msg}

    " + f"

    blocked instances

    " + f"{blocked_items}" + f'
    ' + f'{self._csrf_field()}' + f' ' + f'' + f"
    " + f"

    keyword filters

    " + f'
    ' + f'{self._csrf_field()}' + f'' + f'' + f"
    " + f"

    synced instances

    " + f"{synced_items}" + f'
    ' + f'{self._csrf_field()}' + f' ' + f' ' + f'' + f"
    " + f'
    ' + f'back to forum' + ) + + def handle_block(self, body): + instance = body.get("instance", [""])[0].strip().replace("<", "").replace(">", "") + if len(instance) != 32: + return self.handle_moderation("Invalid instance hash (must be 32 hex chars).") + blocked = self._blocked_instances() + blocked.add(instance) + self.fdb.set_setting("blocked_instances", ",".join(blocked)) + return self.handle_moderation(f"Blocked {instance[:16]}...") + + def handle_unblock(self, body): + instance = body.get("instance", [""])[0].strip() + blocked = self._blocked_instances() + blocked.discard(instance) + self.fdb.set_setting("blocked_instances", ",".join(blocked)) + return self.handle_moderation(f"Unblocked {instance[:16]}...") + + def handle_filters(self, body): + keywords = body.get("keywords", [""])[0].strip() + self.fdb.set_setting("keyword_filters", keywords) + return self.handle_moderation("Filters saved.") + + def handle_sync_add(self, body): + instance = body.get("instance", [""])[0].strip().replace("<", "").replace(">", "") + name = body.get("name", [""])[0].strip() + if len(instance) != 32: + return self.handle_moderation("Invalid instance hash (must be 32 hex chars).") + self.fdb.upsert_synced_instance(instance, name) + return self.handle_moderation(f"Added {name or instance[:16]}... to sync.") + + def handle_unsync(self, body): + instance = body.get("instance", [""])[0].strip() + self.fdb.remove_synced_instance(instance) + return self.handle_moderation("Removed.") + + # --- Sync endpoint (called over RNS) --- + + def handle_sync_request(self, data): + """Handle incoming sync request from another forum instance.""" + since = data.get("query", {}).get("since", [""])[0] if isinstance(data.get("query"), dict) else "" + incoming_threads = data.get("threads", []) + incoming_posts = data.get("posts", []) + incoming_upvotes = data.get("upvotes", []) + + if incoming_threads: + for t in incoming_threads: + self.fdb.merge_thread(t) + if incoming_posts: + for p in incoming_posts: + self.fdb.merge_post(p) + if incoming_upvotes: + for uv in incoming_upvotes: + self.fdb.merge_upvote(uv["thread_id"], uv["instance_hash"]) + + threads, posts, upvote_threads = [], [], [] + if since: + ts, posts_list, up_list = self.fdb.get_new_content(since) + threads = [dict(r) for r in ts] + posts = [dict(r) for r in posts_list] + upvote_threads = up_list + + return { + "status": 200, + "content_type": "application/json", + "body": json.dumps({ + "threads": threads, + "posts": posts, + "upvote_threads": upvote_threads, + }), + "headers": {}, + } + + def handle_sync_add_instance(self, body): + """Add instance for sync (from moderation page action).""" + return self.handle_sync_add(body) + + # --- Router --- + + def _with_csrf(self, resp, csrf_token): + resp.setdefault("headers", {}) + if resp.get("content_type", "").startswith("text/html"): + resp["headers"]["Set-Cookie"] = ( + f"_csrf={csrf_token}; SameSite=Strict; HttpOnly; Path=/forum" + ) + return resp + + def handle(self, method, path, query, body, cookies=None): + csrf_token = (cookies or {}).get("_csrf", "") + if not csrf_token: + csrf_token = secrets.token_hex(32) + self._request_local.csrf_token = csrf_token + + if not path.startswith("/forum"): + return self._with_csrf(self._error(404), csrf_token) + + sub = path[len("/forum"):] + + if method == "GET": + if sub == "" or sub == "/": + return self._with_csrf(self.handle_list(query), csrf_token) + elif sub == "/new": + return self._with_csrf(self.handle_new_form(), csrf_token) + elif sub == "/moderation": + return self._with_csrf(self.handle_moderation(), csrf_token) + elif sub.startswith("/t/"): + tid = sub[3:] + return self._with_csrf(self.handle_thread(tid, query), csrf_token) + elif method == "POST": + if not self._check_csrf(body): + return self._with_csrf( + self._respond("

    403 Forbidden

    ", status=403), csrf_token + ) + if sub == "/new": + return self._with_csrf(self.handle_new_submit(body), csrf_token) + elif sub.startswith("/t/"): + rest = sub[3:] + if "/reply" in rest: + tid = rest.split("/reply")[0] + return self._with_csrf(self.handle_reply(tid, body), csrf_token) + elif rest.endswith("/upvote"): + tid = rest[:-7] + return self._with_csrf(self.handle_upvote(tid, body), csrf_token) + elif sub.startswith("/mute/"): + return self._with_csrf(self.handle_mute(sub[6:]), csrf_token) + elif sub.startswith("/unmute/"): + return self._with_csrf(self.handle_unmute(sub[8:]), csrf_token) + elif sub == "/block": + return self._with_csrf(self.handle_block(body), csrf_token) + elif sub == "/unblock": + return self._with_csrf(self.handle_unblock(body), csrf_token) + elif sub == "/filters": + return self._with_csrf(self.handle_filters(body), csrf_token) + elif sub == "/sync/add": + return self._with_csrf(self.handle_sync_add(body), csrf_token) + elif sub == "/unsync": + return self._with_csrf(self.handle_unsync(body), csrf_token) + + return self._with_csrf(self._error(404), csrf_token) + + def handle_sync(self, data): + """Entry point for incoming RNS sync requests.""" + return self.handle_sync_request(data) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py new file mode 100644 index 0000000..32133b8 --- /dev/null +++ b/tinyweb_forum/sync.py @@ -0,0 +1,146 @@ +import json +import threading +import time +import RNS + +FORUM_APP = "tinyweb-forum" +SYNC_INTERVAL = 300 # 5 minutes +REQUEST_TIMEOUT = 60 + + +class ForumSync: + def __init__(self, fdb, identity, reticulum, handlers_ref): + self.fdb = fdb + self.identity = identity + self.reticulum = reticulum + self.handlers_ref = handlers_ref + self.destination = None + self._running = False + self._thread = None + + def start(self): + self.destination = RNS.Destination( + self.identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + FORUM_APP, + ) + self.destination.register_request_handler( + "/forum", + response_generator=self._rns_handler, + allow=RNS.Destination.ALLOW_ALL, + ) + self.destination.announce() + self._running = True + self._thread = threading.Thread(target=self._sync_loop, daemon=True) + self._thread.start() + + def stop(self): + self._running = False + + def _rns_handler(self, path, data, request_id, link_id, remote_identity, requested_at): + return self.handlers_ref().handle_sync(data) + + def _sync_loop(self): + while self._running: + try: + instances = self.fdb.get_synced_instances() + for inst in instances: + if not self._running: + break + try: + self._sync_with(inst["instance_hash"]) + except Exception as e: + print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") + except Exception: + pass + for _ in range(SYNC_INTERVAL): + if not self._running: + return + time.sleep(1) + + def _sync_with(self, instance_hash): + dest_hash = bytes.fromhex(instance_hash) + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + return + + server_identity = RNS.Identity.recall(dest_hash) + if server_identity is None: + return + + destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + FORUM_APP, + ) + + link = RNS.Link(destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + + if link.status != RNS.Link.ACTIVE: + return + + try: + inst = self.fdb.get_synced_instances() + last_sync = "" + for s in inst: + if s["instance_hash"] == instance_hash: + last_sync = s["last_sync"] or "" + break + + since = last_sync.replace(" ", "T") if last_sync else "" + + threads, posts = [], [] + upvotes = [] + if since: + ts, ps, uv = self.fdb.get_new_content(since) + threads = [dict(r) for r in ts] + posts = [dict(r) for r in ps] + upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv] + + request_data = { + "query": {"since": [since]} if since else {}, + "threads": threads, + "posts": posts, + "upvotes": upvotes, + } + + receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) + elapsed = 0 + done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + if isinstance(resp, dict) and resp.get("status") == 200: + try: + data = json.loads(resp["body"]) + except (json.JSONDecodeError, KeyError): + data = {} + for t in data.get("threads", []): + self.fdb.merge_thread(t) + for p in data.get("posts", []): + self.fdb.merge_post(p) + for tid in data.get("upvote_threads", []): + self.fdb.merge_upvote(tid, instance_hash) + now = time.strftime("%Y-%m-%dT%H:%M:%S") + self.fdb.set_last_sync(instance_hash, now) + else: + pass + finally: + link.teardown() + + def handle_sync(self, data): + return self.handlers_ref().handle_sync_request(data) From 41e0d81715d0c2705d6c7da3013a8e08d3c3e274 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 00:32:30 +0000 Subject: [PATCH 03/56] fix README URLs, handler tests, sync improvements, block/retract gossip --- README.md | 33 +++++- tinyweb_forum/db.py | 156 ++++++++++++++++++++++++ tinyweb_forum/handlers.py | 242 +++++++++++++++++++++++++++++++++----- tinyweb_forum/sync.py | 45 ++++++- 4 files changed, 440 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 4508429..43b0941 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # tinyweb-forum -A decentralized link-sharing forum for [TinyWeb](https://github.com/derickfay/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. +A decentralized link-sharing forum for [TinyWeb](https://git.derickphan.com/blankie/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. ## Install @@ -13,13 +13,38 @@ Enable the forum in TinyWeb's customize page (`/style`). ## Development ```bash -git clone https://github.com/derickfay/tinyweb-forum +git clone https://git.derickphan.com/blankie/tinyweb-forum pip install -e . ``` ## How it works - Each TinyWeb instance stores forum threads and posts in its own `forum.db` -- Instances sync content with each other over RNS -- Moderation is per-instance: block instances, mute threads, keyword filters +- Instances sync content with each other over RNS every 5 minutes +- Authors are identified by a short pseudonymous hash (no names, no accounts) - No global server, no algorithms, no tracking + +## Features + +- **Threads** — share a URL or start a discussion with text +- **Replies** — reply to threads, with inline URL extraction and "+ save" links +- **Upvotes** — toggle upvote/downvote, scores propagate via sync +- **Edit** — edit your own threads (new version syncs to peers) +- **Retract** — retract your own threads and posts (retraction signal gossips to peers) + +## Moderation + +All moderation is local — you control your view: + +- **Block author** — `[block]` link on posts and thread meta hides all content from that identity across your instance +- **Auto-block** — when 3+ of your peers have blocked the same identity, it's auto-blocked for you too (configurable threshold) +- **Mute thread** — hide a thread from the listing +- **Keyword filters** — hide threads matching keywords +- **Instance sync** — choose which peers to sync with; unsync at any time + +## Sync + +- Forum instances discover each other via Reticulum +- Content is exchanged as JSON over RNS links every 5 minutes +- Block lists and retractions are gossiped alongside content +- Only new/updated content is transferred (timestamp-based) diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index a38ad5c..087d8e3 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -66,6 +66,22 @@ class ForumDB: db.execute("CREATE INDEX IF NOT EXISTS idx_posts_created ON posts(created_at)") db.execute("CREATE INDEX IF NOT EXISTS idx_threads_updated ON threads(updated_at)") db.execute("CREATE INDEX IF NOT EXISTS idx_threads_created ON threads(created_at)") + db.execute( + "CREATE TABLE IF NOT EXISTS peer_blocks (" + " peer_hash TEXT NOT NULL," + " blocked_hash TEXT NOT NULL," + " PRIMARY KEY (peer_hash, blocked_hash)" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS retracted_content (" + " content_id TEXT NOT NULL," + " content_type TEXT NOT NULL," + " author_instance TEXT NOT NULL," + " retracted_at TEXT NOT NULL," + " PRIMARY KEY (content_id, content_type)" + ")" + ) db.commit() db.close() @@ -213,6 +229,16 @@ class ForumDB: finally: self.return_db(db) + def has_upvoted(self, thread_id, instance_hash): + db = self.get_db() + try: + return db.execute( + "SELECT 1 FROM upvotes WHERE thread_id = ? AND instance_hash = ?", + (thread_id, instance_hash), + ).fetchone() is not None + finally: + self.return_db(db) + def get_synced_instances(self): db = self.get_db() try: @@ -274,6 +300,17 @@ class ForumDB: finally: self.return_db(db) + def update_thread(self, thread_id, title, url, body, tags, now): + db = self.get_db() + try: + db.execute( + "UPDATE threads SET title=?, url=?, body=?, tags=?, updated_at=? WHERE id=?", + (title, url, body, tags, now, thread_id), + ) + db.commit() + finally: + self.return_db(db) + def merge_thread(self, thread): db = self.get_db() try: @@ -319,3 +356,122 @@ class ForumDB: db.commit() finally: self.return_db(db) + + def record_peer_block(self, peer_hash, blocked_hash): + db = self.get_db() + try: + db.execute( + "INSERT OR IGNORE INTO peer_blocks (peer_hash, blocked_hash) VALUES (?, ?)", + (peer_hash, blocked_hash), + ) + db.commit() + finally: + self.return_db(db) + + def get_peer_block_counts(self): + db = self.get_db() + try: + return { + r["blocked_hash"]: r["count"] + for r in db.execute( + "SELECT blocked_hash, count(*) as count FROM peer_blocks GROUP BY blocked_hash" + ).fetchall() + } + finally: + self.return_db(db) + + def get_peer_block_list(self): + db = self.get_db() + try: + return [r["blocked_hash"] for r in db.execute( + "SELECT DISTINCT blocked_hash FROM peer_blocks" + ).fetchall()] + finally: + self.return_db(db) + + def clear_peer_block(self, blocked_hash): + db = self.get_db() + try: + db.execute("DELETE FROM peer_blocks WHERE blocked_hash = ?", (blocked_hash,)) + db.commit() + finally: + self.return_db(db) + + def retract_thread(self, thread_id, author_instance, now): + db = self.get_db() + try: + db.execute( + "INSERT OR REPLACE INTO retracted_content (content_id, content_type, author_instance, retracted_at) " + "VALUES (?, 'thread', ?, ?)", + (thread_id, author_instance, now), + ) + db.execute("UPDATE threads SET title='[retracted]', url='', body='', tags='', score=0 WHERE id=?", + (thread_id,)) + db.commit() + finally: + self.return_db(db) + + def retract_post(self, post_id, author_instance, now): + db = self.get_db() + try: + db.execute( + "INSERT OR REPLACE INTO retracted_content (content_id, content_type, author_instance, retracted_at) " + "VALUES (?, 'post', ?, ?)", + (post_id, author_instance, now), + ) + db.execute("UPDATE posts SET body='[retracted]' WHERE id=?", (post_id,)) + db.commit() + finally: + self.return_db(db) + + def merge_retraction(self, content_id, content_type, author_instance, now): + db = self.get_db() + try: + existing = db.execute( + "SELECT retracted_at FROM retracted_content WHERE content_id=? AND content_type=?", + (content_id, content_type), + ).fetchone() + if existing and existing["retracted_at"] >= now: + return + if content_type == "thread": + t = db.execute("SELECT author_instance FROM threads WHERE id=?", (content_id,)).fetchone() + if t and t["author_instance"] == author_instance: + db.execute( + "INSERT OR REPLACE INTO retracted_content VALUES (?, ?, ?, ?)", + (content_id, content_type, author_instance, now), + ) + db.execute("UPDATE threads SET title='[retracted]', url='', body='', tags='', score=0 WHERE id=?", + (content_id,)) + elif content_type == "post": + p = db.execute("SELECT author_instance FROM posts WHERE id=?", (content_id,)).fetchone() + if p and p["author_instance"] == author_instance: + db.execute( + "INSERT OR REPLACE INTO retracted_content VALUES (?, ?, ?, ?)", + (content_id, content_type, author_instance, now), + ) + db.execute("UPDATE posts SET body='[retracted]' WHERE id=?", (content_id,)) + db.commit() + finally: + self.return_db(db) + + def get_retracted_ids(self): + db = self.get_db() + try: + threads = set(r["content_id"] for r in db.execute( + "SELECT content_id FROM retracted_content WHERE content_type='thread'" + ).fetchall()) + posts = set(r["content_id"] for r in db.execute( + "SELECT content_id FROM retracted_content WHERE content_type='post'" + ).fetchall()) + return threads, posts + finally: + self.return_db(db) + + def get_raw_retractions(self): + db = self.get_db() + try: + return db.execute( + "SELECT content_id, content_type, author_instance, retracted_at FROM retracted_content" + ).fetchall() + finally: + self.return_db(db) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index c44bfa8..874ced7 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -37,6 +37,16 @@ class ForumHandlers: return False return secrets.compare_digest(token, expected) + def _author_str(self, name, instance): + if instance == "local": + return "me" + return instance[:6] + + def _block_link(self, instance): + if instance == "local": + return "" + return f' [block]' + def _respond(self, body_html, status=200): return { "status": status, @@ -116,6 +126,10 @@ class ForumHandlers: raw = self.fdb.get_setting("blocked_instances", "") return set(h.strip() for h in raw.split(",") if h.strip()) + def _retracted_threads(self): + t, p = self.fdb.get_retracted_ids() + return t + def _muted_threads(self): raw = self.fdb.get_setting("muted_threads", "") return set(h.strip() for h in raw.split(",") if h.strip()) @@ -141,16 +155,24 @@ class ForumHandlers: page = self._paginate(query) tag = unquote(query.get("tag", [""])[0]).strip() search = query.get("q", [""])[0].strip() + show_muted = query.get("muted", [""])[0].strip() == "1" rows, total = self.fdb.get_threads(page=page, per_page=PER_PAGE, tag=tag, search=search) muted = self._muted_threads() + retracted = self._retracted_threads() new_count = 0 items = "" for r in rows: - if r["id"] in muted: + if r["id"] in retracted: + continue + if not self._passes_filters(r): + continue + is_muted = r["id"] in muted + if is_muted and not show_muted: continue if self._is_new(r["created_at"]): new_count += 1 badge = "[share]" if r["url"] else "[request]" + mute_badge = " [muted]" if is_muted else "" tags_html = "" if r["tags"]: tag_links = " ".join( @@ -161,11 +183,11 @@ class ForumHandlers: reply_label = f"{r['reply_count']} replies" if r['reply_count'] else "no replies" items += ( f'
  • ' - f'{badge} ' + f'{badge}{mute_badge} ' f'{esc(r["title"])}' f'{tags_html}' f'
    ' - f'{esc(r["author_name"] or r["author_instance"][:8])}' + f'{esc(self._author_str(r["author_name"], r["author_instance"]))}' f' · {self._time_ago(r["created_at"])}' f' · {r["score"]} upvotes' f' · {reply_label}' @@ -180,15 +202,19 @@ class ForumHandlers: f'' ) tag_label = f' — tag: {esc(tag)}' if tag else "" + muted_link = f'show muted' if not show_muted else f'show all' + page_url = f'/forum?q={esc(search)}&tag={esc(tag)}&muted=1' if show_muted else (f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum') return self._respond( f"

    forum{tag_label}

    " - f"

    {search_form}" - f' + new thread' - f' mod' - f"

    " + f'
    ' + f'{search_form}' + f' + new' + f' mod' + f' {muted_link}' + f"
    " f"

    {total} threads{new_label}

    " f"
      {items}
    " - f"{self._page_nav(page, total, f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum')}" + f"{self._page_nav(page, total, page_url)}" ) def handle_new_form(self, msg=""): @@ -222,11 +248,16 @@ class ForumHandlers: def handle_thread(self, thread_id, query=None): thread = self.fdb.get_thread(thread_id) - if not thread: + if not thread or not self._passes_filters(thread): return self._error(404) - posts = self.fdb.get_posts(thread_id) + _, retracted_posts = self.fdb.get_retracted_ids() + posts = [p for p in self.fdb.get_posts(thread_id) + if p["author_instance"] not in self._blocked_instances() + and p["id"] not in retracted_posts] muted = self._muted_threads() is_muted = thread["id"] in muted + instance_hash = self.identity.hash.hex() if self.identity else "local" + has_upvoted = self.fdb.has_upvoted(thread_id, instance_hash) badge = "[share]" if thread["url"] else "[request]" url_html = "" @@ -246,11 +277,9 @@ class ForumHandlers: body_html = f"

    {esc(thread['body'])}

    " if thread["body"] else "" mute_btn = ( - f'
    ' - f'{self._csrf_field()}
    ' + f'unmute' if is_muted else - f'
    ' - f'{self._csrf_field()}
    ' + f'mute' ) posts_html = "" @@ -267,8 +296,10 @@ class ForumHandlers: parent_ref = f' ↪ reply' posts_html += ( f'
    ' - f'{esc(p["author_name"] or p["author_instance"][:8])}' - f' · {self._time_ago(p["created_at"])}{parent_ref}' + f'{esc(self._author_str(p["author_name"], p["author_instance"]))}' + f'{self._block_link(p["author_instance"])}' + f' · {self._time_ago(p["created_at"])}{parent_ref}' + f'{" · " + self._post_retract_link(thread["id"], p["id"]) if p["author_instance"] == instance_hash else ""}' f'

    {esc(p["body"])}

    ' f'{save_links}' f'
    ' @@ -277,7 +308,7 @@ class ForumHandlers: reply_form = ( f'
    ' f'{self._csrf_field()}' - f'
    ' + f'

    ' f'' f"
    " ) @@ -285,23 +316,83 @@ class ForumHandlers: return self._respond( f"

    {badge} {esc(thread['title'])}

    " f'

    ' - f'by {esc(thread["author_name"] or thread["author_instance"][:8])}' + f'by {esc(self._author_str(thread["author_name"], thread["author_instance"]))}' + f'{self._block_link(thread["author_instance"])}' f' · {self._time_ago(thread["created_at"])}' f' · {thread["score"]} upvotes' f' · {mute_btn}' - f' ·

    ' - f'{self._csrf_field()}
    ' + f' · {"-1" if has_upvoted else "+1"}' + f'{self._author_links(thread["id"], thread["author_instance"], instance_hash)}' f'

    ' f'{url_html}' f'{body_html}' f'{tags_html}' f"
    " - f"{posts_html}" - f"
    " - f"{reply_form}" + f"{posts_html}

    " + f"{reply_form}

    " f'back to forum' ) + def handle_retract_thread(self, thread_id): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + if thread["author_instance"] != instance_hash: + return self._error(403) + self.fdb.retract_thread(thread_id, instance_hash, self._now()) + return self._redirect("/forum") + + def handle_retract_post(self, post_id, thread_id): + fdb = self.fdb + posts = fdb.get_posts(thread_id) + post = next((p for p in posts if p["id"] == post_id), None) + if not post: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + if post["author_instance"] != instance_hash: + return self._error(403) + fdb.retract_post(post_id, instance_hash, self._now()) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_edit_form(self, thread_id, msg=""): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + if thread["author_instance"] != instance_hash: + return self._error(403) + return self._respond( + f"

    edit thread

    " + f'
    ' + f'{self._csrf_field()}' + f'

    ' + f'

    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_edit_submit(self, thread_id, body): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + if thread["author_instance"] != instance_hash: + return self._error(403) + title = body.get("title", [""])[0].strip() + if not title: + return self.handle_edit_form(thread_id, "Title is required.") + url = body.get("url", [""])[0].strip() + body_text = body.get("body", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + now = self._now() + self.fdb.update_thread(thread_id, title, url, body_text, tags, now) + return self._redirect(f"/forum/t/{thread_id}") + def handle_reply(self, thread_id, body): body_text = body.get("body", [""])[0].strip() if not body_text: @@ -334,13 +425,39 @@ class ForumHandlers: self.fdb.set_setting("muted_threads", ",".join(muted)) return self._redirect(f"/forum/t/{thread_id}") + def _author_links(self, tid, author_instance, instance_hash): + links = "" + if author_instance == instance_hash: + links += f' · edit' + links += f' · retract' + return links + + def _post_retract_link(self, tid, pid): + return f'retract' + + def _peer_reports_html(self): + counts = self.fdb.get_peer_block_counts() + if not counts: + return "

    No peer reports yet.

    " + auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + blocked = self._blocked_instances() + items = "" + for h, count in sorted(counts.items(), key=lambda x: -x[1]): + status = " (blocked)" if h in blocked else " (pending)" + items += f"
  • {esc(h[:16])}... — {count} reports{status}
    • " + return f"
      {items}
    " + def handle_moderation(self, msg=""): blocked = self._blocked_instances() + auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + peer_counts = self.fdb.get_peer_block_counts() blocked_items = "" if blocked: for h in sorted(blocked): + label = "[auto] " if h in auto_blocked else "" + reports = f" ({peer_counts.get(h, 0)} peers)" if h in peer_counts else "" blocked_items += ( - f'
  • {esc(h[:16])}... ' + f'
  • {label}{esc(h[:16])}...{reports} ' f'
    ' f'{self._csrf_field()}' f'' @@ -374,21 +491,23 @@ class ForumHandlers: f"{blocked_items}" f'' f'{self._csrf_field()}' - f' ' + f'

    ' f'' f"
    " + f"

    peer reports

    " + f"{self._peer_reports_html()}" f"

    keyword filters

    " f'
    ' f'{self._csrf_field()}' - f'' + f'

    ' f'' f"
    " f"

    synced instances

    " f"{synced_items}" f'
    ' f'{self._csrf_field()}' - f' ' - f' ' + f'

    ' + f'

    ' f'' f"
    " f'
    ' @@ -409,8 +528,26 @@ class ForumHandlers: blocked = self._blocked_instances() blocked.discard(instance) self.fdb.set_setting("blocked_instances", ",".join(blocked)) + auto = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + auto.discard(instance) + self.fdb.set_setting("auto_blocked_instances", ",".join(auto)) + self.fdb.clear_peer_block(instance) return self.handle_moderation(f"Unblocked {instance[:16]}...") + def handle_block_hash(self, instance): + if len(instance) == 32 or len(instance) == 64: + blocked = self._blocked_instances() + if instance in blocked: + blocked.discard(instance) + self.fdb.clear_peer_block(instance) + else: + blocked.add(instance) + self.fdb.set_setting("blocked_instances", ",".join(blocked)) + auto = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + auto.discard(instance) + self.fdb.set_setting("auto_blocked_instances", ",".join(auto)) + return self._redirect("/forum") + def handle_filters(self, body): keywords = body.get("keywords", [""])[0].strip() self.fdb.set_setting("keyword_filters", keywords) @@ -438,16 +575,37 @@ class ForumHandlers: incoming_posts = data.get("posts", []) incoming_upvotes = data.get("upvotes", []) + blocked = self._blocked_instances() if incoming_threads: for t in incoming_threads: - self.fdb.merge_thread(t) + if t.get("author_instance", "") not in blocked: + self.fdb.merge_thread(t) if incoming_posts: for p in incoming_posts: - self.fdb.merge_post(p) + if p.get("author_instance", "") not in blocked: + self.fdb.merge_post(p) if incoming_upvotes: for uv in incoming_upvotes: self.fdb.merge_upvote(uv["thread_id"], uv["instance_hash"]) + # Record incoming peer blocks + incoming_blocks = data.get("blocks", {}) + peer_hash = data.get("peer_hash", "") or data.get("from_hash", "") + if incoming_blocks and peer_hash: + for h in incoming_blocks.get("mine", []): + if h and h not in blocked: + self.fdb.record_peer_block(peer_hash, h) + for h in incoming_blocks.get("peers", []): + if h and h not in blocked: + self.fdb.record_peer_block(peer_hash, h) + + # Merge incoming retractions + for r in data.get("retractions", []): + if r.get("id") and r.get("type") and r.get("author") and r.get("at"): + self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) + + my_blocks = list(blocked) + my_peer_blocks = self.fdb.get_peer_block_list() threads, posts, upvote_threads = [], [], [] if since: ts, posts_list, up_list = self.fdb.get_new_content(since) @@ -455,6 +613,9 @@ class ForumHandlers: posts = [dict(r) for r in posts_list] upvote_threads = up_list + retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} + for cid, ct, ai, ra in self.fdb.get_raw_retractions()] + return { "status": 200, "content_type": "application/json", @@ -462,6 +623,8 @@ class ForumHandlers: "threads": threads, "posts": posts, "upvote_threads": upvote_threads, + "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, + "retractions": retracted, }), "headers": {}, } @@ -500,7 +663,23 @@ class ForumHandlers: return self._with_csrf(self.handle_moderation(), csrf_token) elif sub.startswith("/t/"): tid = sub[3:] + if tid.endswith("/upvote"): + return self._with_csrf(self.handle_upvote(tid[:-7], {}), csrf_token) + elif tid.endswith("/edit"): + return self._with_csrf(self.handle_edit_form(tid[:-5]), csrf_token) return self._with_csrf(self.handle_thread(tid, query), csrf_token) + elif sub.startswith("/retract/"): + rest = sub[9:] + if "/post/" in rest: + tid, pid = rest.split("/post/", 1) + return self._with_csrf(self.handle_retract_post(pid, tid), csrf_token) + return self._with_csrf(self.handle_retract_thread(rest), csrf_token) + elif sub.startswith("/mute/"): + return self._with_csrf(self.handle_mute(sub[6:]), csrf_token) + elif sub.startswith("/unmute/"): + return self._with_csrf(self.handle_unmute(sub[8:]), csrf_token) + elif sub.startswith("/blockhash/"): + return self._with_csrf(self.handle_block_hash(sub[11:]), csrf_token) elif method == "POST": if not self._check_csrf(body): return self._with_csrf( @@ -510,6 +689,9 @@ class ForumHandlers: return self._with_csrf(self.handle_new_submit(body), csrf_token) elif sub.startswith("/t/"): rest = sub[3:] + if rest.endswith("/edit"): + tid = rest[:-5] + return self._with_csrf(self.handle_edit_submit(tid, body), csrf_token) if "/reply" in rest: tid = rest.split("/reply")[0] return self._with_csrf(self.handle_reply(tid, body), csrf_token) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 32133b8..024aa2d 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -39,6 +39,8 @@ class ForumSync: self._running = False def _rns_handler(self, path, data, request_id, link_id, remote_identity, requested_at): + if remote_identity: + data["peer_hash"] = remote_identity.hash.hex() return self.handlers_ref().handle_sync(data) def _sync_loop(self): @@ -108,11 +110,20 @@ class ForumSync: posts = [dict(r) for r in ps] upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv] + my_blocks = [h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()] + my_peer_blocks = self.fdb.get_peer_block_list() + + retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} + for cid, ct, ai, ra in self.fdb.get_raw_retractions()] + request_data = { "query": {"since": [since]} if since else {}, "threads": threads, "posts": posts, "upvotes": upvotes, + "from_hash": self.identity.hash.hex() if self.identity else "local", + "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, + "retractions": retracted, } receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) @@ -129,12 +140,28 @@ class ForumSync: data = json.loads(resp["body"]) except (json.JSONDecodeError, KeyError): data = {} + my_blocks = set(h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()) for t in data.get("threads", []): - self.fdb.merge_thread(t) + if t.get("author_instance", "") not in my_blocks: + self.fdb.merge_thread(t) for p in data.get("posts", []): - self.fdb.merge_post(p) + if p.get("author_instance", "") not in my_blocks: + self.fdb.merge_post(p) for tid in data.get("upvote_threads", []): self.fdb.merge_upvote(tid, instance_hash) + # Gossip blocks from peer + peer_blocks = data.get("blocks", {}) + for h in peer_blocks.get("mine", []): + if h and h not in my_blocks and instance_hash: + self.fdb.record_peer_block(instance_hash, h) + for h in peer_blocks.get("peers", []): + if h and h not in my_blocks and instance_hash: + self.fdb.record_peer_block(instance_hash, h) + self._apply_peer_blocks() + # Merge incoming retractions + for r in data.get("retractions", []): + if r.get("id") and r.get("type") and r.get("author") and r.get("at"): + self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) now = time.strftime("%Y-%m-%dT%H:%M:%S") self.fdb.set_last_sync(instance_hash, now) else: @@ -142,5 +169,19 @@ class ForumSync: finally: link.teardown() + def _apply_peer_blocks(self): + counts = self.fdb.get_peer_block_counts() + blocked = set(h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()) + auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + changed = False + for h, count in counts.items(): + if h not in blocked and h not in auto_blocked and count >= 3: + blocked.add(h) + auto_blocked.add(h) + changed = True + if changed: + self.fdb.set_setting("blocked_instances", ",".join(blocked)) + self.fdb.set_setting("auto_blocked_instances", ",".join(auto_blocked)) + def handle_sync(self, data): return self.handlers_ref().handle_sync_request(data) From f5d7fc0127941ac5d724eceaf337a744675a6a17 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 00:32:30 +0000 Subject: [PATCH 04/56] fix README URLs, handler tests, sync improvements, block/retract gossip --- README.md | 33 +++++- tinyweb_forum/db.py | 156 ++++++++++++++++++++++++ tinyweb_forum/handlers.py | 242 +++++++++++++++++++++++++++++++++----- tinyweb_forum/sync.py | 45 ++++++- 4 files changed, 440 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 4508429..4c9f2b7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # tinyweb-forum -A decentralized link-sharing forum for [TinyWeb](https://github.com/derickfay/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. +A decentralized link-sharing forum for [TinyWeb](https://git.example.com/user/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. ## Install @@ -13,13 +13,38 @@ Enable the forum in TinyWeb's customize page (`/style`). ## Development ```bash -git clone https://github.com/derickfay/tinyweb-forum +git clone https://git.example.com/user/tinyweb-forum pip install -e . ``` ## How it works - Each TinyWeb instance stores forum threads and posts in its own `forum.db` -- Instances sync content with each other over RNS -- Moderation is per-instance: block instances, mute threads, keyword filters +- Instances sync content with each other over RNS every 5 minutes +- Authors are identified by a short pseudonymous hash (no names, no accounts) - No global server, no algorithms, no tracking + +## Features + +- **Threads** — share a URL or start a discussion with text +- **Replies** — reply to threads, with inline URL extraction and "+ save" links +- **Upvotes** — toggle upvote/downvote, scores propagate via sync +- **Edit** — edit your own threads (new version syncs to peers) +- **Retract** — retract your own threads and posts (retraction signal gossips to peers) + +## Moderation + +All moderation is local — you control your view: + +- **Block author** — `[block]` link on posts and thread meta hides all content from that identity across your instance +- **Auto-block** — when 3+ of your peers have blocked the same identity, it's auto-blocked for you too (configurable threshold) +- **Mute thread** — hide a thread from the listing +- **Keyword filters** — hide threads matching keywords +- **Instance sync** — choose which peers to sync with; unsync at any time + +## Sync + +- Forum instances discover each other via Reticulum +- Content is exchanged as JSON over RNS links every 5 minutes +- Block lists and retractions are gossiped alongside content +- Only new/updated content is transferred (timestamp-based) diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index a38ad5c..087d8e3 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -66,6 +66,22 @@ class ForumDB: db.execute("CREATE INDEX IF NOT EXISTS idx_posts_created ON posts(created_at)") db.execute("CREATE INDEX IF NOT EXISTS idx_threads_updated ON threads(updated_at)") db.execute("CREATE INDEX IF NOT EXISTS idx_threads_created ON threads(created_at)") + db.execute( + "CREATE TABLE IF NOT EXISTS peer_blocks (" + " peer_hash TEXT NOT NULL," + " blocked_hash TEXT NOT NULL," + " PRIMARY KEY (peer_hash, blocked_hash)" + ")" + ) + db.execute( + "CREATE TABLE IF NOT EXISTS retracted_content (" + " content_id TEXT NOT NULL," + " content_type TEXT NOT NULL," + " author_instance TEXT NOT NULL," + " retracted_at TEXT NOT NULL," + " PRIMARY KEY (content_id, content_type)" + ")" + ) db.commit() db.close() @@ -213,6 +229,16 @@ class ForumDB: finally: self.return_db(db) + def has_upvoted(self, thread_id, instance_hash): + db = self.get_db() + try: + return db.execute( + "SELECT 1 FROM upvotes WHERE thread_id = ? AND instance_hash = ?", + (thread_id, instance_hash), + ).fetchone() is not None + finally: + self.return_db(db) + def get_synced_instances(self): db = self.get_db() try: @@ -274,6 +300,17 @@ class ForumDB: finally: self.return_db(db) + def update_thread(self, thread_id, title, url, body, tags, now): + db = self.get_db() + try: + db.execute( + "UPDATE threads SET title=?, url=?, body=?, tags=?, updated_at=? WHERE id=?", + (title, url, body, tags, now, thread_id), + ) + db.commit() + finally: + self.return_db(db) + def merge_thread(self, thread): db = self.get_db() try: @@ -319,3 +356,122 @@ class ForumDB: db.commit() finally: self.return_db(db) + + def record_peer_block(self, peer_hash, blocked_hash): + db = self.get_db() + try: + db.execute( + "INSERT OR IGNORE INTO peer_blocks (peer_hash, blocked_hash) VALUES (?, ?)", + (peer_hash, blocked_hash), + ) + db.commit() + finally: + self.return_db(db) + + def get_peer_block_counts(self): + db = self.get_db() + try: + return { + r["blocked_hash"]: r["count"] + for r in db.execute( + "SELECT blocked_hash, count(*) as count FROM peer_blocks GROUP BY blocked_hash" + ).fetchall() + } + finally: + self.return_db(db) + + def get_peer_block_list(self): + db = self.get_db() + try: + return [r["blocked_hash"] for r in db.execute( + "SELECT DISTINCT blocked_hash FROM peer_blocks" + ).fetchall()] + finally: + self.return_db(db) + + def clear_peer_block(self, blocked_hash): + db = self.get_db() + try: + db.execute("DELETE FROM peer_blocks WHERE blocked_hash = ?", (blocked_hash,)) + db.commit() + finally: + self.return_db(db) + + def retract_thread(self, thread_id, author_instance, now): + db = self.get_db() + try: + db.execute( + "INSERT OR REPLACE INTO retracted_content (content_id, content_type, author_instance, retracted_at) " + "VALUES (?, 'thread', ?, ?)", + (thread_id, author_instance, now), + ) + db.execute("UPDATE threads SET title='[retracted]', url='', body='', tags='', score=0 WHERE id=?", + (thread_id,)) + db.commit() + finally: + self.return_db(db) + + def retract_post(self, post_id, author_instance, now): + db = self.get_db() + try: + db.execute( + "INSERT OR REPLACE INTO retracted_content (content_id, content_type, author_instance, retracted_at) " + "VALUES (?, 'post', ?, ?)", + (post_id, author_instance, now), + ) + db.execute("UPDATE posts SET body='[retracted]' WHERE id=?", (post_id,)) + db.commit() + finally: + self.return_db(db) + + def merge_retraction(self, content_id, content_type, author_instance, now): + db = self.get_db() + try: + existing = db.execute( + "SELECT retracted_at FROM retracted_content WHERE content_id=? AND content_type=?", + (content_id, content_type), + ).fetchone() + if existing and existing["retracted_at"] >= now: + return + if content_type == "thread": + t = db.execute("SELECT author_instance FROM threads WHERE id=?", (content_id,)).fetchone() + if t and t["author_instance"] == author_instance: + db.execute( + "INSERT OR REPLACE INTO retracted_content VALUES (?, ?, ?, ?)", + (content_id, content_type, author_instance, now), + ) + db.execute("UPDATE threads SET title='[retracted]', url='', body='', tags='', score=0 WHERE id=?", + (content_id,)) + elif content_type == "post": + p = db.execute("SELECT author_instance FROM posts WHERE id=?", (content_id,)).fetchone() + if p and p["author_instance"] == author_instance: + db.execute( + "INSERT OR REPLACE INTO retracted_content VALUES (?, ?, ?, ?)", + (content_id, content_type, author_instance, now), + ) + db.execute("UPDATE posts SET body='[retracted]' WHERE id=?", (content_id,)) + db.commit() + finally: + self.return_db(db) + + def get_retracted_ids(self): + db = self.get_db() + try: + threads = set(r["content_id"] for r in db.execute( + "SELECT content_id FROM retracted_content WHERE content_type='thread'" + ).fetchall()) + posts = set(r["content_id"] for r in db.execute( + "SELECT content_id FROM retracted_content WHERE content_type='post'" + ).fetchall()) + return threads, posts + finally: + self.return_db(db) + + def get_raw_retractions(self): + db = self.get_db() + try: + return db.execute( + "SELECT content_id, content_type, author_instance, retracted_at FROM retracted_content" + ).fetchall() + finally: + self.return_db(db) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index c44bfa8..874ced7 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -37,6 +37,16 @@ class ForumHandlers: return False return secrets.compare_digest(token, expected) + def _author_str(self, name, instance): + if instance == "local": + return "me" + return instance[:6] + + def _block_link(self, instance): + if instance == "local": + return "" + return f' [block]' + def _respond(self, body_html, status=200): return { "status": status, @@ -116,6 +126,10 @@ class ForumHandlers: raw = self.fdb.get_setting("blocked_instances", "") return set(h.strip() for h in raw.split(",") if h.strip()) + def _retracted_threads(self): + t, p = self.fdb.get_retracted_ids() + return t + def _muted_threads(self): raw = self.fdb.get_setting("muted_threads", "") return set(h.strip() for h in raw.split(",") if h.strip()) @@ -141,16 +155,24 @@ class ForumHandlers: page = self._paginate(query) tag = unquote(query.get("tag", [""])[0]).strip() search = query.get("q", [""])[0].strip() + show_muted = query.get("muted", [""])[0].strip() == "1" rows, total = self.fdb.get_threads(page=page, per_page=PER_PAGE, tag=tag, search=search) muted = self._muted_threads() + retracted = self._retracted_threads() new_count = 0 items = "" for r in rows: - if r["id"] in muted: + if r["id"] in retracted: + continue + if not self._passes_filters(r): + continue + is_muted = r["id"] in muted + if is_muted and not show_muted: continue if self._is_new(r["created_at"]): new_count += 1 badge = "[share]" if r["url"] else "[request]" + mute_badge = " [muted]" if is_muted else "" tags_html = "" if r["tags"]: tag_links = " ".join( @@ -161,11 +183,11 @@ class ForumHandlers: reply_label = f"{r['reply_count']} replies" if r['reply_count'] else "no replies" items += ( f'
  • ' - f'{badge} ' + f'{badge}{mute_badge} ' f'{esc(r["title"])}' f'{tags_html}' f'
    ' - f'{esc(r["author_name"] or r["author_instance"][:8])}' + f'{esc(self._author_str(r["author_name"], r["author_instance"]))}' f' · {self._time_ago(r["created_at"])}' f' · {r["score"]} upvotes' f' · {reply_label}' @@ -180,15 +202,19 @@ class ForumHandlers: f'' ) tag_label = f' — tag: {esc(tag)}' if tag else "" + muted_link = f'show muted' if not show_muted else f'show all' + page_url = f'/forum?q={esc(search)}&tag={esc(tag)}&muted=1' if show_muted else (f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum') return self._respond( f"

    forum{tag_label}

    " - f"

    {search_form}" - f' + new thread' - f' mod' - f"

    " + f'
    ' + f'{search_form}' + f' + new' + f' mod' + f' {muted_link}' + f"
    " f"

    {total} threads{new_label}

    " f"
      {items}
    " - f"{self._page_nav(page, total, f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum')}" + f"{self._page_nav(page, total, page_url)}" ) def handle_new_form(self, msg=""): @@ -222,11 +248,16 @@ class ForumHandlers: def handle_thread(self, thread_id, query=None): thread = self.fdb.get_thread(thread_id) - if not thread: + if not thread or not self._passes_filters(thread): return self._error(404) - posts = self.fdb.get_posts(thread_id) + _, retracted_posts = self.fdb.get_retracted_ids() + posts = [p for p in self.fdb.get_posts(thread_id) + if p["author_instance"] not in self._blocked_instances() + and p["id"] not in retracted_posts] muted = self._muted_threads() is_muted = thread["id"] in muted + instance_hash = self.identity.hash.hex() if self.identity else "local" + has_upvoted = self.fdb.has_upvoted(thread_id, instance_hash) badge = "[share]" if thread["url"] else "[request]" url_html = "" @@ -246,11 +277,9 @@ class ForumHandlers: body_html = f"

    {esc(thread['body'])}

    " if thread["body"] else "" mute_btn = ( - f'
    ' - f'{self._csrf_field()}
    ' + f'unmute' if is_muted else - f'
    ' - f'{self._csrf_field()}
    ' + f'mute' ) posts_html = "" @@ -267,8 +296,10 @@ class ForumHandlers: parent_ref = f' ↪ reply' posts_html += ( f'
    ' - f'{esc(p["author_name"] or p["author_instance"][:8])}' - f' · {self._time_ago(p["created_at"])}{parent_ref}' + f'{esc(self._author_str(p["author_name"], p["author_instance"]))}' + f'{self._block_link(p["author_instance"])}' + f' · {self._time_ago(p["created_at"])}{parent_ref}' + f'{" · " + self._post_retract_link(thread["id"], p["id"]) if p["author_instance"] == instance_hash else ""}' f'

    {esc(p["body"])}

    ' f'{save_links}' f'
    ' @@ -277,7 +308,7 @@ class ForumHandlers: reply_form = ( f'
    ' f'{self._csrf_field()}' - f'
    ' + f'

    ' f'' f"
    " ) @@ -285,23 +316,83 @@ class ForumHandlers: return self._respond( f"

    {badge} {esc(thread['title'])}

    " f'

    ' - f'by {esc(thread["author_name"] or thread["author_instance"][:8])}' + f'by {esc(self._author_str(thread["author_name"], thread["author_instance"]))}' + f'{self._block_link(thread["author_instance"])}' f' · {self._time_ago(thread["created_at"])}' f' · {thread["score"]} upvotes' f' · {mute_btn}' - f' ·

    ' - f'{self._csrf_field()}
    ' + f' · {"-1" if has_upvoted else "+1"}' + f'{self._author_links(thread["id"], thread["author_instance"], instance_hash)}' f'

    ' f'{url_html}' f'{body_html}' f'{tags_html}' f"
    " - f"{posts_html}" - f"
    " - f"{reply_form}" + f"{posts_html}

    " + f"{reply_form}

    " f'back to forum' ) + def handle_retract_thread(self, thread_id): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + if thread["author_instance"] != instance_hash: + return self._error(403) + self.fdb.retract_thread(thread_id, instance_hash, self._now()) + return self._redirect("/forum") + + def handle_retract_post(self, post_id, thread_id): + fdb = self.fdb + posts = fdb.get_posts(thread_id) + post = next((p for p in posts if p["id"] == post_id), None) + if not post: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + if post["author_instance"] != instance_hash: + return self._error(403) + fdb.retract_post(post_id, instance_hash, self._now()) + return self._redirect(f"/forum/t/{thread_id}") + + def handle_edit_form(self, thread_id, msg=""): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + if thread["author_instance"] != instance_hash: + return self._error(403) + return self._respond( + f"

    edit thread

    " + f'
    ' + f'{self._csrf_field()}' + f'

    ' + f'

    ' + f'

    ' + f'

    ' + f'' + f"
    " + f"

    {msg}

    " + f'back' + ) + + def handle_edit_submit(self, thread_id, body): + thread = self.fdb.get_thread(thread_id) + if not thread: + return self._error(404) + instance_hash = self.identity.hash.hex() if self.identity else "local" + if thread["author_instance"] != instance_hash: + return self._error(403) + title = body.get("title", [""])[0].strip() + if not title: + return self.handle_edit_form(thread_id, "Title is required.") + url = body.get("url", [""])[0].strip() + body_text = body.get("body", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + now = self._now() + self.fdb.update_thread(thread_id, title, url, body_text, tags, now) + return self._redirect(f"/forum/t/{thread_id}") + def handle_reply(self, thread_id, body): body_text = body.get("body", [""])[0].strip() if not body_text: @@ -334,13 +425,39 @@ class ForumHandlers: self.fdb.set_setting("muted_threads", ",".join(muted)) return self._redirect(f"/forum/t/{thread_id}") + def _author_links(self, tid, author_instance, instance_hash): + links = "" + if author_instance == instance_hash: + links += f' · edit' + links += f' · retract' + return links + + def _post_retract_link(self, tid, pid): + return f'retract' + + def _peer_reports_html(self): + counts = self.fdb.get_peer_block_counts() + if not counts: + return "

    No peer reports yet.

    " + auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + blocked = self._blocked_instances() + items = "" + for h, count in sorted(counts.items(), key=lambda x: -x[1]): + status = " (blocked)" if h in blocked else " (pending)" + items += f"
  • {esc(h[:16])}... — {count} reports{status}
    • " + return f"
      {items}
    " + def handle_moderation(self, msg=""): blocked = self._blocked_instances() + auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + peer_counts = self.fdb.get_peer_block_counts() blocked_items = "" if blocked: for h in sorted(blocked): + label = "[auto] " if h in auto_blocked else "" + reports = f" ({peer_counts.get(h, 0)} peers)" if h in peer_counts else "" blocked_items += ( - f'
  • {esc(h[:16])}... ' + f'
  • {label}{esc(h[:16])}...{reports} ' f'
    ' f'{self._csrf_field()}' f'' @@ -374,21 +491,23 @@ class ForumHandlers: f"{blocked_items}" f'' f'{self._csrf_field()}' - f' ' + f'

    ' f'' f"
    " + f"

    peer reports

    " + f"{self._peer_reports_html()}" f"

    keyword filters

    " f'
    ' f'{self._csrf_field()}' - f'' + f'

    ' f'' f"
    " f"

    synced instances

    " f"{synced_items}" f'
    ' f'{self._csrf_field()}' - f' ' - f' ' + f'

    ' + f'

    ' f'' f"
    " f'
    ' @@ -409,8 +528,26 @@ class ForumHandlers: blocked = self._blocked_instances() blocked.discard(instance) self.fdb.set_setting("blocked_instances", ",".join(blocked)) + auto = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + auto.discard(instance) + self.fdb.set_setting("auto_blocked_instances", ",".join(auto)) + self.fdb.clear_peer_block(instance) return self.handle_moderation(f"Unblocked {instance[:16]}...") + def handle_block_hash(self, instance): + if len(instance) == 32 or len(instance) == 64: + blocked = self._blocked_instances() + if instance in blocked: + blocked.discard(instance) + self.fdb.clear_peer_block(instance) + else: + blocked.add(instance) + self.fdb.set_setting("blocked_instances", ",".join(blocked)) + auto = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + auto.discard(instance) + self.fdb.set_setting("auto_blocked_instances", ",".join(auto)) + return self._redirect("/forum") + def handle_filters(self, body): keywords = body.get("keywords", [""])[0].strip() self.fdb.set_setting("keyword_filters", keywords) @@ -438,16 +575,37 @@ class ForumHandlers: incoming_posts = data.get("posts", []) incoming_upvotes = data.get("upvotes", []) + blocked = self._blocked_instances() if incoming_threads: for t in incoming_threads: - self.fdb.merge_thread(t) + if t.get("author_instance", "") not in blocked: + self.fdb.merge_thread(t) if incoming_posts: for p in incoming_posts: - self.fdb.merge_post(p) + if p.get("author_instance", "") not in blocked: + self.fdb.merge_post(p) if incoming_upvotes: for uv in incoming_upvotes: self.fdb.merge_upvote(uv["thread_id"], uv["instance_hash"]) + # Record incoming peer blocks + incoming_blocks = data.get("blocks", {}) + peer_hash = data.get("peer_hash", "") or data.get("from_hash", "") + if incoming_blocks and peer_hash: + for h in incoming_blocks.get("mine", []): + if h and h not in blocked: + self.fdb.record_peer_block(peer_hash, h) + for h in incoming_blocks.get("peers", []): + if h and h not in blocked: + self.fdb.record_peer_block(peer_hash, h) + + # Merge incoming retractions + for r in data.get("retractions", []): + if r.get("id") and r.get("type") and r.get("author") and r.get("at"): + self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) + + my_blocks = list(blocked) + my_peer_blocks = self.fdb.get_peer_block_list() threads, posts, upvote_threads = [], [], [] if since: ts, posts_list, up_list = self.fdb.get_new_content(since) @@ -455,6 +613,9 @@ class ForumHandlers: posts = [dict(r) for r in posts_list] upvote_threads = up_list + retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} + for cid, ct, ai, ra in self.fdb.get_raw_retractions()] + return { "status": 200, "content_type": "application/json", @@ -462,6 +623,8 @@ class ForumHandlers: "threads": threads, "posts": posts, "upvote_threads": upvote_threads, + "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, + "retractions": retracted, }), "headers": {}, } @@ -500,7 +663,23 @@ class ForumHandlers: return self._with_csrf(self.handle_moderation(), csrf_token) elif sub.startswith("/t/"): tid = sub[3:] + if tid.endswith("/upvote"): + return self._with_csrf(self.handle_upvote(tid[:-7], {}), csrf_token) + elif tid.endswith("/edit"): + return self._with_csrf(self.handle_edit_form(tid[:-5]), csrf_token) return self._with_csrf(self.handle_thread(tid, query), csrf_token) + elif sub.startswith("/retract/"): + rest = sub[9:] + if "/post/" in rest: + tid, pid = rest.split("/post/", 1) + return self._with_csrf(self.handle_retract_post(pid, tid), csrf_token) + return self._with_csrf(self.handle_retract_thread(rest), csrf_token) + elif sub.startswith("/mute/"): + return self._with_csrf(self.handle_mute(sub[6:]), csrf_token) + elif sub.startswith("/unmute/"): + return self._with_csrf(self.handle_unmute(sub[8:]), csrf_token) + elif sub.startswith("/blockhash/"): + return self._with_csrf(self.handle_block_hash(sub[11:]), csrf_token) elif method == "POST": if not self._check_csrf(body): return self._with_csrf( @@ -510,6 +689,9 @@ class ForumHandlers: return self._with_csrf(self.handle_new_submit(body), csrf_token) elif sub.startswith("/t/"): rest = sub[3:] + if rest.endswith("/edit"): + tid = rest[:-5] + return self._with_csrf(self.handle_edit_submit(tid, body), csrf_token) if "/reply" in rest: tid = rest.split("/reply")[0] return self._with_csrf(self.handle_reply(tid, body), csrf_token) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 32133b8..024aa2d 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -39,6 +39,8 @@ class ForumSync: self._running = False def _rns_handler(self, path, data, request_id, link_id, remote_identity, requested_at): + if remote_identity: + data["peer_hash"] = remote_identity.hash.hex() return self.handlers_ref().handle_sync(data) def _sync_loop(self): @@ -108,11 +110,20 @@ class ForumSync: posts = [dict(r) for r in ps] upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv] + my_blocks = [h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()] + my_peer_blocks = self.fdb.get_peer_block_list() + + retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} + for cid, ct, ai, ra in self.fdb.get_raw_retractions()] + request_data = { "query": {"since": [since]} if since else {}, "threads": threads, "posts": posts, "upvotes": upvotes, + "from_hash": self.identity.hash.hex() if self.identity else "local", + "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, + "retractions": retracted, } receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) @@ -129,12 +140,28 @@ class ForumSync: data = json.loads(resp["body"]) except (json.JSONDecodeError, KeyError): data = {} + my_blocks = set(h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()) for t in data.get("threads", []): - self.fdb.merge_thread(t) + if t.get("author_instance", "") not in my_blocks: + self.fdb.merge_thread(t) for p in data.get("posts", []): - self.fdb.merge_post(p) + if p.get("author_instance", "") not in my_blocks: + self.fdb.merge_post(p) for tid in data.get("upvote_threads", []): self.fdb.merge_upvote(tid, instance_hash) + # Gossip blocks from peer + peer_blocks = data.get("blocks", {}) + for h in peer_blocks.get("mine", []): + if h and h not in my_blocks and instance_hash: + self.fdb.record_peer_block(instance_hash, h) + for h in peer_blocks.get("peers", []): + if h and h not in my_blocks and instance_hash: + self.fdb.record_peer_block(instance_hash, h) + self._apply_peer_blocks() + # Merge incoming retractions + for r in data.get("retractions", []): + if r.get("id") and r.get("type") and r.get("author") and r.get("at"): + self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) now = time.strftime("%Y-%m-%dT%H:%M:%S") self.fdb.set_last_sync(instance_hash, now) else: @@ -142,5 +169,19 @@ class ForumSync: finally: link.teardown() + def _apply_peer_blocks(self): + counts = self.fdb.get_peer_block_counts() + blocked = set(h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()) + auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) + changed = False + for h, count in counts.items(): + if h not in blocked and h not in auto_blocked and count >= 3: + blocked.add(h) + auto_blocked.add(h) + changed = True + if changed: + self.fdb.set_setting("blocked_instances", ",".join(blocked)) + self.fdb.set_setting("auto_blocked_instances", ",".join(auto_blocked)) + def handle_sync(self, data): return self.handlers_ref().handle_sync_request(data) From 74dd108011b7f01fd6fb00c14ae237281216bb84 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 00:34:16 +0000 Subject: [PATCH 05/56] add sync tests: thread, reply, upvote, block, retraction, bidirectional, gossip, update --- tests/test_sync.py | 346 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100644 tests/test_sync.py diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..e1368b1 --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,346 @@ +import json +import os +import tempfile +import time +import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from tinyweb_forum.db import ForumDB +from tinyweb_forum.handlers import ForumHandlers + + +def make_handler(data_dir, site_name="test"): + fdb = ForumDB(data_dir) + handlers = ForumHandlers(fdb, None, None, None, site_name=site_name) + return fdb, handlers + + +def test_sync_thread(): + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Hello World", "https://example.com", + "First post", "test,hello", + "abc123", "", now) + + # Simulate sync A -> B + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], + "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + t = db_b.get_thread(thread_id) + assert t is not None, "thread should exist in B" + assert t["title"] == "Hello World" + assert t["url"] == "https://example.com" + assert t["tags"] == "test,hello" + assert t["author_instance"] == "abc123" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_reply(): + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Thread", "", "Body", "", + "abc123", "", now) + post_id = "p1" + db_a.create_post(post_id, thread_id, "", "A reply", "def456", "", now) + + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [dict(db_a.get_posts(thread_id)[0])], + "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + posts = db_b.get_posts(thread_id) + assert len(list(posts)) == 1 + assert list(posts)[0]["body"] == "A reply" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_upvote(): + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Thread", "", "", "", + "abc123", "", now) + + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], + "upvotes": [{"thread_id": thread_id, "instance_hash": "xyz789"}], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + t = db_b.get_thread(thread_id) + assert t is not None + assert t["score"] == 1 + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_blocks_content(): + """Threads from blocked authors should not sync.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + # Block the author + db_b.set_setting("blocked_instances", "abc123") + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Blocked Thread", "", "", "", + "abc123", "", now) + + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], + "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + t = db_b.get_thread(thread_id) + assert t is None, "blocked author's thread should not sync" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_retraction(): + """Retraction signal from author should retract content on peer.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "To Retract", "", "", "", + "abc123", "", now) + + # Sync thread to B first + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + assert db_b.get_thread(thread_id) is not None + + # Now retract in A and sync + later = "2026-06-05T13:00:00" + db_a.retract_thread(thread_id, "abc123", later) + hb.handle_sync_request({ + "query": {}, + "threads": [], "posts": [], "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [{"id": thread_id, "type": "thread", + "author": "abc123", "at": later}], + }) + + t = db_b.get_thread(thread_id) + assert t is not None + assert t["title"] == "[retracted]" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_bidirectional(): + """A and B sync each other's threads.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + + # A creates a thread + db_a.create_thread("t_a", "From A", "", "", "", + "aaa111", "", now) + + # B creates a thread + db_b.create_thread("t_b", "From B", "", "", "", + "bbb222", "", now) + + # A syncs B's content + ha.handle_sync_request({ + "query": {}, + "threads": [dict(db_b.get_thread("t_b"))], + "posts": [], "upvotes": [], + "from_hash": "bbb222", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + # B syncs A's content + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread("t_a"))], + "posts": [], "upvotes": [], + "from_hash": "aaa111", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + assert db_a.get_thread("t_b") is not None, "A should have B's thread" + assert db_b.get_thread("t_a") is not None, "B should have A's thread" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_block_gossip(): + """Peer blocks should be recorded and auto-block at threshold.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + + # A has blocked "malicious" + db_a.set_setting("blocked_instances", "malicious") + + # A syncs to B — B should record A's blocks + hb.handle_sync_request({ + "query": {}, + "threads": [], "posts": [], "upvotes": [], + "from_hash": "peer_a", + "blocks": {"mine": ["malicious"], "peers": []}, + "retractions": [], + }) + + blocks_a = db_b.get_peer_block_list() + assert "malicious" in blocks_a + + # Need 2 more peers to reach threshold 3 + hb.handle_sync_request({ + "query": {}, + "threads": [], "posts": [], "upvotes": [], + "from_hash": "peer_b", + "blocks": {"mine": ["malicious"], "peers": []}, + "retractions": [], + }) + hb.handle_sync_request({ + "query": {}, + "threads": [], "posts": [], "upvotes": [], + "from_hash": "peer_c", + "blocks": {"mine": ["malicious"], "peers": []}, + "retractions": [], + }) + + # _apply_peer_blocks would be called during sync loop + from tinyweb_forum.sync import ForumSync + # Mock just the _apply_peer_blocks logic + counts = db_b.get_peer_block_counts() + assert counts.get("malicious", 0) >= 3 + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_updated_content(): + """Updated thread should overwrite older version.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Old Title", "", "", "", + "abc123", "", now) + + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + assert db_b.get_thread(thread_id)["title"] == "Old Title" + + # Update in A + later = "2026-06-05T14:00:00" + db_a.update_thread(thread_id, "New Title", "", "", "", later) + + # Resync + hb.handle_sync_request({ + "query": {"since": [now]}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + assert db_b.get_thread(thread_id)["title"] == "New Title" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +if __name__ == "__main__": + test_sync_thread() + test_sync_reply() + test_sync_upvote() + test_sync_blocks_content() + test_sync_retraction() + test_sync_bidirectional() + test_sync_block_gossip() + test_sync_updated_content() + print("all sync tests passed") From b39d70dd6cc494e448827deec2c48fa1fff9ec98 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 00:34:16 +0000 Subject: [PATCH 06/56] add sync tests: thread, reply, upvote, block, retraction, bidirectional, gossip, update --- tests/test_sync.py | 346 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100644 tests/test_sync.py diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 0000000..e1368b1 --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,346 @@ +import json +import os +import tempfile +import time +import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from tinyweb_forum.db import ForumDB +from tinyweb_forum.handlers import ForumHandlers + + +def make_handler(data_dir, site_name="test"): + fdb = ForumDB(data_dir) + handlers = ForumHandlers(fdb, None, None, None, site_name=site_name) + return fdb, handlers + + +def test_sync_thread(): + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Hello World", "https://example.com", + "First post", "test,hello", + "abc123", "", now) + + # Simulate sync A -> B + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], + "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + t = db_b.get_thread(thread_id) + assert t is not None, "thread should exist in B" + assert t["title"] == "Hello World" + assert t["url"] == "https://example.com" + assert t["tags"] == "test,hello" + assert t["author_instance"] == "abc123" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_reply(): + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Thread", "", "Body", "", + "abc123", "", now) + post_id = "p1" + db_a.create_post(post_id, thread_id, "", "A reply", "def456", "", now) + + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [dict(db_a.get_posts(thread_id)[0])], + "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + posts = db_b.get_posts(thread_id) + assert len(list(posts)) == 1 + assert list(posts)[0]["body"] == "A reply" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_upvote(): + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Thread", "", "", "", + "abc123", "", now) + + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], + "upvotes": [{"thread_id": thread_id, "instance_hash": "xyz789"}], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + t = db_b.get_thread(thread_id) + assert t is not None + assert t["score"] == 1 + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_blocks_content(): + """Threads from blocked authors should not sync.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + # Block the author + db_b.set_setting("blocked_instances", "abc123") + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Blocked Thread", "", "", "", + "abc123", "", now) + + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], + "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + t = db_b.get_thread(thread_id) + assert t is None, "blocked author's thread should not sync" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_retraction(): + """Retraction signal from author should retract content on peer.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "To Retract", "", "", "", + "abc123", "", now) + + # Sync thread to B first + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + assert db_b.get_thread(thread_id) is not None + + # Now retract in A and sync + later = "2026-06-05T13:00:00" + db_a.retract_thread(thread_id, "abc123", later) + hb.handle_sync_request({ + "query": {}, + "threads": [], "posts": [], "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [{"id": thread_id, "type": "thread", + "author": "abc123", "at": later}], + }) + + t = db_b.get_thread(thread_id) + assert t is not None + assert t["title"] == "[retracted]" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_bidirectional(): + """A and B sync each other's threads.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + + # A creates a thread + db_a.create_thread("t_a", "From A", "", "", "", + "aaa111", "", now) + + # B creates a thread + db_b.create_thread("t_b", "From B", "", "", "", + "bbb222", "", now) + + # A syncs B's content + ha.handle_sync_request({ + "query": {}, + "threads": [dict(db_b.get_thread("t_b"))], + "posts": [], "upvotes": [], + "from_hash": "bbb222", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + # B syncs A's content + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread("t_a"))], + "posts": [], "upvotes": [], + "from_hash": "aaa111", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + + assert db_a.get_thread("t_b") is not None, "A should have B's thread" + assert db_b.get_thread("t_a") is not None, "B should have A's thread" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_block_gossip(): + """Peer blocks should be recorded and auto-block at threshold.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + + # A has blocked "malicious" + db_a.set_setting("blocked_instances", "malicious") + + # A syncs to B — B should record A's blocks + hb.handle_sync_request({ + "query": {}, + "threads": [], "posts": [], "upvotes": [], + "from_hash": "peer_a", + "blocks": {"mine": ["malicious"], "peers": []}, + "retractions": [], + }) + + blocks_a = db_b.get_peer_block_list() + assert "malicious" in blocks_a + + # Need 2 more peers to reach threshold 3 + hb.handle_sync_request({ + "query": {}, + "threads": [], "posts": [], "upvotes": [], + "from_hash": "peer_b", + "blocks": {"mine": ["malicious"], "peers": []}, + "retractions": [], + }) + hb.handle_sync_request({ + "query": {}, + "threads": [], "posts": [], "upvotes": [], + "from_hash": "peer_c", + "blocks": {"mine": ["malicious"], "peers": []}, + "retractions": [], + }) + + # _apply_peer_blocks would be called during sync loop + from tinyweb_forum.sync import ForumSync + # Mock just the _apply_peer_blocks logic + counts = db_b.get_peer_block_counts() + assert counts.get("malicious", 0) >= 3 + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +def test_sync_updated_content(): + """Updated thread should overwrite older version.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + + now = "2026-06-05T12:00:00" + thread_id = "t1" + db_a.create_thread(thread_id, "Old Title", "", "", "", + "abc123", "", now) + + hb.handle_sync_request({ + "query": {}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + assert db_b.get_thread(thread_id)["title"] == "Old Title" + + # Update in A + later = "2026-06-05T14:00:00" + db_a.update_thread(thread_id, "New Title", "", "", "", later) + + # Resync + hb.handle_sync_request({ + "query": {"since": [now]}, + "threads": [dict(db_a.get_thread(thread_id))], + "posts": [], "upvotes": [], + "from_hash": "abc123", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + }) + assert db_b.get_thread(thread_id)["title"] == "New Title" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + + +if __name__ == "__main__": + test_sync_thread() + test_sync_reply() + test_sync_upvote() + test_sync_blocks_content() + test_sync_retraction() + test_sync_bidirectional() + test_sync_block_gossip() + test_sync_updated_content() + print("all sync tests passed") From 0de6be8f3dc3808bcf12be4fe1ce44361a43eee9 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 00:43:18 +0000 Subject: [PATCH 07/56] auto peer discovery: gossip known peers during sync, discover from_hash automatically --- tests/test_sync.py | 45 +++++++++++++++++++++++++++++++++++++++ tinyweb_forum/db.py | 22 +++++++++++++++++++ tinyweb_forum/handlers.py | 11 ++++++++++ tinyweb_forum/sync.py | 10 ++++++++- 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index e1368b1..8e535cb 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -334,6 +334,50 @@ def test_sync_updated_content(): shutil.rmtree(dir_b) +def test_sync_peer_discovery(): + """Peers should discover each other through sync gossip.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + dir_c = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + db_c, hc = make_handler(dir_c) + + now = "2026-06-05T12:00:00" + db_a.create_thread("t1", "From A", "", "", "", "aaa", "", now) + db_b.create_thread("t2", "From B", "", "", "", "bbb", "", now) + db_c.create_thread("t3", "From C", "", "", "", "ccc", "", now) + + # Seed: A knows B, B knows C + db_a.add_known_peer("bbb") + db_b.add_known_peer("ccc") + # C doesn't know anyone yet + + # B syncs with C — B sends its known peers (ccc's hash not in B's known peers since B is talking to C) + # Actually, B knows C (bbb -> ccc), so when B sends sync request to C, + # B includes known_peers. C learns about B. + hc.handle_sync_request({ + "query": {}, + "threads": [dict(db_b.get_thread("t2"))], + "posts": [], "upvotes": [], + "from_hash": "bbb", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + "known_peers": ["aaa"], # B tells C about A + }) + + # C should now know about B (from auto-discovery of from_hash) and A (from known_peers) + known = [r["instance_hash"] for r in db_c.get_synced_instances()] + assert "bbb" in known, "C should auto-discover B from from_hash" + assert "aaa" in known, "C should discover A from known_peers gossip" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + shutil.rmtree(dir_c) + + if __name__ == "__main__": test_sync_thread() test_sync_reply() @@ -343,4 +387,5 @@ if __name__ == "__main__": test_sync_bidirectional() test_sync_block_gossip() test_sync_updated_content() + test_sync_peer_discovery() print("all sync tests passed") diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index 087d8e3..b81c8e7 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -246,6 +246,28 @@ class ForumDB: finally: self.return_db(db) + def add_known_peer(self, instance_hash): + """Add a discovered peer to the sync list (auto-discovery).""" + db = self.get_db() + try: + db.execute( + "INSERT OR IGNORE INTO synced_instances (instance_hash) VALUES (?)", + (instance_hash,), + ) + db.commit() + finally: + self.return_db(db) + + def get_all_known_hashes(self): + """Get all known instance hashes for peer discovery gossip.""" + db = self.get_db() + try: + return [r["instance_hash"] for r in db.execute( + "SELECT instance_hash FROM synced_instances" + ).fetchall()] + finally: + self.return_db(db) + def upsert_synced_instance(self, instance_hash, name=""): db = self.get_db() try: diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 874ced7..8e45f9a 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -604,6 +604,14 @@ class ForumHandlers: if r.get("id") and r.get("type") and r.get("author") and r.get("at"): self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) + # Auto-discover the peer that synced with us and their known peers + from_hash = data.get("from_hash", "") + if from_hash and from_hash not in blocked: + self.fdb.add_known_peer(from_hash) + for peer_hash in data.get("known_peers", []): + if peer_hash and peer_hash != from_hash and peer_hash not in blocked: + self.fdb.add_known_peer(peer_hash) + my_blocks = list(blocked) my_peer_blocks = self.fdb.get_peer_block_list() threads, posts, upvote_threads = [], [], [] @@ -616,6 +624,8 @@ class ForumHandlers: retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} for cid, ct, ai, ra in self.fdb.get_raw_retractions()] + known_peers = [h for h in self.fdb.get_all_known_hashes() if h != from_hash] + return { "status": 200, "content_type": "application/json", @@ -625,6 +635,7 @@ class ForumHandlers: "upvote_threads": upvote_threads, "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, "retractions": retracted, + "known_peers": known_peers, }), "headers": {}, } diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 024aa2d..05e6f5c 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -116,14 +116,18 @@ class ForumSync: retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} for cid, ct, ai, ra in self.fdb.get_raw_retractions()] + my_hash = self.identity.hash.hex() if self.identity else "local" + known_peers = [h for h in self.fdb.get_all_known_hashes() if h != instance_hash and h != my_hash] + request_data = { "query": {"since": [since]} if since else {}, "threads": threads, "posts": posts, "upvotes": upvotes, - "from_hash": self.identity.hash.hex() if self.identity else "local", + "from_hash": my_hash, "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, "retractions": retracted, + "known_peers": known_peers, } receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) @@ -162,6 +166,10 @@ class ForumSync: for r in data.get("retractions", []): if r.get("id") and r.get("type") and r.get("author") and r.get("at"): self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) + # Discover new peers from gossip + for peer_hash in data.get("known_peers", []): + if peer_hash and peer_hash != my_hash and peer_hash != instance_hash: + self.fdb.add_known_peer(peer_hash) now = time.strftime("%Y-%m-%dT%H:%M:%S") self.fdb.set_last_sync(instance_hash, now) else: From 55a6d22482ca76f1d4149a9c4833213b0dd7162a Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 00:43:18 +0000 Subject: [PATCH 08/56] auto peer discovery: gossip known peers during sync, discover from_hash automatically --- tests/test_sync.py | 45 +++++++++++++++++++++++++++++++++++++++ tinyweb_forum/db.py | 22 +++++++++++++++++++ tinyweb_forum/handlers.py | 11 ++++++++++ tinyweb_forum/sync.py | 10 ++++++++- 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index e1368b1..8e535cb 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -334,6 +334,50 @@ def test_sync_updated_content(): shutil.rmtree(dir_b) +def test_sync_peer_discovery(): + """Peers should discover each other through sync gossip.""" + dir_a = tempfile.mkdtemp() + dir_b = tempfile.mkdtemp() + dir_c = tempfile.mkdtemp() + try: + db_a, ha = make_handler(dir_a) + db_b, hb = make_handler(dir_b) + db_c, hc = make_handler(dir_c) + + now = "2026-06-05T12:00:00" + db_a.create_thread("t1", "From A", "", "", "", "aaa", "", now) + db_b.create_thread("t2", "From B", "", "", "", "bbb", "", now) + db_c.create_thread("t3", "From C", "", "", "", "ccc", "", now) + + # Seed: A knows B, B knows C + db_a.add_known_peer("bbb") + db_b.add_known_peer("ccc") + # C doesn't know anyone yet + + # B syncs with C — B sends its known peers (ccc's hash not in B's known peers since B is talking to C) + # Actually, B knows C (bbb -> ccc), so when B sends sync request to C, + # B includes known_peers. C learns about B. + hc.handle_sync_request({ + "query": {}, + "threads": [dict(db_b.get_thread("t2"))], + "posts": [], "upvotes": [], + "from_hash": "bbb", + "blocks": {"mine": [], "peers": []}, + "retractions": [], + "known_peers": ["aaa"], # B tells C about A + }) + + # C should now know about B (from auto-discovery of from_hash) and A (from known_peers) + known = [r["instance_hash"] for r in db_c.get_synced_instances()] + assert "bbb" in known, "C should auto-discover B from from_hash" + assert "aaa" in known, "C should discover A from known_peers gossip" + finally: + import shutil + shutil.rmtree(dir_a) + shutil.rmtree(dir_b) + shutil.rmtree(dir_c) + + if __name__ == "__main__": test_sync_thread() test_sync_reply() @@ -343,4 +387,5 @@ if __name__ == "__main__": test_sync_bidirectional() test_sync_block_gossip() test_sync_updated_content() + test_sync_peer_discovery() print("all sync tests passed") diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index 087d8e3..b81c8e7 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -246,6 +246,28 @@ class ForumDB: finally: self.return_db(db) + def add_known_peer(self, instance_hash): + """Add a discovered peer to the sync list (auto-discovery).""" + db = self.get_db() + try: + db.execute( + "INSERT OR IGNORE INTO synced_instances (instance_hash) VALUES (?)", + (instance_hash,), + ) + db.commit() + finally: + self.return_db(db) + + def get_all_known_hashes(self): + """Get all known instance hashes for peer discovery gossip.""" + db = self.get_db() + try: + return [r["instance_hash"] for r in db.execute( + "SELECT instance_hash FROM synced_instances" + ).fetchall()] + finally: + self.return_db(db) + def upsert_synced_instance(self, instance_hash, name=""): db = self.get_db() try: diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 874ced7..8e45f9a 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -604,6 +604,14 @@ class ForumHandlers: if r.get("id") and r.get("type") and r.get("author") and r.get("at"): self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) + # Auto-discover the peer that synced with us and their known peers + from_hash = data.get("from_hash", "") + if from_hash and from_hash not in blocked: + self.fdb.add_known_peer(from_hash) + for peer_hash in data.get("known_peers", []): + if peer_hash and peer_hash != from_hash and peer_hash not in blocked: + self.fdb.add_known_peer(peer_hash) + my_blocks = list(blocked) my_peer_blocks = self.fdb.get_peer_block_list() threads, posts, upvote_threads = [], [], [] @@ -616,6 +624,8 @@ class ForumHandlers: retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} for cid, ct, ai, ra in self.fdb.get_raw_retractions()] + known_peers = [h for h in self.fdb.get_all_known_hashes() if h != from_hash] + return { "status": 200, "content_type": "application/json", @@ -625,6 +635,7 @@ class ForumHandlers: "upvote_threads": upvote_threads, "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, "retractions": retracted, + "known_peers": known_peers, }), "headers": {}, } diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 024aa2d..05e6f5c 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -116,14 +116,18 @@ class ForumSync: retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} for cid, ct, ai, ra in self.fdb.get_raw_retractions()] + my_hash = self.identity.hash.hex() if self.identity else "local" + known_peers = [h for h in self.fdb.get_all_known_hashes() if h != instance_hash and h != my_hash] + request_data = { "query": {"since": [since]} if since else {}, "threads": threads, "posts": posts, "upvotes": upvotes, - "from_hash": self.identity.hash.hex() if self.identity else "local", + "from_hash": my_hash, "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, "retractions": retracted, + "known_peers": known_peers, } receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) @@ -162,6 +166,10 @@ class ForumSync: for r in data.get("retractions", []): if r.get("id") and r.get("type") and r.get("author") and r.get("at"): self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) + # Discover new peers from gossip + for peer_hash in data.get("known_peers", []): + if peer_hash and peer_hash != my_hash and peer_hash != instance_hash: + self.fdb.add_known_peer(peer_hash) now = time.strftime("%Y-%m-%dT%H:%M:%S") self.fdb.set_last_sync(instance_hash, now) else: From 6a1f47818df536c26d60738813c1eff2925f2c00 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 00:55:03 +0000 Subject: [PATCH 09/56] truly public forum: auto-discover other instances via RNS announce handler --- tests/test_sync.py | 52 +++++++++++++++++++++++++++++++++++++++++++ tinyweb_forum/sync.py | 29 +++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 8e535cb..6ad6833 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -378,6 +378,57 @@ def test_sync_peer_discovery(): shutil.rmtree(dir_c) +def test_announce_handler(): + """Announce handler should auto-discover peers.""" + dir_a = tempfile.mkdtemp() + try: + fdb = ForumDB(dir_a) + from tinyweb_forum.sync import _ForumAnnounceHandler + + # Mock identity — RNS identity.hash returns bytes + class MockIdentity: + def __init__(self, h): + self._hash_hex = h + @property + def hash(self): + return bytes.fromhex(self._hash_hex) + + local_hex = "aa" * 16 + peer_hex = "bb" * 16 + + handler = _ForumAnnounceHandler(fdb, MockIdentity(local_hex)) + + class MockAnnouncedIdentity: + def __init__(self, h): + self._hash_hex = h + @property + def hash(self): + return bytes.fromhex(self._hash_hex) + + # Announce from a peer + handler.received_announce( + destination_hash=bytes.fromhex(peer_hex), + announced_identity=MockAnnouncedIdentity(peer_hex), + app_data=b"tinyweb-forum", + ) + + known = [r["instance_hash"] for r in fdb.get_synced_instances()] + assert peer_hex in known, "peer should be added to sync list" + + # Announce from ourselves should be ignored + handler.received_announce( + destination_hash=bytes.fromhex(local_hex), + announced_identity=MockAnnouncedIdentity(local_hex), + app_data=b"tinyweb-forum", + ) + + known = [r["instance_hash"] for r in fdb.get_synced_instances()] + assert known.count(local_hex) == 0, "own announce should be ignored" + finally: + import shutil + shutil.rmtree(dir_a) + + if __name__ == "__main__": test_sync_thread() test_sync_reply() @@ -388,4 +439,5 @@ if __name__ == "__main__": test_sync_block_gossip() test_sync_updated_content() test_sync_peer_discovery() + test_announce_handler() print("all sync tests passed") diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 05e6f5c..098dfc0 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -8,6 +8,24 @@ SYNC_INTERVAL = 300 # 5 minutes REQUEST_TIMEOUT = 60 +class _ForumAnnounceHandler: + """Receives announces from other forum instances and auto-discovers them.""" + + aspect_filter = FORUM_APP + receive_path_responses = False + + def __init__(self, fdb, identity): + self.fdb = fdb + self.my_hash = identity.hash.hex() if identity else "local" + + def received_announce(self, destination_hash, announced_identity, app_data): + if announced_identity is None: + return + peer_hash = announced_identity.hash.hex() + if peer_hash and peer_hash != self.my_hash: + self.fdb.add_known_peer(peer_hash) + + class ForumSync: def __init__(self, fdb, identity, reticulum, handlers_ref): self.fdb = fdb @@ -15,6 +33,7 @@ class ForumSync: self.reticulum = reticulum self.handlers_ref = handlers_ref self.destination = None + self._announce_handler = None self._running = False self._thread = None @@ -30,13 +49,21 @@ class ForumSync: response_generator=self._rns_handler, allow=RNS.Destination.ALLOW_ALL, ) - self.destination.announce() + self.destination.announce(app_data=FORUM_APP.encode("utf-8")) + # Auto-discover other forum instances via announces + self._announce_handler = _ForumAnnounceHandler(self.fdb, self.identity) + RNS.Transport.register_announce_handler(self._announce_handler) self._running = True self._thread = threading.Thread(target=self._sync_loop, daemon=True) self._thread.start() def stop(self): self._running = False + if self._announce_handler: + try: + RNS.Transport.deregister_announce_handler(self._announce_handler) + except Exception: + pass def _rns_handler(self, path, data, request_id, link_id, remote_identity, requested_at): if remote_identity: From 5890897896c4d405cc5877b1bca6c5a0706aec32 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 00:55:03 +0000 Subject: [PATCH 10/56] truly public forum: auto-discover other instances via RNS announce handler --- tests/test_sync.py | 52 +++++++++++++++++++++++++++++++++++++++++++ tinyweb_forum/sync.py | 29 +++++++++++++++++++++++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 8e535cb..6ad6833 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -378,6 +378,57 @@ def test_sync_peer_discovery(): shutil.rmtree(dir_c) +def test_announce_handler(): + """Announce handler should auto-discover peers.""" + dir_a = tempfile.mkdtemp() + try: + fdb = ForumDB(dir_a) + from tinyweb_forum.sync import _ForumAnnounceHandler + + # Mock identity — RNS identity.hash returns bytes + class MockIdentity: + def __init__(self, h): + self._hash_hex = h + @property + def hash(self): + return bytes.fromhex(self._hash_hex) + + local_hex = "aa" * 16 + peer_hex = "bb" * 16 + + handler = _ForumAnnounceHandler(fdb, MockIdentity(local_hex)) + + class MockAnnouncedIdentity: + def __init__(self, h): + self._hash_hex = h + @property + def hash(self): + return bytes.fromhex(self._hash_hex) + + # Announce from a peer + handler.received_announce( + destination_hash=bytes.fromhex(peer_hex), + announced_identity=MockAnnouncedIdentity(peer_hex), + app_data=b"tinyweb-forum", + ) + + known = [r["instance_hash"] for r in fdb.get_synced_instances()] + assert peer_hex in known, "peer should be added to sync list" + + # Announce from ourselves should be ignored + handler.received_announce( + destination_hash=bytes.fromhex(local_hex), + announced_identity=MockAnnouncedIdentity(local_hex), + app_data=b"tinyweb-forum", + ) + + known = [r["instance_hash"] for r in fdb.get_synced_instances()] + assert known.count(local_hex) == 0, "own announce should be ignored" + finally: + import shutil + shutil.rmtree(dir_a) + + if __name__ == "__main__": test_sync_thread() test_sync_reply() @@ -388,4 +439,5 @@ if __name__ == "__main__": test_sync_block_gossip() test_sync_updated_content() test_sync_peer_discovery() + test_announce_handler() print("all sync tests passed") diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 05e6f5c..098dfc0 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -8,6 +8,24 @@ SYNC_INTERVAL = 300 # 5 minutes REQUEST_TIMEOUT = 60 +class _ForumAnnounceHandler: + """Receives announces from other forum instances and auto-discovers them.""" + + aspect_filter = FORUM_APP + receive_path_responses = False + + def __init__(self, fdb, identity): + self.fdb = fdb + self.my_hash = identity.hash.hex() if identity else "local" + + def received_announce(self, destination_hash, announced_identity, app_data): + if announced_identity is None: + return + peer_hash = announced_identity.hash.hex() + if peer_hash and peer_hash != self.my_hash: + self.fdb.add_known_peer(peer_hash) + + class ForumSync: def __init__(self, fdb, identity, reticulum, handlers_ref): self.fdb = fdb @@ -15,6 +33,7 @@ class ForumSync: self.reticulum = reticulum self.handlers_ref = handlers_ref self.destination = None + self._announce_handler = None self._running = False self._thread = None @@ -30,13 +49,21 @@ class ForumSync: response_generator=self._rns_handler, allow=RNS.Destination.ALLOW_ALL, ) - self.destination.announce() + self.destination.announce(app_data=FORUM_APP.encode("utf-8")) + # Auto-discover other forum instances via announces + self._announce_handler = _ForumAnnounceHandler(self.fdb, self.identity) + RNS.Transport.register_announce_handler(self._announce_handler) self._running = True self._thread = threading.Thread(target=self._sync_loop, daemon=True) self._thread.start() def stop(self): self._running = False + if self._announce_handler: + try: + RNS.Transport.deregister_announce_handler(self._announce_handler) + except Exception: + pass def _rns_handler(self, path, data, request_id, link_id, remote_identity, requested_at): if remote_identity: From 68a9b569d8a7307deaceb571edc3dfe1201ac8dd Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 00:55:23 +0000 Subject: [PATCH 11/56] update moderation page: note auto-discovery, bootstrap fallback --- tinyweb_forum/handlers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 8e45f9a..72c083f 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -487,6 +487,7 @@ class ForumHandlers: return self._respond( f"

    forum moderation

    " f"

    {msg}

    " + f"

    Forum instances on the mesh are discovered and synced automatically.

    " f"

    blocked instances

    " f"{blocked_items}" f'
    ' @@ -504,6 +505,8 @@ class ForumHandlers: f"
    " f"

    synced instances

    " f"{synced_items}" + f"

    Instances are discovered automatically via mesh announces. " + f"You can also manually add a friend's instance hash to bootstrap.

    " f'
    ' f'{self._csrf_field()}' f'

    ' From 01ae0e6ef5f9f2e3fa229bae743df01243b2af17 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 00:55:23 +0000 Subject: [PATCH 12/56] update moderation page: note auto-discovery, bootstrap fallback --- tinyweb_forum/handlers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 8e45f9a..72c083f 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -487,6 +487,7 @@ class ForumHandlers: return self._respond( f"

    forum moderation

    " f"

    {msg}

    " + f"

    Forum instances on the mesh are discovered and synced automatically.

    " f"

    blocked instances

    " f"{blocked_items}" f'' @@ -504,6 +505,8 @@ class ForumHandlers: f"" f"

    synced instances

    " f"{synced_items}" + f"

    Instances are discovered automatically via mesh announces. " + f"You can also manually add a friend's instance hash to bootstrap.

    " f'
    ' f'{self._csrf_field()}' f'

    ' From ed541440b43d04566b34fc48f12827d51cbc7d91 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 01:00:47 +0000 Subject: [PATCH 13/56] auto-discovery toggle, auto-prune with customizable retention --- tests/test_sync.py | 23 ++++++++++++++++++++++ tinyweb_forum/db.py | 26 +++++++++++++++++++++++++ tinyweb_forum/handlers.py | 40 ++++++++++++++++++++++++++++++++++++++- tinyweb_forum/sync.py | 30 ++++++++++++++++++++++++++--- 4 files changed, 115 insertions(+), 4 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 6ad6833..65cb597 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -429,6 +429,28 @@ def test_announce_handler(): shutil.rmtree(dir_a) +def test_prune_old_content(): + """Old threads should be pruned after retention period.""" + dir_a = tempfile.mkdtemp() + try: + fdb = ForumDB(dir_a) + now = "2026-06-05T12:00:00" + old = "2026-01-01T12:00:00" + fdb.create_thread("new_t", "New", "", "", "", "aaa", "", now) + fdb.create_thread("old_t", "Old", "", "", "", "bbb", "", old) + fdb.create_post("old_p", "old_t", "", "old reply", "bbb", "", old) + + # Prune with 30 day retention — old thread is ~5 months old + fdb.prune_old_content(30) + + assert fdb.get_thread("new_t") is not None, "new thread should survive" + assert fdb.get_thread("old_t") is None, "old thread should be pruned" + assert len(list(fdb.get_posts("old_t"))) == 0, "old posts should be pruned" + finally: + import shutil + shutil.rmtree(dir_a) + + if __name__ == "__main__": test_sync_thread() test_sync_reply() @@ -440,4 +462,5 @@ if __name__ == "__main__": test_sync_updated_content() test_sync_peer_discovery() test_announce_handler() + test_prune_old_content() print("all sync tests passed") diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index b81c8e7..6ab12de 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -1,6 +1,7 @@ import sqlite3 import os import threading +from datetime import datetime, timedelta FORUM_DB = "forum.db" @@ -497,3 +498,28 @@ class ForumDB: ).fetchall() finally: self.return_db(db) + + def prune_old_content(self, retention_days): + """Delete threads and posts older than retention_days.""" + db = self.get_db() + try: + cutoff = (datetime.utcnow() - timedelta(days=retention_days)).strftime("%Y-%m-%dT%H:%M:%S") + # Delete posts in old threads + db.execute( + "DELETE FROM posts WHERE thread_id IN " + "(SELECT id FROM threads WHERE updated_at < ?)", + (cutoff,), + ) + # Delete orphaned posts (thread already deleted) + db.execute( + "DELETE FROM posts WHERE thread_id NOT IN (SELECT id FROM threads)" + ) + # Delete old threads + db.execute("DELETE FROM threads WHERE updated_at < ?", (cutoff,)) + # Clean up orphaned upvotes + db.execute( + "DELETE FROM upvotes WHERE thread_id NOT IN (SELECT id FROM threads)" + ) + db.commit() + finally: + self.return_db(db) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 72c083f..36fdbef 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -484,10 +484,28 @@ class ForumHandlers: ) synced_items = f"
      {synced_items}
    " if synced_items else "

    No instances synced yet.

    " + auto_discover = self.fdb.get_setting("forum_auto_discover", "1") + auto_discover_checked = " checked" if auto_discover == "1" else "" + retention_days = self.fdb.get_setting("forum_retention_days", "30") + return self._respond( f"

    forum moderation

    " f"

    {msg}

    " - f"

    Forum instances on the mesh are discovered and synced automatically.

    " + f"

    auto-discovery

    " + f'' + f'{self._csrf_field()}' + f'

    " + f'' + f"" + f"

    storage

    " + f'
    ' + f'{self._csrf_field()}' + f'' + f"
    Older threads are pruned automatically (default: 30). Set to 0 to keep everything.

    " + f'' + f"
    " f"

    blocked instances

    " f"{blocked_items}" f'
    ' @@ -556,6 +574,22 @@ class ForumHandlers: self.fdb.set_setting("keyword_filters", keywords) return self.handle_moderation("Filters saved.") + def handle_auto_discover(self, body): + enabled = "1" if body.get("enabled") else "0" + self.fdb.set_setting("forum_auto_discover", enabled) + if self.sync: + self.sync.set_auto_discover(enabled == "1") + return self.handle_moderation(f"Auto-discovery {'enabled' if enabled == '1' else 'disabled'}.") + + def handle_storage(self, body): + days = body.get("retention_days", ["30"])[0].strip() + try: + days = max(0, int(days)) + except ValueError: + return self.handle_moderation("Invalid retention days.") + self.fdb.set_setting("forum_retention_days", str(days)) + return self.handle_moderation(f"Storage retention set to {days} days.") + def handle_sync_add(self, body): instance = body.get("instance", [""])[0].strip().replace("<", "").replace(">", "") name = body.get("name", [""])[0].strip() @@ -726,6 +760,10 @@ class ForumHandlers: return self._with_csrf(self.handle_sync_add(body), csrf_token) elif sub == "/unsync": return self._with_csrf(self.handle_unsync(body), csrf_token) + elif sub == "/auto_discover": + return self._with_csrf(self.handle_auto_discover(body), csrf_token) + elif sub == "/storage": + return self._with_csrf(self.handle_storage(body), csrf_token) return self._with_csrf(self._error(404), csrf_token) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 098dfc0..440479d 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -50,20 +50,35 @@ class ForumSync: allow=RNS.Destination.ALLOW_ALL, ) self.destination.announce(app_data=FORUM_APP.encode("utf-8")) - # Auto-discover other forum instances via announces - self._announce_handler = _ForumAnnounceHandler(self.fdb, self.identity) - RNS.Transport.register_announce_handler(self._announce_handler) + if self.fdb.get_setting("forum_auto_discover", "1") == "1": + self._enable_announce_handler() self._running = True self._thread = threading.Thread(target=self._sync_loop, daemon=True) self._thread.start() def stop(self): self._running = False + self._disable_announce_handler() + + def set_auto_discover(self, enabled): + self.fdb.set_setting("forum_auto_discover", "1" if enabled else "0") + if enabled: + self._enable_announce_handler() + else: + self._disable_announce_handler() + + def _enable_announce_handler(self): + if self._announce_handler is None: + self._announce_handler = _ForumAnnounceHandler(self.fdb, self.identity) + RNS.Transport.register_announce_handler(self._announce_handler) + + def _disable_announce_handler(self): if self._announce_handler: try: RNS.Transport.deregister_announce_handler(self._announce_handler) except Exception: pass + self._announce_handler = None def _rns_handler(self, path, data, request_id, link_id, remote_identity, requested_at): if remote_identity: @@ -71,6 +86,7 @@ class ForumSync: return self.handlers_ref().handle_sync(data) def _sync_loop(self): + prune_counter = 0 while self._running: try: instances = self.fdb.get_synced_instances() @@ -83,6 +99,14 @@ class ForumSync: print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") except Exception: pass + prune_counter += 1 + if prune_counter >= 6: + prune_counter = 0 + try: + days = int(self.fdb.get_setting("forum_retention_days", "30")) + self.fdb.prune_old_content(days) + except Exception: + pass for _ in range(SYNC_INTERVAL): if not self._running: return From bc6e093a922b34df7f56e3ac3896e3aa8caf9d03 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 01:00:47 +0000 Subject: [PATCH 14/56] auto-discovery toggle, auto-prune with customizable retention --- tests/test_sync.py | 23 ++++++++++++++++++++++ tinyweb_forum/db.py | 26 +++++++++++++++++++++++++ tinyweb_forum/handlers.py | 40 ++++++++++++++++++++++++++++++++++++++- tinyweb_forum/sync.py | 30 ++++++++++++++++++++++++++--- 4 files changed, 115 insertions(+), 4 deletions(-) diff --git a/tests/test_sync.py b/tests/test_sync.py index 6ad6833..65cb597 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -429,6 +429,28 @@ def test_announce_handler(): shutil.rmtree(dir_a) +def test_prune_old_content(): + """Old threads should be pruned after retention period.""" + dir_a = tempfile.mkdtemp() + try: + fdb = ForumDB(dir_a) + now = "2026-06-05T12:00:00" + old = "2026-01-01T12:00:00" + fdb.create_thread("new_t", "New", "", "", "", "aaa", "", now) + fdb.create_thread("old_t", "Old", "", "", "", "bbb", "", old) + fdb.create_post("old_p", "old_t", "", "old reply", "bbb", "", old) + + # Prune with 30 day retention — old thread is ~5 months old + fdb.prune_old_content(30) + + assert fdb.get_thread("new_t") is not None, "new thread should survive" + assert fdb.get_thread("old_t") is None, "old thread should be pruned" + assert len(list(fdb.get_posts("old_t"))) == 0, "old posts should be pruned" + finally: + import shutil + shutil.rmtree(dir_a) + + if __name__ == "__main__": test_sync_thread() test_sync_reply() @@ -440,4 +462,5 @@ if __name__ == "__main__": test_sync_updated_content() test_sync_peer_discovery() test_announce_handler() + test_prune_old_content() print("all sync tests passed") diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index b81c8e7..6ab12de 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -1,6 +1,7 @@ import sqlite3 import os import threading +from datetime import datetime, timedelta FORUM_DB = "forum.db" @@ -497,3 +498,28 @@ class ForumDB: ).fetchall() finally: self.return_db(db) + + def prune_old_content(self, retention_days): + """Delete threads and posts older than retention_days.""" + db = self.get_db() + try: + cutoff = (datetime.utcnow() - timedelta(days=retention_days)).strftime("%Y-%m-%dT%H:%M:%S") + # Delete posts in old threads + db.execute( + "DELETE FROM posts WHERE thread_id IN " + "(SELECT id FROM threads WHERE updated_at < ?)", + (cutoff,), + ) + # Delete orphaned posts (thread already deleted) + db.execute( + "DELETE FROM posts WHERE thread_id NOT IN (SELECT id FROM threads)" + ) + # Delete old threads + db.execute("DELETE FROM threads WHERE updated_at < ?", (cutoff,)) + # Clean up orphaned upvotes + db.execute( + "DELETE FROM upvotes WHERE thread_id NOT IN (SELECT id FROM threads)" + ) + db.commit() + finally: + self.return_db(db) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 72c083f..36fdbef 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -484,10 +484,28 @@ class ForumHandlers: ) synced_items = f"
      {synced_items}
    " if synced_items else "

    No instances synced yet.

    " + auto_discover = self.fdb.get_setting("forum_auto_discover", "1") + auto_discover_checked = " checked" if auto_discover == "1" else "" + retention_days = self.fdb.get_setting("forum_retention_days", "30") + return self._respond( f"

    forum moderation

    " f"

    {msg}

    " - f"

    Forum instances on the mesh are discovered and synced automatically.

    " + f"

    auto-discovery

    " + f'' + f'{self._csrf_field()}' + f'

    " + f'' + f"" + f"

    storage

    " + f'
    ' + f'{self._csrf_field()}' + f'' + f"
    Older threads are pruned automatically (default: 30). Set to 0 to keep everything.

    " + f'' + f"
    " f"

    blocked instances

    " f"{blocked_items}" f'
    ' @@ -556,6 +574,22 @@ class ForumHandlers: self.fdb.set_setting("keyword_filters", keywords) return self.handle_moderation("Filters saved.") + def handle_auto_discover(self, body): + enabled = "1" if body.get("enabled") else "0" + self.fdb.set_setting("forum_auto_discover", enabled) + if self.sync: + self.sync.set_auto_discover(enabled == "1") + return self.handle_moderation(f"Auto-discovery {'enabled' if enabled == '1' else 'disabled'}.") + + def handle_storage(self, body): + days = body.get("retention_days", ["30"])[0].strip() + try: + days = max(0, int(days)) + except ValueError: + return self.handle_moderation("Invalid retention days.") + self.fdb.set_setting("forum_retention_days", str(days)) + return self.handle_moderation(f"Storage retention set to {days} days.") + def handle_sync_add(self, body): instance = body.get("instance", [""])[0].strip().replace("<", "").replace(">", "") name = body.get("name", [""])[0].strip() @@ -726,6 +760,10 @@ class ForumHandlers: return self._with_csrf(self.handle_sync_add(body), csrf_token) elif sub == "/unsync": return self._with_csrf(self.handle_unsync(body), csrf_token) + elif sub == "/auto_discover": + return self._with_csrf(self.handle_auto_discover(body), csrf_token) + elif sub == "/storage": + return self._with_csrf(self.handle_storage(body), csrf_token) return self._with_csrf(self._error(404), csrf_token) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 098dfc0..440479d 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -50,20 +50,35 @@ class ForumSync: allow=RNS.Destination.ALLOW_ALL, ) self.destination.announce(app_data=FORUM_APP.encode("utf-8")) - # Auto-discover other forum instances via announces - self._announce_handler = _ForumAnnounceHandler(self.fdb, self.identity) - RNS.Transport.register_announce_handler(self._announce_handler) + if self.fdb.get_setting("forum_auto_discover", "1") == "1": + self._enable_announce_handler() self._running = True self._thread = threading.Thread(target=self._sync_loop, daemon=True) self._thread.start() def stop(self): self._running = False + self._disable_announce_handler() + + def set_auto_discover(self, enabled): + self.fdb.set_setting("forum_auto_discover", "1" if enabled else "0") + if enabled: + self._enable_announce_handler() + else: + self._disable_announce_handler() + + def _enable_announce_handler(self): + if self._announce_handler is None: + self._announce_handler = _ForumAnnounceHandler(self.fdb, self.identity) + RNS.Transport.register_announce_handler(self._announce_handler) + + def _disable_announce_handler(self): if self._announce_handler: try: RNS.Transport.deregister_announce_handler(self._announce_handler) except Exception: pass + self._announce_handler = None def _rns_handler(self, path, data, request_id, link_id, remote_identity, requested_at): if remote_identity: @@ -71,6 +86,7 @@ class ForumSync: return self.handlers_ref().handle_sync(data) def _sync_loop(self): + prune_counter = 0 while self._running: try: instances = self.fdb.get_synced_instances() @@ -83,6 +99,14 @@ class ForumSync: print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") except Exception: pass + prune_counter += 1 + if prune_counter >= 6: + prune_counter = 0 + try: + days = int(self.fdb.get_setting("forum_retention_days", "30")) + self.fdb.prune_old_content(days) + except Exception: + pass for _ in range(SYNC_INTERVAL): if not self._running: return From fabac927462ad3edd77c8fb72d375966226908b3 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 01:01:01 +0000 Subject: [PATCH 15/56] README: auto-discovery and storage retention docs --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 43b0941..26ebdd1 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,14 @@ All moderation is local — you control your view: ## Sync -- Forum instances discover each other via Reticulum +- Forum instances auto-discover each other via RNS announces - Content is exchanged as JSON over RNS links every 5 minutes +- Peer discovery propagates through gossip — each instance shares its known peers - Block lists and retractions are gossiped alongside content - Only new/updated content is transferred (timestamp-based) +- Auto-discovery can be disabled in the moderation page + +## Storage + +- Threads are pruned after 30 days by default (configurable in moderation page) +- Set retention to 0 to keep everything indefinitely From d86349292ad5d927c49e7dc87a92759b0c652917 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 01:01:01 +0000 Subject: [PATCH 16/56] README: auto-discovery and storage retention docs --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c9f2b7..3509d4f 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,14 @@ All moderation is local — you control your view: ## Sync -- Forum instances discover each other via Reticulum +- Forum instances auto-discover each other via RNS announces - Content is exchanged as JSON over RNS links every 5 minutes +- Peer discovery propagates through gossip — each instance shares its known peers - Block lists and retractions are gossiped alongside content - Only new/updated content is transferred (timestamp-based) +- Auto-discovery can be disabled in the moderation page + +## Storage + +- Threads are pruned after 30 days by default (configurable in moderation page) +- Set retention to 0 to keep everything indefinitely From f16907d6a27308b6157ebe6292e41b5204c78683 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 01:04:17 +0000 Subject: [PATCH 17/56] max title (200) and body (10k) length limits, fix unescaped body in edit form --- tinyweb_forum/handlers.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 36fdbef..4baf1d4 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -5,7 +5,8 @@ from datetime import datetime from urllib.parse import unquote -PER_PAGE = 20 +MAX_TITLE_LENGTH = 200 +MAX_BODY_LENGTH = 10000 RECENT_SECONDS = 86400 * 7 # "new" = within last 7 days @@ -222,9 +223,11 @@ class ForumHandlers: f"

    new thread

    " f'' f'{self._csrf_field()}' - f'

    ' + f'
    ' + f"max {MAX_TITLE_LENGTH} characters

    " f'

    ' - f'

    ' + f'
    ' + f"max {MAX_BODY_LENGTH} characters

    " f'

    ' f'' f"" @@ -239,6 +242,10 @@ class ForumHandlers: tags = body.get("tags", [""])[0].strip() if not title: return self.handle_new_form("Title is required.") + if len(title) > MAX_TITLE_LENGTH: + return self.handle_new_form(f"Title too long (max {MAX_TITLE_LENGTH} characters).") + if len(body_text) > MAX_BODY_LENGTH: + return self.handle_new_form(f"Body too long (max {MAX_BODY_LENGTH} characters).") thread_id = secrets.token_hex(16) author_instance = self.identity.hash.hex() if self.identity else "local" author_name = self.site_name @@ -308,7 +315,8 @@ class ForumHandlers: reply_form = ( f'
    ' f'{self._csrf_field()}' - f'

    ' + f'
    ' + f"max {MAX_BODY_LENGTH} characters

    " f'' f"
    " ) @@ -366,9 +374,11 @@ class ForumHandlers: f"

    edit thread

    " f'
    ' f'{self._csrf_field()}' - f'

    ' + f'
    ' + f"max {MAX_TITLE_LENGTH} characters

    " f'

    ' - f'

    ' + f'
    ' + f"max {MAX_BODY_LENGTH} characters

    " f'

    ' f'' f"
    " @@ -386,8 +396,12 @@ class ForumHandlers: title = body.get("title", [""])[0].strip() if not title: return self.handle_edit_form(thread_id, "Title is required.") + if len(title) > MAX_TITLE_LENGTH: + return self.handle_edit_form(thread_id, f"Title too long (max {MAX_TITLE_LENGTH} characters).") url = body.get("url", [""])[0].strip() body_text = body.get("body", [""])[0].strip() + if len(body_text) > MAX_BODY_LENGTH: + return self.handle_edit_form(thread_id, f"Body too long (max {MAX_BODY_LENGTH} characters).") tags = body.get("tags", [""])[0].strip() now = self._now() self.fdb.update_thread(thread_id, title, url, body_text, tags, now) @@ -397,6 +411,8 @@ class ForumHandlers: body_text = body.get("body", [""])[0].strip() if not body_text: return self._redirect(f"/forum/t/{thread_id}") + if len(body_text) > MAX_BODY_LENGTH: + return self._respond(f"

    Body too long (max {MAX_BODY_LENGTH} characters). back

    ") parent_id = body.get("parent_id", [""])[0].strip() author_instance = self.identity.hash.hex() if self.identity else "local" author_name = self.site_name From a1af7eb64fb5cef76f296c87e18cd9866a3d4185 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 01:04:17 +0000 Subject: [PATCH 18/56] max title (200) and body (10k) length limits, fix unescaped body in edit form --- tinyweb_forum/handlers.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 36fdbef..4baf1d4 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -5,7 +5,8 @@ from datetime import datetime from urllib.parse import unquote -PER_PAGE = 20 +MAX_TITLE_LENGTH = 200 +MAX_BODY_LENGTH = 10000 RECENT_SECONDS = 86400 * 7 # "new" = within last 7 days @@ -222,9 +223,11 @@ class ForumHandlers: f"

    new thread

    " f'
    ' f'{self._csrf_field()}' - f'

    ' + f'
    ' + f"max {MAX_TITLE_LENGTH} characters

    " f'

    ' - f'

    ' + f'
    ' + f"max {MAX_BODY_LENGTH} characters

    " f'

    ' f'' f"
    " @@ -239,6 +242,10 @@ class ForumHandlers: tags = body.get("tags", [""])[0].strip() if not title: return self.handle_new_form("Title is required.") + if len(title) > MAX_TITLE_LENGTH: + return self.handle_new_form(f"Title too long (max {MAX_TITLE_LENGTH} characters).") + if len(body_text) > MAX_BODY_LENGTH: + return self.handle_new_form(f"Body too long (max {MAX_BODY_LENGTH} characters).") thread_id = secrets.token_hex(16) author_instance = self.identity.hash.hex() if self.identity else "local" author_name = self.site_name @@ -308,7 +315,8 @@ class ForumHandlers: reply_form = ( f'
    ' f'{self._csrf_field()}' - f'

    ' + f'
    ' + f"max {MAX_BODY_LENGTH} characters

    " f'' f"
    " ) @@ -366,9 +374,11 @@ class ForumHandlers: f"

    edit thread

    " f'
    ' f'{self._csrf_field()}' - f'

    ' + f'
    ' + f"max {MAX_TITLE_LENGTH} characters

    " f'

    ' - f'

    ' + f'
    ' + f"max {MAX_BODY_LENGTH} characters

    " f'

    ' f'' f"
    " @@ -386,8 +396,12 @@ class ForumHandlers: title = body.get("title", [""])[0].strip() if not title: return self.handle_edit_form(thread_id, "Title is required.") + if len(title) > MAX_TITLE_LENGTH: + return self.handle_edit_form(thread_id, f"Title too long (max {MAX_TITLE_LENGTH} characters).") url = body.get("url", [""])[0].strip() body_text = body.get("body", [""])[0].strip() + if len(body_text) > MAX_BODY_LENGTH: + return self.handle_edit_form(thread_id, f"Body too long (max {MAX_BODY_LENGTH} characters).") tags = body.get("tags", [""])[0].strip() now = self._now() self.fdb.update_thread(thread_id, title, url, body_text, tags, now) @@ -397,6 +411,8 @@ class ForumHandlers: body_text = body.get("body", [""])[0].strip() if not body_text: return self._redirect(f"/forum/t/{thread_id}") + if len(body_text) > MAX_BODY_LENGTH: + return self._respond(f"

    Body too long (max {MAX_BODY_LENGTH} characters). back

    ") parent_id = body.get("parent_id", [""])[0].strip() author_instance = self.identity.hash.hex() if self.identity else "local" author_name = self.site_name From edf3d23cd1dad540b0920555ef726b011e1f19f8 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 01:58:32 +0000 Subject: [PATCH 19/56] gossip sync: sync with random 20 peers per cycle instead of all peers --- README.md | 2 ++ tinyweb_forum/db.py | 7 ++++++- tinyweb_forum/sync.py | 8 +++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 26ebdd1..d26c622 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ All moderation is local — you control your view: - Forum instances auto-discover each other via RNS announces - Content is exchanged as JSON over RNS links every 5 minutes - Peer discovery propagates through gossip — each instance shares its known peers +- At scale (>20 peers), sync uses random gossip: each cycle picks 20 random peers instead of all peers +- This ensures content converges epidemically regardless of network size (content reaches all nodes within ~O(log N) cycles) - Block lists and retractions are gossiped alongside content - Only new/updated content is transferred (timestamp-based) - Auto-discovery can be disabled in the moderation page diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index 6ab12de..8a77715 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -54,9 +54,14 @@ class ForumDB: "CREATE TABLE IF NOT EXISTS synced_instances (" " instance_hash TEXT PRIMARY KEY," " name TEXT DEFAULT ''," - " last_sync TEXT DEFAULT ''" + " last_sync TEXT DEFAULT ''," + " status TEXT DEFAULT 'active'" ")" ) + try: + db.execute("ALTER TABLE synced_instances ADD COLUMN status TEXT DEFAULT 'active'") + except Exception: + pass db.execute( "CREATE TABLE IF NOT EXISTS settings (" " key TEXT PRIMARY KEY," diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 440479d..ab4e7b3 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -1,4 +1,5 @@ import json +import random import threading import time import RNS @@ -6,6 +7,7 @@ import RNS FORUM_APP = "tinyweb-forum" SYNC_INTERVAL = 300 # 5 minutes REQUEST_TIMEOUT = 60 +GOSSIP_FANOUT = 20 # random peers to sync per cycle class _ForumAnnounceHandler: @@ -90,7 +92,11 @@ class ForumSync: while self._running: try: instances = self.fdb.get_synced_instances() - for inst in instances: + # Gossip: sync with random subset for scaling + # If <= GOSSIP_FANOUT peers, sync with all (current behavior) + # If more, sync with random FANOUT per cycle — content spreads epidemically + random.shuffle(instances) + for inst in instances[:GOSSIP_FANOUT]: if not self._running: break try: From d77c28cfaa4628d339ad79fc076117d5b2bcffd6 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 01:58:32 +0000 Subject: [PATCH 20/56] gossip sync: sync with random 20 peers per cycle instead of all peers --- README.md | 2 ++ tinyweb_forum/db.py | 7 ++++++- tinyweb_forum/sync.py | 8 +++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3509d4f..7c60aa0 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ All moderation is local — you control your view: - Forum instances auto-discover each other via RNS announces - Content is exchanged as JSON over RNS links every 5 minutes - Peer discovery propagates through gossip — each instance shares its known peers +- At scale (>20 peers), sync uses random gossip: each cycle picks 20 random peers instead of all peers +- This ensures content converges epidemically regardless of network size (content reaches all nodes within ~O(log N) cycles) - Block lists and retractions are gossiped alongside content - Only new/updated content is transferred (timestamp-based) - Auto-discovery can be disabled in the moderation page diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index 6ab12de..8a77715 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -54,9 +54,14 @@ class ForumDB: "CREATE TABLE IF NOT EXISTS synced_instances (" " instance_hash TEXT PRIMARY KEY," " name TEXT DEFAULT ''," - " last_sync TEXT DEFAULT ''" + " last_sync TEXT DEFAULT ''," + " status TEXT DEFAULT 'active'" ")" ) + try: + db.execute("ALTER TABLE synced_instances ADD COLUMN status TEXT DEFAULT 'active'") + except Exception: + pass db.execute( "CREATE TABLE IF NOT EXISTS settings (" " key TEXT PRIMARY KEY," diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 440479d..ab4e7b3 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -1,4 +1,5 @@ import json +import random import threading import time import RNS @@ -6,6 +7,7 @@ import RNS FORUM_APP = "tinyweb-forum" SYNC_INTERVAL = 300 # 5 minutes REQUEST_TIMEOUT = 60 +GOSSIP_FANOUT = 20 # random peers to sync per cycle class _ForumAnnounceHandler: @@ -90,7 +92,11 @@ class ForumSync: while self._running: try: instances = self.fdb.get_synced_instances() - for inst in instances: + # Gossip: sync with random subset for scaling + # If <= GOSSIP_FANOUT peers, sync with all (current behavior) + # If more, sync with random FANOUT per cycle — content spreads epidemically + random.shuffle(instances) + for inst in instances[:GOSSIP_FANOUT]: if not self._running: break try: From 3e450103dff9ff91aedc942a490c25d9e7e8b036 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 02:02:25 +0000 Subject: [PATCH 21/56] manual sync by default: sync now button, auto-sync toggle --- tinyweb_forum/handlers.py | 33 ++++++++++++++++++++++++++++++++- tinyweb_forum/sync.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 4baf1d4..9efcfa3 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -211,6 +211,7 @@ class ForumHandlers: f'{search_form}' f' + new' f' mod' + f' sync now' f' {muted_link}' f"" f"

    {total} threads{new_label}

    " @@ -502,16 +503,26 @@ class ForumHandlers: auto_discover = self.fdb.get_setting("forum_auto_discover", "1") auto_discover_checked = " checked" if auto_discover == "1" else "" + auto_sync = self.fdb.get_setting("forum_auto_sync", "0") + auto_sync_checked = " checked" if auto_sync == "1" else "" retention_days = self.fdb.get_setting("forum_retention_days", "30") return self._respond( f"

    forum moderation

    " f"

    {msg}

    " + f'

    sync now

    ' f"

    auto-discovery

    " f'
    ' f'{self._csrf_field()}' f'

    " + f" automatically discover other forum instances on the mesh

    " + f'' + f"
    " + f"

    auto-sync

    " + f'
    ' + f'{self._csrf_field()}' + f'

    " f'' f"
    " f"

    storage

    " @@ -606,6 +617,22 @@ class ForumHandlers: self.fdb.set_setting("forum_retention_days", str(days)) return self.handle_moderation(f"Storage retention set to {days} days.") + def handle_auto_sync(self, body): + enabled = "1" if body.get("enabled") else "0" + self.fdb.set_setting("forum_auto_sync", enabled) + if self.sync: + self.sync.set_auto_sync(enabled == "1") + return self.handle_moderation(f"Auto-sync {'enabled' if enabled == '1' else 'disabled'}.") + + def handle_sync_now(self): + if not self.sync: + return self._respond("

    Sync not available.

    ") + count = self.sync.sync_now() + msg = f"Synced with {count} instance{'s' if count != 1 else ''}." + if count == 0: + msg = "No peers to sync with." + return self._respond(f"

    {msg}

    back to forum

    ") + def handle_sync_add(self, body): instance = body.get("instance", [""])[0].strip().replace("<", "").replace(">", "") name = body.get("name", [""])[0].strip() @@ -744,6 +771,8 @@ class ForumHandlers: return self._with_csrf(self.handle_unmute(sub[8:]), csrf_token) elif sub.startswith("/blockhash/"): return self._with_csrf(self.handle_block_hash(sub[11:]), csrf_token) + elif sub == "/sync/now": + return self._with_csrf(self.handle_sync_now(), csrf_token) elif method == "POST": if not self._check_csrf(body): return self._with_csrf( @@ -780,6 +809,8 @@ class ForumHandlers: return self._with_csrf(self.handle_auto_discover(body), csrf_token) elif sub == "/storage": return self._with_csrf(self.handle_storage(body), csrf_token) + elif sub == "/auto_sync": + return self._with_csrf(self.handle_auto_sync(body), csrf_token) return self._with_csrf(self._error(404), csrf_token) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index ab4e7b3..99e1985 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -55,13 +55,41 @@ class ForumSync: if self.fdb.get_setting("forum_auto_discover", "1") == "1": self._enable_announce_handler() self._running = True - self._thread = threading.Thread(target=self._sync_loop, daemon=True) - self._thread.start() + if self.fdb.get_setting("forum_auto_sync", "0") == "1": + self._start_sync_loop() def stop(self): self._running = False self._disable_announce_handler() + def set_auto_sync(self, enabled): + self.fdb.set_setting("forum_auto_sync", "1" if enabled else "0") + if enabled: + self._start_sync_loop() + else: + pass # current cycle finishes, no new one starts + + def sync_now(self): + """Run one sync cycle immediately. Returns count of peers synced.""" + instances = self.fdb.get_synced_instances() + random.shuffle(instances) + count = 0 + for inst in instances[:GOSSIP_FANOUT]: + if not self._running: + break + try: + self._sync_with(inst["instance_hash"]) + count += 1 + except Exception as e: + print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") + return count + + def _start_sync_loop(self): + if self._thread and self._thread.is_alive(): + return + self._thread = threading.Thread(target=self._sync_loop, daemon=True) + self._thread.start() + def set_auto_discover(self, enabled): self.fdb.set_setting("forum_auto_discover", "1" if enabled else "0") if enabled: From b118cf5898003d6671a3abaace977585e82b0af7 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 02:02:25 +0000 Subject: [PATCH 22/56] manual sync by default: sync now button, auto-sync toggle --- tinyweb_forum/handlers.py | 33 ++++++++++++++++++++++++++++++++- tinyweb_forum/sync.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 4baf1d4..9efcfa3 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -211,6 +211,7 @@ class ForumHandlers: f'{search_form}' f' + new' f' mod' + f' sync now' f' {muted_link}' f"" f"

    {total} threads{new_label}

    " @@ -502,16 +503,26 @@ class ForumHandlers: auto_discover = self.fdb.get_setting("forum_auto_discover", "1") auto_discover_checked = " checked" if auto_discover == "1" else "" + auto_sync = self.fdb.get_setting("forum_auto_sync", "0") + auto_sync_checked = " checked" if auto_sync == "1" else "" retention_days = self.fdb.get_setting("forum_retention_days", "30") return self._respond( f"

    forum moderation

    " f"

    {msg}

    " + f'

    sync now

    ' f"

    auto-discovery

    " f'
    ' f'{self._csrf_field()}' f'

    " + f" automatically discover other forum instances on the mesh

    " + f'' + f"
    " + f"

    auto-sync

    " + f'
    ' + f'{self._csrf_field()}' + f'

    " f'' f"
    " f"

    storage

    " @@ -606,6 +617,22 @@ class ForumHandlers: self.fdb.set_setting("forum_retention_days", str(days)) return self.handle_moderation(f"Storage retention set to {days} days.") + def handle_auto_sync(self, body): + enabled = "1" if body.get("enabled") else "0" + self.fdb.set_setting("forum_auto_sync", enabled) + if self.sync: + self.sync.set_auto_sync(enabled == "1") + return self.handle_moderation(f"Auto-sync {'enabled' if enabled == '1' else 'disabled'}.") + + def handle_sync_now(self): + if not self.sync: + return self._respond("

    Sync not available.

    ") + count = self.sync.sync_now() + msg = f"Synced with {count} instance{'s' if count != 1 else ''}." + if count == 0: + msg = "No peers to sync with." + return self._respond(f"

    {msg}

    back to forum

    ") + def handle_sync_add(self, body): instance = body.get("instance", [""])[0].strip().replace("<", "").replace(">", "") name = body.get("name", [""])[0].strip() @@ -744,6 +771,8 @@ class ForumHandlers: return self._with_csrf(self.handle_unmute(sub[8:]), csrf_token) elif sub.startswith("/blockhash/"): return self._with_csrf(self.handle_block_hash(sub[11:]), csrf_token) + elif sub == "/sync/now": + return self._with_csrf(self.handle_sync_now(), csrf_token) elif method == "POST": if not self._check_csrf(body): return self._with_csrf( @@ -780,6 +809,8 @@ class ForumHandlers: return self._with_csrf(self.handle_auto_discover(body), csrf_token) elif sub == "/storage": return self._with_csrf(self.handle_storage(body), csrf_token) + elif sub == "/auto_sync": + return self._with_csrf(self.handle_auto_sync(body), csrf_token) return self._with_csrf(self._error(404), csrf_token) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index ab4e7b3..99e1985 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -55,13 +55,41 @@ class ForumSync: if self.fdb.get_setting("forum_auto_discover", "1") == "1": self._enable_announce_handler() self._running = True - self._thread = threading.Thread(target=self._sync_loop, daemon=True) - self._thread.start() + if self.fdb.get_setting("forum_auto_sync", "0") == "1": + self._start_sync_loop() def stop(self): self._running = False self._disable_announce_handler() + def set_auto_sync(self, enabled): + self.fdb.set_setting("forum_auto_sync", "1" if enabled else "0") + if enabled: + self._start_sync_loop() + else: + pass # current cycle finishes, no new one starts + + def sync_now(self): + """Run one sync cycle immediately. Returns count of peers synced.""" + instances = self.fdb.get_synced_instances() + random.shuffle(instances) + count = 0 + for inst in instances[:GOSSIP_FANOUT]: + if not self._running: + break + try: + self._sync_with(inst["instance_hash"]) + count += 1 + except Exception as e: + print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") + return count + + def _start_sync_loop(self): + if self._thread and self._thread.is_alive(): + return + self._thread = threading.Thread(target=self._sync_loop, daemon=True) + self._thread.start() + def set_auto_discover(self, enabled): self.fdb.set_setting("forum_auto_discover", "1" if enabled else "0") if enabled: From 4bd4eb12a502672f482b886657c50732bcdbcb4d Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 02:04:43 +0000 Subject: [PATCH 23/56] fix: restore PER_PAGE constant --- tinyweb_forum/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 9efcfa3..8d125b0 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -7,6 +7,7 @@ from urllib.parse import unquote MAX_TITLE_LENGTH = 200 MAX_BODY_LENGTH = 10000 +PER_PAGE = 20 RECENT_SECONDS = 86400 * 7 # "new" = within last 7 days From 688030c0b4594fc2bde71d53ad79dd1c99910ee4 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 02:04:43 +0000 Subject: [PATCH 24/56] fix: restore PER_PAGE constant --- tinyweb_forum/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 9efcfa3..8d125b0 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -7,6 +7,7 @@ from urllib.parse import unquote MAX_TITLE_LENGTH = 200 MAX_BODY_LENGTH = 10000 +PER_PAGE = 20 RECENT_SECONDS = 86400 * 7 # "new" = within last 7 days From 1250a67f70ca2fc537879e89045933a44c0c5c0f Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 02:07:31 +0000 Subject: [PATCH 25/56] cleaner form layout: CSS, forum-form class, remove hardcoded sizes/br tags --- tinyweb_forum/handlers.py | 73 ++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 8d125b0..3ec6bb9 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -16,6 +16,17 @@ def esc(s): return html.escape(str(s)) +FORUM_CSS = """ + +""" + + class ForumHandlers: def __init__(self, fdb, sync, identity, reticulum, site_name="me"): self.fdb = fdb @@ -53,7 +64,7 @@ class ForumHandlers: return { "status": status, "content_type": "text/html; charset=utf-8", - "body": body_html, + "body": FORUM_CSS + body_html, "headers": {}, } @@ -223,14 +234,14 @@ class ForumHandlers: def handle_new_form(self, msg=""): return self._respond( f"

    new thread

    " - f'
    ' + f'' f'{self._csrf_field()}' - f'
    ' - f"max {MAX_TITLE_LENGTH} characters

    " - f'

    ' - f'
    ' - f"max {MAX_BODY_LENGTH} characters

    " - f'

    ' + f'' + f"max {MAX_TITLE_LENGTH} characters" + f'' + f'' + f"max {MAX_BODY_LENGTH} characters" + f'' f'' f"
    " f"

    {msg}

    " @@ -315,10 +326,10 @@ class ForumHandlers: ) reply_form = ( - f'
    ' + f'' f'{self._csrf_field()}' - f'
    ' - f"max {MAX_BODY_LENGTH} characters

    " + f'' + f"max {MAX_BODY_LENGTH} characters" f'' f"
    " ) @@ -374,14 +385,14 @@ class ForumHandlers: return self._error(403) return self._respond( f"

    edit thread

    " - f'
    ' + f'' f'{self._csrf_field()}' - f'
    ' - f"max {MAX_TITLE_LENGTH} characters

    " - f'

    ' - f'
    ' - f"max {MAX_BODY_LENGTH} characters

    " - f'

    ' + f'' + f"max {MAX_TITLE_LENGTH} characters" + f'' + f'' + f"max {MAX_BODY_LENGTH} characters" + f'' f'' f"
    " f"

    {msg}

    " @@ -513,50 +524,50 @@ class ForumHandlers: f"

    {msg}

    " f'

    sync now

    ' f"

    auto-discovery

    " - f'
    ' + f'' f'{self._csrf_field()}' f'

    " + f" automatically discover other forum instances on the mesh" f'' f"
    " f"

    auto-sync

    " - f'
    ' + f'' f'{self._csrf_field()}' f'

    " + f" automatically sync content every 5 minutes" f'' f"
    " f"

    storage

    " - f'
    ' + f'' f'{self._csrf_field()}' f'' - f"
    Older threads are pruned automatically (default: 30). Set to 0 to keep everything.

    " + f"Older threads are pruned automatically (default: 30). Set to 0 to keep everything." f'' f"
    " f"

    blocked instances

    " f"{blocked_items}" - f'
    ' + f'' f'{self._csrf_field()}' - f'

    ' + f'' f'' f"
    " f"

    peer reports

    " f"{self._peer_reports_html()}" f"

    keyword filters

    " - f'
    ' + f'' f'{self._csrf_field()}' - f'

    ' + f'' f'' f"
    " f"

    synced instances

    " f"{synced_items}" f"

    Instances are discovered automatically via mesh announces. " f"You can also manually add a friend's instance hash to bootstrap.

    " - f'
    ' + f'' f'{self._csrf_field()}' - f'

    ' - f'

    ' + f'' + f'' f'' f"
    " f'
    ' From dd22dc371456a4f93a4a7621354cc228f0b09a3f Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 02:07:31 +0000 Subject: [PATCH 26/56] cleaner form layout: CSS, forum-form class, remove hardcoded sizes/br tags --- tinyweb_forum/handlers.py | 73 ++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 8d125b0..3ec6bb9 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -16,6 +16,17 @@ def esc(s): return html.escape(str(s)) +FORUM_CSS = """ + +""" + + class ForumHandlers: def __init__(self, fdb, sync, identity, reticulum, site_name="me"): self.fdb = fdb @@ -53,7 +64,7 @@ class ForumHandlers: return { "status": status, "content_type": "text/html; charset=utf-8", - "body": body_html, + "body": FORUM_CSS + body_html, "headers": {}, } @@ -223,14 +234,14 @@ class ForumHandlers: def handle_new_form(self, msg=""): return self._respond( f"

    new thread

    " - f'
    ' + f'' f'{self._csrf_field()}' - f'
    ' - f"max {MAX_TITLE_LENGTH} characters

    " - f'

    ' - f'
    ' - f"max {MAX_BODY_LENGTH} characters

    " - f'

    ' + f'' + f"max {MAX_TITLE_LENGTH} characters" + f'' + f'' + f"max {MAX_BODY_LENGTH} characters" + f'' f'' f"
    " f"

    {msg}

    " @@ -315,10 +326,10 @@ class ForumHandlers: ) reply_form = ( - f'
    ' + f'' f'{self._csrf_field()}' - f'
    ' - f"max {MAX_BODY_LENGTH} characters

    " + f'' + f"max {MAX_BODY_LENGTH} characters" f'' f"
    " ) @@ -374,14 +385,14 @@ class ForumHandlers: return self._error(403) return self._respond( f"

    edit thread

    " - f'
    ' + f'' f'{self._csrf_field()}' - f'
    ' - f"max {MAX_TITLE_LENGTH} characters

    " - f'

    ' - f'
    ' - f"max {MAX_BODY_LENGTH} characters

    " - f'

    ' + f'' + f"max {MAX_TITLE_LENGTH} characters" + f'' + f'' + f"max {MAX_BODY_LENGTH} characters" + f'' f'' f"
    " f"

    {msg}

    " @@ -513,50 +524,50 @@ class ForumHandlers: f"

    {msg}

    " f'

    sync now

    ' f"

    auto-discovery

    " - f'
    ' + f'' f'{self._csrf_field()}' f'

    " + f" automatically discover other forum instances on the mesh" f'' f"
    " f"

    auto-sync

    " - f'
    ' + f'' f'{self._csrf_field()}' f'

    " + f" automatically sync content every 5 minutes" f'' f"
    " f"

    storage

    " - f'
    ' + f'' f'{self._csrf_field()}' f'' - f"
    Older threads are pruned automatically (default: 30). Set to 0 to keep everything.

    " + f"Older threads are pruned automatically (default: 30). Set to 0 to keep everything." f'' f"
    " f"

    blocked instances

    " f"{blocked_items}" - f'
    ' + f'' f'{self._csrf_field()}' - f'

    ' + f'' f'' f"
    " f"

    peer reports

    " f"{self._peer_reports_html()}" f"

    keyword filters

    " - f'
    ' + f'' f'{self._csrf_field()}' - f'

    ' + f'' f'' f"
    " f"

    synced instances

    " f"{synced_items}" f"

    Instances are discovered automatically via mesh announces. " f"You can also manually add a friend's instance hash to bootstrap.

    " - f'
    ' + f'' f'{self._csrf_field()}' - f'

    ' - f'

    ' + f'' + f'' f'' f"
    " f'
    ' From da5b7c44a6552c2e38cbd426ddbda6fff8dc7188 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 02:10:12 +0000 Subject: [PATCH 27/56] hide block link + show 'me' for own content by matching identity hash --- tinyweb_forum/handlers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 3ec6bb9..5875210 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -50,13 +50,20 @@ class ForumHandlers: return False return secrets.compare_digest(token, expected) - def _author_str(self, name, instance): + def _is_local(self, instance): if instance == "local": + return True + if self.identity and instance == self.identity.hash.hex(): + return True + return False + + def _author_str(self, name, instance): + if self._is_local(instance): return "me" return instance[:6] def _block_link(self, instance): - if instance == "local": + if self._is_local(instance): return "" return f' [block]' From 58479e751b6c9859fce946117131b189cd21ca9a Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 02:10:12 +0000 Subject: [PATCH 28/56] hide block link + show 'me' for own content by matching identity hash --- tinyweb_forum/handlers.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 3ec6bb9..5875210 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -50,13 +50,20 @@ class ForumHandlers: return False return secrets.compare_digest(token, expected) - def _author_str(self, name, instance): + def _is_local(self, instance): if instance == "local": + return True + if self.identity and instance == self.identity.hash.hex(): + return True + return False + + def _author_str(self, name, instance): + if self._is_local(instance): return "me" return instance[:6] def _block_link(self, instance): - if instance == "local": + if self._is_local(instance): return "" return f' [block]' From ea61f371c74597a6aa8da5ef065920553ec8b051 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 02:16:46 +0000 Subject: [PATCH 29/56] update README: manual sync by default, not configurable, pruning detail --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d26c622..30c5fc7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # tinyweb-forum -A decentralized link-sharing forum for [TinyWeb](https://git.derickphan.com/blankie/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. +A decentralized link-sharing forum for [TinyWeb](https://git.derickphan.com/blankie/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. No accounts, no sign-up, no central server. ## Install @@ -20,7 +20,9 @@ pip install -e . ## How it works - Each TinyWeb instance stores forum threads and posts in its own `forum.db` -- Instances sync content with each other over RNS every 5 minutes +- Instances auto-discover each other on the mesh via RNS announces — no manual setup +- You click "sync now" to exchange content; auto-sync every 5 minutes is optional (toggle on moderation page) +- At scale, sync uses epidemic gossip: each cycle picks 20 random peers instead of all peers, converging within ~O(log N) cycles - Authors are identified by a short pseudonymous hash (no names, no accounts) - No global server, no algorithms, no tracking @@ -31,29 +33,32 @@ pip install -e . - **Upvotes** — toggle upvote/downvote, scores propagate via sync - **Edit** — edit your own threads (new version syncs to peers) - **Retract** — retract your own threads and posts (retraction signal gossips to peers) +- **Peer discovery** — instances share known peers during sync, growing the network organically ## Moderation All moderation is local — you control your view: - **Block author** — `[block]` link on posts and thread meta hides all content from that identity across your instance -- **Auto-block** — when 3+ of your peers have blocked the same identity, it's auto-blocked for you too (configurable threshold) +- **Auto-block** — when 3+ of your peers have blocked the same identity, it's auto-blocked for you too (threshold is 3) - **Mute thread** — hide a thread from the listing - **Keyword filters** — hide threads matching keywords - **Instance sync** — choose which peers to sync with; unsync at any time ## Sync -- Forum instances auto-discover each other via RNS announces -- Content is exchanged as JSON over RNS links every 5 minutes -- Peer discovery propagates through gossip — each instance shares its known peers -- At scale (>20 peers), sync uses random gossip: each cycle picks 20 random peers instead of all peers -- This ensures content converges epidemically regardless of network size (content reaches all nodes within ~O(log N) cycles) +- Instances auto-discover each other via RNS announces — just run the forum and any other forum on the mesh finds you +- Content is exchanged as JSON over RNS links +- **Manual by default** — click "sync now" on the forum listing or moderation page +- Auto-sync every 5 minutes can be enabled on the moderation page +- Auto-discovery can also be disabled on the moderation page (manual instance add only) +- Peer discovery propagates through gossip — each instance shares its known peers during sync - Block lists and retractions are gossiped alongside content - Only new/updated content is transferred (timestamp-based) -- Auto-discovery can be disabled in the moderation page +- When >20 peers, each cycle syncs with a random 20 — content converges epidemically ## Storage -- Threads are pruned after 30 days by default (configurable in moderation page) +- Threads older than 30 days are auto-pruned (configurable on the moderation page) - Set retention to 0 to keep everything indefinitely +- Forum DB is stored at `~/.tinyweb/forum.db` From 677a03f19d1b3b150b04d54f70bd16ae22ca0f33 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 02:16:46 +0000 Subject: [PATCH 30/56] update README: manual sync by default, not configurable, pruning detail --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7c60aa0..743d04d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # tinyweb-forum -A decentralized link-sharing forum for [TinyWeb](https://git.example.com/user/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. +A decentralized link-sharing forum for [TinyWeb](https://git.example.com/user/tinyweb). Share URLs and discuss them with other TinyWeb instances over the Reticulum mesh. No accounts, no sign-up, no central server. ## Install @@ -20,7 +20,9 @@ pip install -e . ## How it works - Each TinyWeb instance stores forum threads and posts in its own `forum.db` -- Instances sync content with each other over RNS every 5 minutes +- Instances auto-discover each other on the mesh via RNS announces — no manual setup +- You click "sync now" to exchange content; auto-sync every 5 minutes is optional (toggle on moderation page) +- At scale, sync uses epidemic gossip: each cycle picks 20 random peers instead of all peers, converging within ~O(log N) cycles - Authors are identified by a short pseudonymous hash (no names, no accounts) - No global server, no algorithms, no tracking @@ -31,29 +33,32 @@ pip install -e . - **Upvotes** — toggle upvote/downvote, scores propagate via sync - **Edit** — edit your own threads (new version syncs to peers) - **Retract** — retract your own threads and posts (retraction signal gossips to peers) +- **Peer discovery** — instances share known peers during sync, growing the network organically ## Moderation All moderation is local — you control your view: - **Block author** — `[block]` link on posts and thread meta hides all content from that identity across your instance -- **Auto-block** — when 3+ of your peers have blocked the same identity, it's auto-blocked for you too (configurable threshold) +- **Auto-block** — when 3+ of your peers have blocked the same identity, it's auto-blocked for you too (threshold is 3) - **Mute thread** — hide a thread from the listing - **Keyword filters** — hide threads matching keywords - **Instance sync** — choose which peers to sync with; unsync at any time ## Sync -- Forum instances auto-discover each other via RNS announces -- Content is exchanged as JSON over RNS links every 5 minutes -- Peer discovery propagates through gossip — each instance shares its known peers -- At scale (>20 peers), sync uses random gossip: each cycle picks 20 random peers instead of all peers -- This ensures content converges epidemically regardless of network size (content reaches all nodes within ~O(log N) cycles) +- Instances auto-discover each other via RNS announces — just run the forum and any other forum on the mesh finds you +- Content is exchanged as JSON over RNS links +- **Manual by default** — click "sync now" on the forum listing or moderation page +- Auto-sync every 5 minutes can be enabled on the moderation page +- Auto-discovery can also be disabled on the moderation page (manual instance add only) +- Peer discovery propagates through gossip — each instance shares its known peers during sync - Block lists and retractions are gossiped alongside content - Only new/updated content is transferred (timestamp-based) -- Auto-discovery can be disabled in the moderation page +- When >20 peers, each cycle syncs with a random 20 — content converges epidemically ## Storage -- Threads are pruned after 30 days by default (configurable in moderation page) +- Threads older than 30 days are auto-pruned (configurable on the moderation page) - Set retention to 0 to keep everything indefinitely +- Forum DB is stored at `~/.tinyweb/forum.db` From 06571fd3ed5a3e600d7547ad4aeea9808c45a1e5 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 02:18:51 +0000 Subject: [PATCH 31/56] add Security section to README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 30c5fc7..405dde6 100644 --- a/README.md +++ b/README.md @@ -62,3 +62,10 @@ All moderation is local — you control your view: - Threads older than 30 days are auto-pruned (configurable on the moderation page) - Set retention to 0 to keep everything indefinitely - Forum DB is stored at `~/.tinyweb/forum.db` + +## Security + +- **No authentication** — The forum inherits TinyWeb's access model. Anyone who can reach the HTTP port (localhost by default) can post, edit, retract, block, and change moderation settings. See TinyWeb's Security section for details on `--bind 0.0.0.0`. +- **Retractions are voluntary** — Retracting a thread or post sends a signal to peers, but any peer can ignore it and keep serving the content. "Retract" is a polite request, not a guaranteed delete. +- **Block gossip can be gamed** — Auto-block triggers after 3 peer reports. On Reticulum this requires 3+ real instances to collude, which is impractical at mesh scale, but is not cryptographically enforced. +- **No rate limiting** — Forum POST endpoints have no throttling. Low risk since the HTTP port is localhost-only by default. From 53d19a5b9a1f63df7a93d118743325482f17d044 Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 02:18:51 +0000 Subject: [PATCH 32/56] add Security section to README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 743d04d..7c02df3 100644 --- a/README.md +++ b/README.md @@ -62,3 +62,10 @@ All moderation is local — you control your view: - Threads older than 30 days are auto-pruned (configurable on the moderation page) - Set retention to 0 to keep everything indefinitely - Forum DB is stored at `~/.tinyweb/forum.db` + +## Security + +- **No authentication** — The forum inherits TinyWeb's access model. Anyone who can reach the HTTP port (localhost by default) can post, edit, retract, block, and change moderation settings. See TinyWeb's Security section for details on `--bind 0.0.0.0`. +- **Retractions are voluntary** — Retracting a thread or post sends a signal to peers, but any peer can ignore it and keep serving the content. "Retract" is a polite request, not a guaranteed delete. +- **Block gossip can be gamed** — Auto-block triggers after 3 peer reports. On Reticulum this requires 3+ real instances to collude, which is impractical at mesh scale, but is not cryptographically enforced. +- **No rate limiting** — Forum POST endpoints have no throttling. Low risk since the HTTP port is localhost-only by default. From b1ea8647deeeb2bf954777aae3980626065b28f0 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 02:32:33 +0000 Subject: [PATCH 33/56] fix checkbox layout: exclude from full-width, flex label --- tinyweb_forum/handlers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 5875210..91eca8d 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -19,9 +19,11 @@ def esc(s): FORUM_CSS = """ """ @@ -533,14 +535,14 @@ class ForumHandlers: f"

    auto-discovery

    " f'
    ' f'{self._csrf_field()}' - f'
  • ' + f'
    ' f'{badge}{mute_badge} ' f'{esc(r["title"])}' f'{tags_html}' - f'
    ' - f'{esc(self._author_str(r["author_name"], r["author_instance"]))}' + f'
    ' + f'
    ' + f'{esc(self._author_str(r["author_name"], r["author_instance"]))}' f' · {self._time_ago(r["created_at"])}' f' · {r["score"]} upvotes' - f' · {reply_label}' + f' · {reply_label}' + f'
    ' f'
    • ' ) if not items: @@ -265,7 +272,7 @@ class ForumHandlers: new_label = f" ({new_count} new)" if new_count else "" search_form = ( f'
    ' - f'' + f'' f'
    ' ) tag_label = f' — tag: {esc(tag)}' if tag else "" @@ -273,15 +280,14 @@ class ForumHandlers: page_url = f'/forum?q={esc(search)}&tag={esc(tag)}&muted=1' if show_muted else (f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum') return self._respond( f"

    forum{tag_label}

    " - f'
    ' + f'
    ' f'{search_form}' - f"
    " - f'
    ' + f'
    ' f'+ new' f'mod' f'sync now' f'{muted_link}' - f"
    " + f'
    ' f"

    {total} threads{new_label}

    " f'
      {items}
    ' f"{self._page_nav(page, total, page_url)}" From 40ba5c98243924ae144ac91c02f6c1d8f898525d Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 04:24:15 +0000 Subject: [PATCH 48/56] clean layout: toolbar row with search+actions, structured thread list items --- tinyweb_forum/handlers.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index c2678f9..7cb6b52 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -45,17 +45,18 @@ FORUM_CSS = """ .forum-form label.checkbox-label { display: flex; align-items: center; gap: 6px; margin-bottom: 8px; } .forum-form label.inline-label { display: inline-flex; align-items: center; gap: 4px; margin-bottom: 8px; white-space: nowrap; } .forum-form small { display: block; margin-bottom: 8px; } -.forum-search { margin: 0.5rem 0; } -.forum-search form { display: block; } -.forum-search input[name=q] { - width: 100%; box-sizing: border-box; padding: 10px 12px; +.forum-toolbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin: 0.5rem 0; } +.forum-toolbar form { flex: 1; min-width: 160px; } +.forum-toolbar input[name=q] { + width: 100%; box-sizing: border-box; padding: 8px 12px; background: rgba(8, 18, 22, 0.8); border: 1px solid rgba(40, 70, 65, 0.4); - border-radius: 4px; color: #90b4ac; font-size: 0.95rem; + border-radius: 4px; color: #90b4ac; font-size: 0.9rem; transition: border-color 0.2s, box-shadow 0.3s; } -.forum-search input[name=q]:focus { +.forum-toolbar input[name=q]:focus { outline: none; border-color: rgba(80, 140, 110, 0.5); box-shadow: 0 0 18px rgba(100, 200, 150, 0.06); } +.forum-toolbar-actions { display: flex; flex-wrap: wrap; gap: 6px; } .forum-actions { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 0.5rem 0; } a.forum-action, a.forum-action-inline { color: #5a7880; text-decoration: none; border-bottom: none; font-size: 0.88rem; @@ -67,8 +68,11 @@ a.forum-action-inline { padding: 2px 6px; border: none; } a.forum-action-inline:hover { border: none; } p.meta { font-size: 0.85rem; color: #3a5560; } .forum-list { list-style: none; padding-left: 0; } -.forum-list li { padding: 0.6rem 0; border-bottom: 1px solid rgba(30, 55, 50, 0.2); } +.forum-list li { padding: 0.8rem 0; border-bottom: 1px solid rgba(30, 55, 50, 0.2); } .forum-list li:last-child { border-bottom: none; } +.forum-list .thread-title { margin-bottom: 0.15rem; } +.forum-list .thread-meta { font-size: 0.78rem; color: #3a5560; } +.forum-list a { border-bottom: none; } .forum-form input[type=text], .forum-form input[type=url] { font-family: inherit; } .forum .post { border-left-color: rgba(40, 70, 65, 0.3); } """ @@ -250,14 +254,17 @@ class ForumHandlers: reply_label = f"{r['reply_count']} replies" if r['reply_count'] else "no replies" items += ( f'
  • ' + f'
    ' f'{badge}{mute_badge} ' f'{esc(r["title"])}' f'{tags_html}' - f'
    ' - f'{esc(self._author_str(r["author_name"], r["author_instance"]))}' + f'
    ' + f'
    ' + f'{esc(self._author_str(r["author_name"], r["author_instance"]))}' f' · {self._time_ago(r["created_at"])}' f' · {r["score"]} upvotes' - f' · {reply_label}' + f' · {reply_label}' + f'
    ' f'
    • ' ) if not items: @@ -265,7 +272,7 @@ class ForumHandlers: new_label = f" ({new_count} new)" if new_count else "" search_form = ( f'
    ' - f'' + f'' f'
    ' ) tag_label = f' — tag: {esc(tag)}' if tag else "" @@ -273,15 +280,14 @@ class ForumHandlers: page_url = f'/forum?q={esc(search)}&tag={esc(tag)}&muted=1' if show_muted else (f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum') return self._respond( f"

    forum{tag_label}

    " - f'
    ' + f'
    ' f'{search_form}' - f"
    " - f'
    ' + f'
    ' f'+ new' f'mod' f'sync now' f'{muted_link}' - f"
    " + f'
    ' f"

    {total} threads{new_label}

    " f'
      {items}
    ' f"{self._page_nav(page, total, page_url)}" From ce6d31c357e7fde6c1c241409b8b10908bf0deb7 Mon Sep 17 00:00:00 2001 From: blankie Date: Fri, 5 Jun 2026 04:28:02 +0000 Subject: [PATCH 49/56] strip dark colors from FORUM_CSS, let theme handle input/button visuals --- tinyweb_forum/handlers.py | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 7cb6b52..5ff8d3d 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -21,23 +21,15 @@ FORUM_CSS = """ .forum-form { max-width: 500px; } .forum-form input:not([type=checkbox]):not([type=radio]), .forum-form textarea { width: 100%; box-sizing: border-box; padding: 10px 12px; margin-bottom: 12px; - background: rgba(8, 18, 22, 0.8); border: 1px solid rgba(40, 70, 65, 0.4); - border-radius: 4px; color: #90b4ac; font-size: 0.95rem; - transition: border-color 0.2s, box-shadow 0.3s; + font-size: 0.95rem; } .forum-form input:not([type=checkbox]):not([type=radio]):focus, .forum-form textarea:focus { - outline: none; border-color: rgba(80, 140, 110, 0.5); box-shadow: 0 0 18px rgba(100, 200, 150, 0.06); + outline: none; } .forum-form input[type=checkbox] { width: auto; margin: 0; } .forum-form button { padding: 10px 20px; margin-bottom: 12px; - background: rgba(10, 22, 25, 0.8); border: 1px solid rgba(40, 70, 65, 0.4); - border-radius: 4px; color: #5a7880; font-size: 0.88rem; cursor: pointer; - transition: background 0.2s, color 0.2s, border-color 0.2s, box-shadow 0.3s; -} -.forum-form button:hover { - background: rgba(15, 35, 35, 0.8); color: #90b4ac; - border-color: rgba(80, 130, 110, 0.5); box-shadow: 0 0 12px rgba(100, 200, 150, 0.06); + cursor: pointer; } .forum-form textarea { font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace; font-size: 0.85rem; line-height: 1.6; resize: vertical; } .forum-form label.inline-label input { width: 60px; display: inline; } @@ -47,34 +39,24 @@ FORUM_CSS = """ .forum-form small { display: block; margin-bottom: 8px; } .forum-toolbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin: 0.5rem 0; } .forum-toolbar form { flex: 1; min-width: 160px; } -.forum-toolbar input[name=q] { - width: 100%; box-sizing: border-box; padding: 8px 12px; - background: rgba(8, 18, 22, 0.8); border: 1px solid rgba(40, 70, 65, 0.4); - border-radius: 4px; color: #90b4ac; font-size: 0.9rem; - transition: border-color 0.2s, box-shadow 0.3s; -} -.forum-toolbar input[name=q]:focus { - outline: none; border-color: rgba(80, 140, 110, 0.5); box-shadow: 0 0 18px rgba(100, 200, 150, 0.06); -} +.forum-toolbar input[name=q] { width: 100%; box-sizing: border-box; padding: 8px 12px; font-size: 0.9rem; } +.forum-toolbar input[name=q]:focus { outline: none; } .forum-toolbar-actions { display: flex; flex-wrap: wrap; gap: 6px; } .forum-actions { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 0.5rem 0; } a.forum-action, a.forum-action-inline { - color: #5a7880; text-decoration: none; border-bottom: none; font-size: 0.88rem; - padding: 8px 14px; border: 1px solid rgba(40, 70, 65, 0.3); border-radius: 4px; + text-decoration: none; border-bottom: none; font-size: 0.88rem; + padding: 8px 14px; transition: background 0.2s, color 0.2s, border-color 0.2s; } -a.forum-action:hover, a.forum-action-inline:hover { color: #90b4ac; border-color: rgba(80, 130, 110, 0.5); background: rgba(10, 22, 25, 0.6); } a.forum-action-inline { padding: 2px 6px; border: none; } a.forum-action-inline:hover { border: none; } -p.meta { font-size: 0.85rem; color: #3a5560; } +p.meta { font-size: 0.85rem; } .forum-list { list-style: none; padding-left: 0; } -.forum-list li { padding: 0.8rem 0; border-bottom: 1px solid rgba(30, 55, 50, 0.2); } +.forum-list li { padding: 0.8rem 0; border-bottom: 1px solid; } .forum-list li:last-child { border-bottom: none; } .forum-list .thread-title { margin-bottom: 0.15rem; } -.forum-list .thread-meta { font-size: 0.78rem; color: #3a5560; } +.forum-list .thread-meta { font-size: 0.78rem; } .forum-list a { border-bottom: none; } -.forum-form input[type=text], .forum-form input[type=url] { font-family: inherit; } -.forum .post { border-left-color: rgba(40, 70, 65, 0.3); } """ From d9c5234fc7552a43dd0773e35c19eed697038e8f Mon Sep 17 00:00:00 2001 From: user Date: Fri, 5 Jun 2026 04:28:02 +0000 Subject: [PATCH 50/56] strip dark colors from FORUM_CSS, let theme handle input/button visuals --- tinyweb_forum/handlers.py | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 7cb6b52..5ff8d3d 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -21,23 +21,15 @@ FORUM_CSS = """ .forum-form { max-width: 500px; } .forum-form input:not([type=checkbox]):not([type=radio]), .forum-form textarea { width: 100%; box-sizing: border-box; padding: 10px 12px; margin-bottom: 12px; - background: rgba(8, 18, 22, 0.8); border: 1px solid rgba(40, 70, 65, 0.4); - border-radius: 4px; color: #90b4ac; font-size: 0.95rem; - transition: border-color 0.2s, box-shadow 0.3s; + font-size: 0.95rem; } .forum-form input:not([type=checkbox]):not([type=radio]):focus, .forum-form textarea:focus { - outline: none; border-color: rgba(80, 140, 110, 0.5); box-shadow: 0 0 18px rgba(100, 200, 150, 0.06); + outline: none; } .forum-form input[type=checkbox] { width: auto; margin: 0; } .forum-form button { padding: 10px 20px; margin-bottom: 12px; - background: rgba(10, 22, 25, 0.8); border: 1px solid rgba(40, 70, 65, 0.4); - border-radius: 4px; color: #5a7880; font-size: 0.88rem; cursor: pointer; - transition: background 0.2s, color 0.2s, border-color 0.2s, box-shadow 0.3s; -} -.forum-form button:hover { - background: rgba(15, 35, 35, 0.8); color: #90b4ac; - border-color: rgba(80, 130, 110, 0.5); box-shadow: 0 0 12px rgba(100, 200, 150, 0.06); + cursor: pointer; } .forum-form textarea { font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace; font-size: 0.85rem; line-height: 1.6; resize: vertical; } .forum-form label.inline-label input { width: 60px; display: inline; } @@ -47,34 +39,24 @@ FORUM_CSS = """ .forum-form small { display: block; margin-bottom: 8px; } .forum-toolbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin: 0.5rem 0; } .forum-toolbar form { flex: 1; min-width: 160px; } -.forum-toolbar input[name=q] { - width: 100%; box-sizing: border-box; padding: 8px 12px; - background: rgba(8, 18, 22, 0.8); border: 1px solid rgba(40, 70, 65, 0.4); - border-radius: 4px; color: #90b4ac; font-size: 0.9rem; - transition: border-color 0.2s, box-shadow 0.3s; -} -.forum-toolbar input[name=q]:focus { - outline: none; border-color: rgba(80, 140, 110, 0.5); box-shadow: 0 0 18px rgba(100, 200, 150, 0.06); -} +.forum-toolbar input[name=q] { width: 100%; box-sizing: border-box; padding: 8px 12px; font-size: 0.9rem; } +.forum-toolbar input[name=q]:focus { outline: none; } .forum-toolbar-actions { display: flex; flex-wrap: wrap; gap: 6px; } .forum-actions { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 0.5rem 0; } a.forum-action, a.forum-action-inline { - color: #5a7880; text-decoration: none; border-bottom: none; font-size: 0.88rem; - padding: 8px 14px; border: 1px solid rgba(40, 70, 65, 0.3); border-radius: 4px; + text-decoration: none; border-bottom: none; font-size: 0.88rem; + padding: 8px 14px; transition: background 0.2s, color 0.2s, border-color 0.2s; } -a.forum-action:hover, a.forum-action-inline:hover { color: #90b4ac; border-color: rgba(80, 130, 110, 0.5); background: rgba(10, 22, 25, 0.6); } a.forum-action-inline { padding: 2px 6px; border: none; } a.forum-action-inline:hover { border: none; } -p.meta { font-size: 0.85rem; color: #3a5560; } +p.meta { font-size: 0.85rem; } .forum-list { list-style: none; padding-left: 0; } -.forum-list li { padding: 0.8rem 0; border-bottom: 1px solid rgba(30, 55, 50, 0.2); } +.forum-list li { padding: 0.8rem 0; border-bottom: 1px solid; } .forum-list li:last-child { border-bottom: none; } .forum-list .thread-title { margin-bottom: 0.15rem; } -.forum-list .thread-meta { font-size: 0.78rem; color: #3a5560; } +.forum-list .thread-meta { font-size: 0.78rem; } .forum-list a { border-bottom: none; } -.forum-form input[type=text], .forum-form input[type=url] { font-family: inherit; } -.forum .post { border-left-color: rgba(40, 70, 65, 0.3); } """ From 497079c7a74ee867e06ae457c5e64904b5d5f160 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 6 Jun 2026 01:19:40 +0000 Subject: [PATCH 51/56] Architecture B: Bloom Gossip + Implicit Replication - Tag bloom filter as primary peer discovery (2048 bits x 3 hashes) - Filter table gossip for transitive peer discovery - Scoped queries for on-demand tag lookup - Liveness tracking + peer eviction (5 failures, 7-day TTL) - Topic subscriptions filter content sync at both ends - Two-theme CSS system (default minimal + kodama2) - Status bar on all pages (topics, peers, filters) - Bracketless tags, grouped moderation page, cleaner forms - Match main site heading level (h1 -> h2) --- tinyweb_forum/bloom.py | 48 ++++ tinyweb_forum/db.py | 193 +++++++++++++++- tinyweb_forum/handlers.py | 467 +++++++++++++++++++++++++++----------- tinyweb_forum/sync.py | 331 ++++++++++++++++++++++++--- 4 files changed, 875 insertions(+), 164 deletions(-) create mode 100644 tinyweb_forum/bloom.py diff --git a/tinyweb_forum/bloom.py b/tinyweb_forum/bloom.py new file mode 100644 index 0000000..a481e01 --- /dev/null +++ b/tinyweb_forum/bloom.py @@ -0,0 +1,48 @@ +import hashlib + + +class BloomFilter: + def __init__(self, size=2048, num_hashes=3): + self.size = size + self.num_hashes = num_hashes + self.bits = bytearray(size // 8 + 1) + + def _hash_positions(self, item): + h = hashlib.sha256(item.encode("utf-8")).digest() + for i in range(self.num_hashes): + val = int.from_bytes(h[i*4:(i+1)*4], "big") % self.size + yield val + + def add(self, item): + for pos in self._hash_positions(item): + self.bits[pos // 8] |= 1 << (pos % 8) + + def might_contain(self, item): + return all( + bool(self.bits[pos // 8] & (1 << (pos % 8))) + for pos in self._hash_positions(item) + ) + + @property + def bytes(self): + return bytes(self.bits) + + @classmethod + def from_bytes(cls, data, size=2048, num_hashes=3): + bf = cls(size=size, num_hashes=num_hashes) + bf.bits = bytearray(data) + return bf + + @staticmethod + def from_items(items, size=2048, num_hashes=3): + bf = BloomFilter(size=size, num_hashes=num_hashes) + for item in items: + bf.add(item) + return bf + + @staticmethod + def from_tags(tags, size=2048, num_hashes=3): + bf = BloomFilter(size=size, num_hashes=num_hashes) + for tag in tags: + bf.add(tag.strip().lower()) + return bf diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index 8a77715..8f396d7 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -1,8 +1,11 @@ import sqlite3 import os import threading +import time from datetime import datetime, timedelta +from tinyweb_forum.bloom import BloomFilter + FORUM_DB = "forum.db" @@ -88,6 +91,20 @@ class ForumDB: " PRIMARY KEY (content_id, content_type)" ")" ) + db.execute( + "CREATE TABLE IF NOT EXISTS peer_filters (" + " peer_hash TEXT PRIMARY KEY," + " bloom_bytes BLOB," + " bloom_size INTEGER DEFAULT 2048," + " bloom_hashes INTEGER DEFAULT 3," + " tag_count INTEGER DEFAULT 0," + " last_seen REAL NOT NULL" + ")" + ) + try: + db.execute("ALTER TABLE synced_instances ADD COLUMN consecutive_failures INTEGER DEFAULT 0") + except Exception: + pass db.commit() db.close() @@ -509,22 +526,190 @@ class ForumDB: db = self.get_db() try: cutoff = (datetime.utcnow() - timedelta(days=retention_days)).strftime("%Y-%m-%dT%H:%M:%S") - # Delete posts in old threads db.execute( "DELETE FROM posts WHERE thread_id IN " "(SELECT id FROM threads WHERE updated_at < ?)", (cutoff,), ) - # Delete orphaned posts (thread already deleted) db.execute( "DELETE FROM posts WHERE thread_id NOT IN (SELECT id FROM threads)" ) - # Delete old threads db.execute("DELETE FROM threads WHERE updated_at < ?", (cutoff,)) - # Clean up orphaned upvotes db.execute( "DELETE FROM upvotes WHERE thread_id NOT IN (SELECT id FROM threads)" ) db.commit() finally: self.return_db(db) + + def get_tag_cloud(self, limit=50): + db = self.get_db() + try: + rows = db.execute("SELECT tags FROM threads").fetchall() + counts = {} + for r in rows: + if r["tags"]: + for t in r["tags"].split(","): + tag = t.strip().lower() + if tag: + counts[tag] = counts.get(tag, 0) + 1 + return sorted(counts.items(), key=lambda x: -x[1])[:limit] + finally: + self.return_db(db) + + def get_threads_by_topics(self, topics, since="", limit=200): + db = self.get_db() + try: + params = [] + where = [] + if topics: + clauses = [] + for t in topics: + clauses.append("t.tags LIKE ?") + params.append(f"%{t}%") + where.append("(" + " OR ".join(clauses) + ")") + if since: + where.append("t.updated_at > ?") + params.append(since) + where_clause = (" WHERE " + " AND ".join(where)) if where else "" + return db.execute( + "SELECT t.*, (SELECT count(*) FROM posts p WHERE p.thread_id = t.id) AS reply_count " + f"FROM threads t{where_clause} ORDER BY t.updated_at DESC LIMIT ?", + params + [limit], + ).fetchall() + finally: + self.return_db(db) + + def get_posts_by_thread_ids(self, thread_ids): + if not thread_ids: + return [] + db = self.get_db() + try: + placeholders = ",".join("?" for _ in thread_ids) + return db.execute( + f"SELECT * FROM posts WHERE thread_id IN ({placeholders}) ORDER BY created_at ASC", + thread_ids, + ).fetchall() + finally: + self.return_db(db) + + def get_new_upvotes_since(self, since, thread_ids=None): + db = self.get_db() + try: + query = ( + "SELECT thread_id FROM upvotes u " + "WHERE NOT EXISTS (SELECT 1 FROM threads t WHERE t.id = u.thread_id AND t.updated_at > ?)" + ) + params = [since] + if thread_ids: + placeholders = ",".join("?" for _ in thread_ids) + query += f" AND u.thread_id IN ({placeholders})" + params.extend(thread_ids) + return [r["thread_id"] for r in db.execute(query, params).fetchall()] + finally: + self.return_db(db) + + # --- Filter Table (Architecture B: Bloom Gossip) --- + + def store_peer_filter(self, peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count): + db = self.get_db() + try: + db.execute( + "INSERT OR REPLACE INTO peer_filters " + "(peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count, last_seen) " + "VALUES (?, ?, ?, ?, ?, ?)", + (peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count, time.time()), + ) + db.commit() + finally: + self.return_db(db) + + def get_peer_filter(self, peer_hash): + db = self.get_db() + try: + row = db.execute( + "SELECT * FROM peer_filters WHERE peer_hash = ?", (peer_hash,) + ).fetchone() + if row: + return dict(row) + return None + finally: + self.return_db(db) + + def get_all_filters(self): + """Return all stored peer filters (for filter table gossip).""" + db = self.get_db() + try: + return [dict(r) for r in db.execute( + "SELECT * FROM peer_filters ORDER BY last_seen DESC" + ).fetchall()] + finally: + self.return_db(db) + + def get_filtered_peers_by_tag(self, tag): + """Return peer hashes whose bloom filter might contain the given tag.""" + tag = tag.strip().lower() + db = self.get_db() + try: + matches = [] + for r in db.execute("SELECT * FROM peer_filters").fetchall(): + bf = BloomFilter.from_bytes(r["bloom_bytes"], r["bloom_size"], r["bloom_hashes"]) + if bf.might_contain(tag): + matches.append(r["peer_hash"]) + return matches + finally: + self.return_db(db) + + def prune_peer_filters(self, max_age_days=7): + db = self.get_db() + try: + cutoff = time.time() - max_age_days * 86400 + db.execute("DELETE FROM peer_filters WHERE last_seen < ?", (cutoff,)) + db.commit() + finally: + self.return_db(db) + + def get_peer_filter_count(self): + db = self.get_db() + try: + return db.execute("SELECT count(*) FROM peer_filters").fetchone()[0] + finally: + self.return_db(db) + + # --- Peer Liveness --- + + def record_sync_result(self, peer_hash, success): + db = self.get_db() + try: + existing = db.execute( + "SELECT consecutive_failures FROM synced_instances WHERE instance_hash = ?", + (peer_hash,), + ).fetchone() + if existing is not None: + new_failures = 0 if success else (existing["consecutive_failures"] + 1) + db.execute( + "UPDATE synced_instances SET consecutive_failures = ? WHERE instance_hash = ?", + (new_failures, peer_hash), + ) + if success: + db.execute( + "UPDATE synced_instances SET status = 'active' WHERE instance_hash = ?", + (peer_hash,), + ) + db.commit() + finally: + self.return_db(db) + + def get_dead_peers(self, max_failures=5): + """Return list of peer hashes with too many consecutive failures.""" + db = self.get_db() + try: + return [ + r["instance_hash"] for r in db.execute( + "SELECT instance_hash FROM synced_instances " + "WHERE consecutive_failures >= ?", + (max_failures,), + ).fetchall() + ] + finally: + self.return_db(db) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 5ff8d3d..4572390 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -16,47 +16,85 @@ def esc(s): return html.escape(str(s)) -FORUM_CSS = """ +from tinyweb_forum.bloom import BloomFilter + + +FORUM_CSS_DEFAULT = """ """ + +FORUM_CSS_KODAMA2 = """ +""" @@ -100,11 +138,16 @@ class ForumHandlers: return "" return f' [block]' + def _forum_css(self): + theme = self.fdb.get_setting("forum_theme", "default") + css = FORUM_CSS_KODAMA2 if theme == "kodama2" else FORUM_CSS_DEFAULT + return css + def _respond(self, body_html, status=200): return { "status": status, "content_type": "text/html; charset=utf-8", - "body": FORUM_CSS + body_html, + "body": self._forum_css() + body_html, "headers": {}, } @@ -125,7 +168,7 @@ class ForumHandlers: } def _error(self, status): - return self._respond(f"

    {status}

    ", status) + return self._respond(f"

    {status}

    ", status) def _paginate(self, query): try: @@ -175,6 +218,14 @@ class ForumHandlers: return False return (datetime.now() - dt).total_seconds() < RECENT_SECONDS + def _get_subscribed_topics(self): + raw = self.fdb.get_setting("topic_subscriptions", "") + return [t.strip().lower() for t in raw.split(",") if t.strip()] + + def _get_subscribed_topics_str(self): + raw = self.fdb.get_setting("topic_subscriptions", "") + return raw + def _blocked_instances(self): raw = self.fdb.get_setting("blocked_instances", "") return set(h.strip() for h in raw.split(",") if h.strip()) @@ -204,6 +255,19 @@ class ForumHandlers: # --- Routes --- + def _status_bar(self): + topics = self._get_subscribed_topics() + topics_str = ", ".join(topics) if topics else "everything" + peer_count = len(self.fdb.get_synced_instances()) + filter_count = self.fdb.get_peer_filter_count() + return ( + f'
    ' + f'subscribed: {esc(topics_str)}' + f'{peer_count} peers' + f'{filter_count} filters' + f'
    ' + ) + def handle_list(self, query): page = self._paginate(query) tag = unquote(query.get("tag", [""])[0]).strip() @@ -224,12 +288,12 @@ class ForumHandlers: continue if self._is_new(r["created_at"]): new_count += 1 - badge = "[share]" if r["url"] else "[request]" - mute_badge = " [muted]" if is_muted else "" + badge = f'share' if r["url"] else f'request' + mute_label = " [muted]" if is_muted else "" tags_html = "" if r["tags"]: tag_links = " ".join( - f'[{esc(t.strip())}]' + f'{esc(t.strip())}' for t in r["tags"].split(",") if t.strip() ) tags_html = f' {tag_links}' @@ -237,7 +301,7 @@ class ForumHandlers: items += ( f'
  • ' f'
    ' - f'{badge}{mute_badge} ' + f'{badge}{mute_label} ' f'{esc(r["title"])}' f'{tags_html}' f'
    ' @@ -250,18 +314,19 @@ class ForumHandlers: f'
    • ' ) if not items: - items = "

    No threads yet.

    " + items = "

    no threads yet.

    " new_label = f" ({new_count} new)" if new_count else "" search_form = ( f'
    ' f'' f'
    ' ) - tag_label = f' — tag: {esc(tag)}' if tag else "" + tag_label = f' — {esc(tag)}' if tag else "" muted_link = f'show muted' if not show_muted else f'show all' page_url = f'/forum?q={esc(search)}&tag={esc(tag)}&muted=1' if show_muted else (f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum') return self._respond( - f"

    forum{tag_label}

    " + f"

    forum{tag_label}

    " + f'{self._status_bar()}' f'
    ' f'{search_form}' f'
    ' @@ -270,26 +335,30 @@ class ForumHandlers: f'sync now' f'{muted_link}' f'
    ' - f"

    {total} threads{new_label}

    " + f"

    {total} threads{new_label}

    " f'
      {items}
    ' f"{self._page_nav(page, total, page_url)}" ) def handle_new_form(self, msg=""): return self._respond( - f"

    new thread

    " + f"

    new thread

    " f'
    ' f'{self._csrf_field()}' + f'' f'' f"max {MAX_TITLE_LENGTH} characters" + f'' f'' + f'' f'' f"max {MAX_BODY_LENGTH} characters" - f'' + f'' + f'' f'' f"
    " f"

    {msg}

    " - f'back' + f'
    back
    ' ) def handle_new_submit(self, body): @@ -323,28 +392,25 @@ class ForumHandlers: instance_hash = self.identity.hash.hex() if self.identity else "local" has_upvoted = self.fdb.has_upvoted(thread_id, instance_hash) - badge = "[share]" if thread["url"] else "[request]" + badge = f'share' if thread["url"] else f'request' url_html = "" if thread["url"]: url_html = ( f'

    {esc(thread["url"])}' - f' (+ save to my index)

    ' + f' (+ save)

    ' ) tags_html = "" if thread["tags"]: tag_links = " ".join( - f'[{esc(t.strip())}]' + f'{esc(t.strip())}' for t in thread["tags"].split(",") if t.strip() ) - tags_html = f'

    {tag_links}

    ' + tags_html = f'

    {tag_links}

    ' body_html = f"

    {esc(thread['body'])}

    " if thread["body"] else "" - mute_btn = ( - f'unmute' - if is_muted else - f'mute' - ) + mute_label = "unmute" if is_muted else "mute" + mute_href = f'/forum/unmute/{thread["id"]}' if is_muted else f'/forum/mute/{thread["id"]}' posts_html = "" for p in posts: @@ -352,18 +418,18 @@ class ForumHandlers: for word in p["body"].split(): w = word.strip().strip(",.!?;:") if w.startswith(("http://", "https://")): - save_links += ( - f' + save' - ) + save_links += f' + save' parent_ref = "" if p["parent_id"]: - parent_ref = f' ↪ reply' + parent_ref = f' ↪ reply' posts_html += ( - f'
    ' - f'{esc(self._author_str(p["author_name"], p["author_instance"]))}' + f'
    ' + f'
    ' + f'{esc(self._author_str(p["author_name"], p["author_instance"]))}' f'{self._block_link(p["author_instance"])}' f' · {self._time_ago(p["created_at"])}{parent_ref}' - f'{" · " + self._post_retract_link(thread["id"], p["id"]) if p["author_instance"] == instance_hash else ""}' + f'{" · " + self._post_retract_link(thread["id"], p["id"]) if p["author_instance"] == instance_hash else ""}' + f'
    ' f'

    {esc(p["body"])}

    ' f'{save_links}' f'
    ' @@ -378,15 +444,16 @@ class ForumHandlers: f"" ) + upvote_label = "-1" if has_upvoted else "+1" return self._respond( - f"

    {badge} {esc(thread['title'])}

    " - f'

    ' + f"

    {badge} {esc(thread['title'])}

    " + f'

    ' f'by {esc(self._author_str(thread["author_name"], thread["author_instance"]))}' f'{self._block_link(thread["author_instance"])}' f' · {self._time_ago(thread["created_at"])}' f' · {thread["score"]} upvotes' - f' · {mute_btn}' - f' · {"-1" if has_upvoted else "+1"}' + f' · {mute_label}' + f' · {upvote_label}' f'{self._author_links(thread["id"], thread["author_instance"], instance_hash)}' f'

    ' f'{url_html}' @@ -428,7 +495,7 @@ class ForumHandlers: if thread["author_instance"] != instance_hash: return self._error(403) return self._respond( - f"

    edit thread

    " + f"

    edit thread

    " f'
    ' f'{self._csrf_field()}' f'' @@ -511,7 +578,7 @@ class ForumHandlers: def _peer_reports_html(self): counts = self.fdb.get_peer_block_counts() if not counts: - return "

    No peer reports yet.

    " + return "

    no peer reports yet

    " auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) blocked = self._blocked_instances() items = "" @@ -524,13 +591,24 @@ class ForumHandlers: blocked = self._blocked_instances() auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) peer_counts = self.fdb.get_peer_block_counts() + filters = self._keyword_filters() + filters_str = ", ".join(filters) if filters else "" + synced = self.fdb.get_synced_instances() + auto_discover = self.fdb.get_setting("forum_auto_discover", "1") + auto_discover_checked = " checked" if auto_discover == "1" else "" + auto_sync = self.fdb.get_setting("forum_auto_sync", "0") + auto_sync_checked = " checked" if auto_sync == "1" else "" + retention_days = self.fdb.get_setting("forum_retention_days", "30") + blocked_items = "" if blocked: for h in sorted(blocked): - label = "[auto] " if h in auto_blocked else "" - reports = f" ({peer_counts.get(h, 0)} peers)" if h in peer_counts else "" + label = "auto" if h in auto_blocked else "" + reports = f" ({peer_counts.get(h, 0)} reports)" if h in peer_counts else "" blocked_items += ( - f'
  • {label}{esc(h[:16])}...{reports} ' + f'
  • ' + f'{esc(h[:16])}...' + f'{" [" + label + "]" if label else ""}{reports} ' f'' f'{self._csrf_field()}' f'' @@ -539,12 +617,8 @@ class ForumHandlers: ) blocked_items = f"
      {blocked_items}
    " else: - blocked_items = "

    No instances blocked.

    " + blocked_items = "

    no instances blocked

    " - filters = self._keyword_filters() - filters_str = ", ".join(filters) if filters else "" - - synced = self.fdb.get_synced_instances() synced_items = "" for s in synced: synced_items += ( @@ -555,67 +629,78 @@ class ForumHandlers: f'
    • ' f'' ) - synced_items = f"
      {synced_items}
    " if synced_items else "

    No instances synced yet.

    " - - auto_discover = self.fdb.get_setting("forum_auto_discover", "1") - auto_discover_checked = " checked" if auto_discover == "1" else "" - auto_sync = self.fdb.get_setting("forum_auto_sync", "0") - auto_sync_checked = " checked" if auto_sync == "1" else "" - retention_days = self.fdb.get_setting("forum_retention_days", "30") + synced_items = f"
      {synced_items}
    " if synced_items else "

    no instances synced

    " return self._respond( - f"

    forum moderation

    " + f"

    moderation

    " f"

    {msg}

    " - f'

    sync now

    ' - f"

    auto-discovery

    " + f'{self._status_bar()}' + + f'
    ' + f'
    subscriptions
    ' + f'
    ' + f'{self._csrf_field()}' + f'' + f"only sync content matching these topics" + f'' + f"
    " + f'
    ' + + f'
    ' + f'
    settings
    ' + f'
    network behavior
    ' f'
    ' f'{self._csrf_field()}' f'" + f" auto-discover peers via announces" f'' f"
    " - f"

    auto-sync

    " f'
    ' f'{self._csrf_field()}' f'" + f" auto-sync every 5 minutes" f'' f"
    " - f"

    storage

    " f'
    ' f'{self._csrf_field()}' - f'' - f"Older threads are pruned automatically (default: 30). Set to 0 to keep everything." + f'' f'' f"
    " - f"

    blocked instances

    " - f"{blocked_items}" - f'
    ' - f'{self._csrf_field()}' - f'' - f'' - f"
    " - f"

    peer reports

    " - f"{self._peer_reports_html()}" - f"

    keyword filters

    " - f'
    ' - f'{self._csrf_field()}' - f'' - f'' - f"
    " - f"

    synced instances

    " - f"{synced_items}" - f"

    Instances are discovered automatically via mesh announces. " - f"You can also manually add a friend's instance hash to bootstrap.

    " + f'
    ' + + f'
    ' + f'
    network
    ' + f'
    {len(synced)} known peers
    ' + f'{synced_items}' f'
    ' f'{self._csrf_field()}' f'' f'' f'' f"
    " - f'
    ' - f'back to forum' + f'
    ' + + f'
    ' + f'
    moderation
    ' + f'{blocked_items}' + f'
    ' + f'{self._csrf_field()}' + f'' + f'' + f"
    " + f'
    peer reports
    ' + f'{self._peer_reports_html()}' + f'
    keyword filters
    ' + f'
    ' + f'{self._csrf_field()}' + f'' + f'' + f"
    " + f'
    ' + + f'
    back
    ' ) def handle_block(self, body): @@ -702,31 +787,122 @@ class ForumHandlers: self.fdb.remove_synced_instance(instance) return self.handle_moderation("Removed.") + def handle_topics(self, body): + topics = body.get("topics", [""])[0].strip() + self.fdb.set_setting("topic_subscriptions", topics) + return self.handle_moderation("Topic subscriptions saved.") + # --- Sync endpoint (called over RNS) --- def handle_sync_request(self, data): - """Handle incoming sync request from another forum instance.""" since = data.get("query", {}).get("since", [""])[0] if isinstance(data.get("query"), dict) else "" incoming_threads = data.get("threads", []) incoming_posts = data.get("posts", []) incoming_upvotes = data.get("upvotes", []) + peer_topics = data.get("my_topics", []) + peer_tag_cloud = data.get("my_tag_cloud", []) + peer_bloom_data = data.get("content_bloom") + peer_tag_bloom_data = data.get("tag_bloom") + peer_filter_table = data.get("filter_table", {}) + scoped_query_tag = data.get("scoped_query", "") blocked = self._blocked_instances() + my_topics = self._get_subscribed_topics() + from_hash = data.get("from_hash", "") + + # Store peer's tag bloom filter (Architecture B discovery) + if peer_tag_bloom_data and from_hash: + self.fdb.store_peer_filter( + peer_hash=from_hash, + bloom_bytes=bytes(peer_tag_bloom_data), + bloom_size=data.get("tag_bloom_size", 2048), + bloom_hashes=data.get("tag_bloom_hashes", 3), + tag_count=len(peer_topics), + ) + + # Merge filter table gossip (transitive peer discovery) + if peer_filter_table and from_hash: + for ph, entry in peer_filter_table.items(): + if isinstance(entry, dict): + bb = entry.get("bloom_bytes") + if bb: + self.fdb.store_peer_filter( + peer_hash=ph, + bloom_bytes=bytes(bb) if isinstance(bb, list) else bb, + bloom_size=entry.get("bloom_size", 2048), + bloom_hashes=entry.get("bloom_hashes", 3), + tag_count=entry.get("tag_count", 0), + ) + + # Handle scoped query: find peers whose bloom might contain the queried tag + scoped_query_results = [] + if scoped_query_tag and from_hash: + scoped_query_results = self.fdb.get_filtered_peers_by_tag(scoped_query_tag) + + # Store peer's topics for future routing (backward compat) + if peer_topics and from_hash: + self.fdb.set_setting(f"peer_topics_{from_hash}", ",".join(peer_topics)) + if peer_tag_cloud and from_hash: + self.fdb.set_setting(f"peer_tag_cloud_{from_hash}", json.dumps(peer_tag_cloud)) + + # Build our tag bloom filter to send back + peer_tag_bs = data.get("tag_bloom_size", 2048) + peer_tag_bh = data.get("tag_bloom_hashes", 3) + my_tag_bloom = BloomFilter.from_tags(my_topics or [], peer_tag_bs, peer_tag_bh) + + # Build our content bloom filter for dedup + our_existing = set() + for t in self.fdb.get_threads_by_topics(peer_topics) if peer_topics else []: + our_existing.add(t["id"]) + our_bloom = BloomFilter.from_items(list(our_existing), data.get("bloom_size", 2048) if data else 2048, data.get("bloom_hashes", 3) if data else 3) + + # Build filter table gossip from our stored filters + all_filters = self.fdb.get_all_filters() + filter_table_gossip = {} + for f in all_filters[:20]: + if f["peer_hash"] != from_hash: + filter_table_gossip[f["peer_hash"]] = { + "bloom_bytes": list(f["bloom_bytes"]), + "bloom_size": f["bloom_size"], + "bloom_hashes": f["bloom_hashes"], + "tag_count": f["tag_count"], + } + + # Parse peer's content bloom for dedup + peer_bloom = None + if peer_bloom_data: + peer_bloom = BloomFilter.from_bytes( + bytes(peer_bloom_data), + data.get("bloom_size", 2048), + data.get("bloom_hashes", 3), + ) + + # Merge incoming content, filtered by bloom and topics if incoming_threads: for t in incoming_threads: - if t.get("author_instance", "") not in blocked: + if t.get("author_instance", "") in blocked: + continue + if peer_bloom and peer_bloom.might_contain(t["id"]): + continue + if not my_topics: self.fdb.merge_thread(t) + else: + t_tags = [tag.strip().lower() for tag in t.get("tags", "").split(",") if tag.strip()] + if set(my_topics) & set(t_tags): + self.fdb.merge_thread(t) if incoming_posts: for p in incoming_posts: - if p.get("author_instance", "") not in blocked: - self.fdb.merge_post(p) + if p.get("author_instance", "") in blocked: + continue + if peer_bloom and peer_bloom.might_contain(p["id"]): + continue + self.fdb.merge_post(p) if incoming_upvotes: for uv in incoming_upvotes: self.fdb.merge_upvote(uv["thread_id"], uv["instance_hash"]) - # Record incoming peer blocks incoming_blocks = data.get("blocks", {}) - peer_hash = data.get("peer_hash", "") or data.get("from_hash", "") + peer_hash = data.get("peer_hash", "") or from_hash if incoming_blocks and peer_hash: for h in incoming_blocks.get("mine", []): if h and h not in blocked: @@ -735,13 +911,10 @@ class ForumHandlers: if h and h not in blocked: self.fdb.record_peer_block(peer_hash, h) - # Merge incoming retractions for r in data.get("retractions", []): if r.get("id") and r.get("type") and r.get("author") and r.get("at"): self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) - # Auto-discover the peer that synced with us and their known peers - from_hash = data.get("from_hash", "") if from_hash and from_hash not in blocked: self.fdb.add_known_peer(from_hash) for peer_hash in data.get("known_peers", []): @@ -750,18 +923,34 @@ class ForumHandlers: my_blocks = list(blocked) my_peer_blocks = self.fdb.get_peer_block_list() + my_tag_cloud = self.fdb.get_tag_cloud(50) + my_tag_list = [t for t, _ in my_tag_cloud] + threads, posts, upvote_threads = [], [], [] if since: - ts, posts_list, up_list = self.fdb.get_new_content(since) - threads = [dict(r) for r in ts] - posts = [dict(r) for r in posts_list] - upvote_threads = up_list + if peer_topics: + rows = self.fdb.get_threads_by_topics(peer_topics, since=since) + threads = [dict(r) for r in rows] + tids = [r["id"] for r in rows] + posts = [dict(p) for p in self.fdb.get_posts_by_thread_ids(tids)] + uv_rows = self.fdb.get_new_upvotes_since(since, tids) + upvote_threads = uv_rows + else: + ts, posts_list, up_list = self.fdb.get_new_content(since) + threads = [dict(r) for r in ts] + posts = [dict(r) for r in posts_list] + upvote_threads = up_list + + known_peers = [h for h in self.fdb.get_all_known_hashes() if h != from_hash] + peer_topics_map = {} + for ph in known_peers[:100]: + pt = self.fdb.get_setting(f"peer_topics_{ph}", "") + if pt: + peer_topics_map[ph] = [t.strip() for t in pt.split(",") if t.strip()] retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} for cid, ct, ai, ra in self.fdb.get_raw_retractions()] - known_peers = [h for h in self.fdb.get_all_known_hashes() if h != from_hash] - return { "status": 200, "content_type": "application/json", @@ -772,6 +961,18 @@ class ForumHandlers: "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, "retractions": retracted, "known_peers": known_peers, + "peer_topics": my_tag_list, + "peer_tag_cloud": my_tag_cloud, + "content_bloom": list(our_bloom.bytes), + "bloom_size": data.get("bloom_size", 2048), + "bloom_hashes": data.get("bloom_hashes", 3), + "peer_topics_map": peer_topics_map, + # Architecture B additions + "tag_bloom": list(my_tag_bloom.bytes), + "tag_bloom_size": peer_tag_bs, + "tag_bloom_hashes": peer_tag_bh, + "filter_table": filter_table_gossip, + "scoped_query_results": scoped_query_results, }), "headers": {}, } @@ -832,7 +1033,7 @@ class ForumHandlers: elif method == "POST": if not self._check_csrf(body): return self._with_csrf( - self._respond("

    403 Forbidden

    ", status=403), csrf_token + self._respond("

    403 Forbidden

    ", status=403), csrf_token ) if sub == "/new": return self._with_csrf(self.handle_new_submit(body), csrf_token) @@ -867,6 +1068,8 @@ class ForumHandlers: return self._with_csrf(self.handle_storage(body), csrf_token) elif sub == "/auto_sync": return self._with_csrf(self.handle_auto_sync(body), csrf_token) + elif sub == "/topics": + return self._with_csrf(self.handle_topics(body), csrf_token) return self._with_csrf(self._error(404), csrf_token) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 99e1985..a9d544d 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -4,15 +4,22 @@ import threading import time import RNS +from tinyweb_forum.bloom import BloomFilter + FORUM_APP = "tinyweb-forum" -SYNC_INTERVAL = 300 # 5 minutes +SYNC_INTERVAL = 300 REQUEST_TIMEOUT = 60 -GOSSIP_FANOUT = 20 # random peers to sync per cycle +GOSSIP_FANOUT = 20 +BLOOM_SIZE = 2048 +BLOOM_HASHES = 3 +TAG_BLOOM_SIZE = 2048 +TAG_BLOOM_HASHES = 3 +FILTER_TABLE_GOSSIP = 20 +MAX_PEER_FAILURES = 5 +FILTER_TABLE_TTL_DAYS = 7 class _ForumAnnounceHandler: - """Receives announces from other forum instances and auto-discovers them.""" - aspect_filter = FORUM_APP receive_path_responses = False @@ -39,6 +46,26 @@ class ForumSync: self._running = False self._thread = None + def _get_subscribed_topics(self): + raw = self.fdb.get_setting("topic_subscriptions", "") + return [t.strip().lower() for t in raw.split(",") if t.strip()] + + def _build_tag_bloom(self, topics=None): + if topics is None: + topics = self._get_subscribed_topics() + return BloomFilter.from_tags(topics or [], TAG_BLOOM_SIZE, TAG_BLOOM_HASHES) + + def _topics_overlap(self, my_topics, their_topics): + if not my_topics or not their_topics: + return True + return bool(set(my_topics) & set(their_topics)) + + def _topics_overlap_bloom(self, my_topics, peer_bloom): + """Check overlap using bloom filter instead of topic list.""" + if not my_topics or peer_bloom is None: + return True + return any(peer_bloom.might_contain(t) for t in my_topics) + def start(self): self.destination = RNS.Destination( self.identity, @@ -66,23 +93,48 @@ class ForumSync: self.fdb.set_setting("forum_auto_sync", "1" if enabled else "0") if enabled: self._start_sync_loop() - else: - pass # current cycle finishes, no new one starts def sync_now(self): - """Run one sync cycle immediately. Returns count of peers synced.""" instances = self.fdb.get_synced_instances() random.shuffle(instances) count = 0 + my_topics = self._get_subscribed_topics() + my_tag_bloom = self._build_tag_bloom(my_topics) + for inst in instances[:GOSSIP_FANOUT]: + if not self._running: + break + pfilter = self.fdb.get_peer_filter(inst["instance_hash"]) + if pfilter: + pbloom = BloomFilter.from_bytes(pfilter["bloom_bytes"], pfilter["bloom_size"], pfilter["bloom_hashes"]) + if not self._topics_overlap_bloom(my_topics, pbloom): + continue + else: + peer_topics = self._peer_tag_topics(inst["instance_hash"]) + if not self._topics_overlap(my_topics, peer_topics): + continue + try: + self._sync_with(inst["instance_hash"], my_topics, my_tag_bloom) + count += 1 + except Exception as e: + self.fdb.record_sync_result(inst["instance_hash"], False) + print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") + return count + + def scoped_query(self, tag): + """Ask known peers if they know anyone with the given tag.""" + tag = tag.strip().lower() + instances = self.fdb.get_synced_instances() + random.shuffle(instances) + results = set() for inst in instances[:GOSSIP_FANOUT]: if not self._running: break try: - self._sync_with(inst["instance_hash"]) - count += 1 - except Exception as e: - print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") - return count + matches = self._scoped_query_peer(inst["instance_hash"], tag) + results.update(matches) + except Exception: + continue + return list(results) def _start_sync_loop(self): if self._thread and self._thread.is_alive(): @@ -120,16 +172,25 @@ class ForumSync: while self._running: try: instances = self.fdb.get_synced_instances() - # Gossip: sync with random subset for scaling - # If <= GOSSIP_FANOUT peers, sync with all (current behavior) - # If more, sync with random FANOUT per cycle — content spreads epidemically + my_topics = self._get_subscribed_topics() + my_tag_bloom = self._build_tag_bloom(my_topics) random.shuffle(instances) for inst in instances[:GOSSIP_FANOUT]: if not self._running: break + pfilter = self.fdb.get_peer_filter(inst["instance_hash"]) + if pfilter: + pbloom = BloomFilter.from_bytes(pfilter["bloom_bytes"], pfilter["bloom_size"], pfilter["bloom_hashes"]) + if not self._topics_overlap_bloom(my_topics, pbloom): + continue + else: + peer_topics = self._peer_tag_topics(inst["instance_hash"]) + if not self._topics_overlap(my_topics, peer_topics): + continue try: - self._sync_with(inst["instance_hash"]) + self._sync_with(inst["instance_hash"], my_topics, my_tag_bloom) except Exception as e: + self.fdb.record_sync_result(inst["instance_hash"], False) print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") except Exception: pass @@ -141,12 +202,99 @@ class ForumSync: self.fdb.prune_old_content(days) except Exception: pass + try: + self.fdb.prune_peer_filters(FILTER_TABLE_TTL_DAYS) + except Exception: + pass + try: + for dead in self.fdb.get_dead_peers(MAX_PEER_FAILURES): + self.fdb.remove_synced_instance(dead) + except Exception: + pass for _ in range(SYNC_INTERVAL): if not self._running: return time.sleep(1) - def _sync_with(self, instance_hash): + def _peer_tag_topics(self, instance_hash): + raw = self.fdb.get_setting(f"peer_topics_{instance_hash}", "") + return [t.strip().lower() for t in raw.split(",") if t.strip()] if raw else [] + + def _store_peer_filter(self, peer_hash, data): + tag_bloom_data = data.get("tag_bloom") + if tag_bloom_data: + self.fdb.store_peer_filter( + peer_hash=peer_hash, + bloom_bytes=bytes(tag_bloom_data), + bloom_size=data.get("tag_bloom_size", TAG_BLOOM_SIZE), + bloom_hashes=data.get("tag_bloom_hashes", TAG_BLOOM_HASHES), + tag_count=len(data.get("peer_topics", [])), + ) + + def _merge_filter_table_gossip(self, filter_table): + if not filter_table: + return + for ph, entry in filter_table.items(): + if isinstance(entry, dict): + bb = entry.get("bloom_bytes") + if bb: + self.fdb.store_peer_filter( + peer_hash=ph, + bloom_bytes=bytes(bb) if isinstance(bb, list) else bb, + bloom_size=entry.get("bloom_size", TAG_BLOOM_SIZE), + bloom_hashes=entry.get("bloom_hashes", TAG_BLOOM_HASHES), + tag_count=entry.get("tag_count", 0), + ) + + def _scoped_query_peer(self, peer_hash, tag): + dest_hash = bytes.fromhex(peer_hash) + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + return [] + server_identity = RNS.Identity.recall(dest_hash) + if server_identity is None: + return [] + destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + FORUM_APP, + ) + link = RNS.Link(destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + if link.status != RNS.Link.ACTIVE: + return [] + try: + my_hash = self.identity.hash.hex() if self.identity else "local" + request_data = { + "scoped_query": tag, + "from_hash": my_hash, + } + receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) + elapsed = 0 + done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + if isinstance(resp, dict) and resp.get("status") == 200: + data = json.loads(resp["body"]) + return data.get("scoped_query_results", []) + return [] + finally: + link.teardown() + + def _sync_with(self, instance_hash, my_topics=None, my_tag_bloom=None): + success = False dest_hash = bytes.fromhex(instance_hash) if not RNS.Transport.has_path(dest_hash): RNS.Transport.request_path(dest_hash) @@ -155,10 +303,12 @@ class ForumSync: time.sleep(0.5) elapsed += 0.5 if not RNS.Transport.has_path(dest_hash): + self.fdb.record_sync_result(instance_hash, False) return server_identity = RNS.Identity.recall(dest_hash) if server_identity is None: + self.fdb.record_sync_result(instance_hash, False) return destination = RNS.Destination( @@ -175,6 +325,7 @@ class ForumSync: elapsed += 0.25 if link.status != RNS.Link.ACTIVE: + self.fdb.record_sync_result(instance_hash, False) return try: @@ -186,14 +337,50 @@ class ForumSync: break since = last_sync.replace(" ", "T") if last_sync else "" + my_hash = self.identity.hash.hex() if self.identity else "local" + + if my_topics is None: + my_topics = self._get_subscribed_topics() + if my_tag_bloom is None: + my_tag_bloom = self._build_tag_bloom(my_topics) + + # Build filter table gossip: share a subset of our known peer filters + all_filters = self.fdb.get_all_filters() + filter_table_gossip = {} + for f in all_filters[:FILTER_TABLE_GOSSIP]: + if f["peer_hash"] != instance_hash and f["peer_hash"] != my_hash: + filter_table_gossip[f["peer_hash"]] = { + "bloom_bytes": list(f["bloom_bytes"]), + "bloom_size": f["bloom_size"], + "bloom_hashes": f["bloom_hashes"], + "tag_count": f["tag_count"], + } threads, posts = [], [] upvotes = [] + my_tag_cloud = self.fdb.get_tag_cloud(50) + my_tag_list = [t for t, _ in my_tag_cloud] + + # Only build content if we have a since timestamp (incremental sync) if since: - ts, ps, uv = self.fdb.get_new_content(since) - threads = [dict(r) for r in ts] - posts = [dict(r) for r in ps] - upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv] + if my_topics: + rows = self.fdb.get_threads_by_topics(my_topics, since=since) + threads = [dict(r) for r in rows] + tids = [r["id"] for r in rows] + posts = [dict(p) for p in self.fdb.get_posts_by_thread_ids(tids)] + uv_rows = self.fdb.get_new_upvotes_since(since, tids) + upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv_rows] + else: + ts, ps, uv = self.fdb.get_new_content(since) + threads = [dict(r) for r in ts] + posts = [dict(r) for r in ps] + upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv] + + # Build content bloom for dedup + existing_ids = set(t["id"] for t in threads) + for t in self.fdb.get_threads_by_topics(my_topics) if my_topics else []: + existing_ids.add(t["id"]) + content_bloom = BloomFilter.from_items(list(existing_ids), BLOOM_SIZE, BLOOM_HASHES) my_blocks = [h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()] my_peer_blocks = self.fdb.get_peer_block_list() @@ -201,8 +388,12 @@ class ForumSync: retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} for cid, ct, ai, ra in self.fdb.get_raw_retractions()] - my_hash = self.identity.hash.hex() if self.identity else "local" known_peers = [h for h in self.fdb.get_all_known_hashes() if h != instance_hash and h != my_hash] + peer_topics_map = {} + for ph in known_peers[:100]: + pt = self.fdb.get_setting(f"peer_topics_{ph}", "") + if pt: + peer_topics_map[ph] = [t.strip() for t in pt.split(",") if t.strip()] request_data = { "query": {"since": [since]} if since else {}, @@ -213,6 +404,17 @@ class ForumSync: "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, "retractions": retracted, "known_peers": known_peers, + "my_topics": my_topics, + "my_tag_cloud": my_tag_cloud, + "content_bloom": list(content_bloom.bytes), + "bloom_size": BLOOM_SIZE, + "bloom_hashes": BLOOM_HASHES, + "peer_topics_map": peer_topics_map, + # Architecture B additions + "tag_bloom": list(my_tag_bloom.bytes), + "tag_bloom_size": TAG_BLOOM_SIZE, + "tag_bloom_hashes": TAG_BLOOM_HASHES, + "filter_table": filter_table_gossip, } receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) @@ -229,16 +431,82 @@ class ForumSync: data = json.loads(resp["body"]) except (json.JSONDecodeError, KeyError): data = {} + success = True + + # Store peer's tag bloom filter + self._store_peer_filter(instance_hash, data) + + # Merge filter table gossip from peer + self._merge_filter_table_gossip(data.get("filter_table", {})) + + # Store peer topics (backward compat) + peer_topics = data.get("peer_topics", []) + if peer_topics: + self.fdb.set_setting(f"peer_topics_{instance_hash}", ",".join(peer_topics)) + peer_tag_cloud = data.get("peer_tag_cloud", []) + if peer_tag_cloud: + self.fdb.set_setting(f"peer_tag_cloud_{instance_hash}", json.dumps(peer_tag_cloud)) + + # Check tag overlap using bloom filter my_blocks = set(h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()) + + peer_tag_bloom_data = data.get("tag_bloom") + peer_tag_bloom = None + if peer_tag_bloom_data: + peer_tag_bloom = BloomFilter.from_bytes( + bytes(peer_tag_bloom_data), + data.get("tag_bloom_size", TAG_BLOOM_SIZE), + data.get("tag_bloom_hashes", TAG_BLOOM_HASHES), + ) + + # If no tag overlap, skip content sync but still store the filter + if my_topics and peer_tag_bloom is not None: + if not self._topics_overlap_bloom(my_topics, peer_tag_bloom): + success = True + now = time.strftime("%Y-%m-%dT%H:%M:%S") + self.fdb.set_last_sync(instance_hash, now) + self.fdb.record_sync_result(instance_hash, True) + return + + # Fall back to topic list overlap check if no bloom + if peer_tag_bloom is None: + incoming_topics = data.get("peer_topics", []) + if my_topics and incoming_topics: + if not self._topics_overlap(my_topics, incoming_topics): + success = True + now = time.strftime("%Y-%m-%dT%H:%M:%S") + self.fdb.set_last_sync(instance_hash, now) + self.fdb.record_sync_result(instance_hash, True) + return + + # Check content bloom for dedup + peer_bloom_data = data.get("content_bloom") + peer_bloom = None + if peer_bloom_data: + peer_bloom = BloomFilter.from_bytes( + bytes(peer_bloom_data), + data.get("bloom_size", BLOOM_SIZE), + data.get("bloom_hashes", BLOOM_HASHES), + ) + for t in data.get("threads", []): if t.get("author_instance", "") not in my_blocks: - self.fdb.merge_thread(t) + if peer_bloom and peer_bloom.might_contain(t["id"]): + continue + if not my_topics: + self.fdb.merge_thread(t) + else: + t_tags = [tag.strip().lower() for tag in t.get("tags", "").split(",") if tag.strip()] + if set(my_topics) & set(t_tags): + self.fdb.merge_thread(t) for p in data.get("posts", []): if p.get("author_instance", "") not in my_blocks: + if peer_bloom and peer_bloom.might_contain(p["id"]): + continue self.fdb.merge_post(p) for tid in data.get("upvote_threads", []): self.fdb.merge_upvote(tid, instance_hash) - # Gossip blocks from peer + peer_blocks = data.get("blocks", {}) for h in peer_blocks.get("mine", []): if h and h not in my_blocks and instance_hash: @@ -247,18 +515,25 @@ class ForumSync: if h and h not in my_blocks and instance_hash: self.fdb.record_peer_block(instance_hash, h) self._apply_peer_blocks() - # Merge incoming retractions + for r in data.get("retractions", []): if r.get("id") and r.get("type") and r.get("author") and r.get("at"): self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) - # Discover new peers from gossip + for peer_hash in data.get("known_peers", []): if peer_hash and peer_hash != my_hash and peer_hash != instance_hash: self.fdb.add_known_peer(peer_hash) + + my_topics_set = set(my_topics) + for ph, pt in data.get("peer_topics_map", {}).items(): + if ph and ph != my_hash and ph != instance_hash: + pt_set = set(t.lower() for t in pt) + if not my_topics or my_topics_set & pt_set: + self.fdb.add_known_peer(ph) + now = time.strftime("%Y-%m-%dT%H:%M:%S") self.fdb.set_last_sync(instance_hash, now) - else: - pass + self.fdb.record_sync_result(instance_hash, success) finally: link.teardown() From 14a885d3147301fc59280dfa2b3b05776959d760 Mon Sep 17 00:00:00 2001 From: user Date: Sat, 6 Jun 2026 01:19:40 +0000 Subject: [PATCH 52/56] Architecture B: Bloom Gossip + Implicit Replication - Tag bloom filter as primary peer discovery (2048 bits x 3 hashes) - Filter table gossip for transitive peer discovery - Scoped queries for on-demand tag lookup - Liveness tracking + peer eviction (5 failures, 7-day TTL) - Topic subscriptions filter content sync at both ends - Two-theme CSS system (default minimal + kodama2) - Status bar on all pages (topics, peers, filters) - Bracketless tags, grouped moderation page, cleaner forms - Match main site heading level (h1 -> h2) --- tinyweb_forum/bloom.py | 48 ++++ tinyweb_forum/db.py | 193 +++++++++++++++- tinyweb_forum/handlers.py | 467 +++++++++++++++++++++++++++----------- tinyweb_forum/sync.py | 331 ++++++++++++++++++++++++--- 4 files changed, 875 insertions(+), 164 deletions(-) create mode 100644 tinyweb_forum/bloom.py diff --git a/tinyweb_forum/bloom.py b/tinyweb_forum/bloom.py new file mode 100644 index 0000000..a481e01 --- /dev/null +++ b/tinyweb_forum/bloom.py @@ -0,0 +1,48 @@ +import hashlib + + +class BloomFilter: + def __init__(self, size=2048, num_hashes=3): + self.size = size + self.num_hashes = num_hashes + self.bits = bytearray(size // 8 + 1) + + def _hash_positions(self, item): + h = hashlib.sha256(item.encode("utf-8")).digest() + for i in range(self.num_hashes): + val = int.from_bytes(h[i*4:(i+1)*4], "big") % self.size + yield val + + def add(self, item): + for pos in self._hash_positions(item): + self.bits[pos // 8] |= 1 << (pos % 8) + + def might_contain(self, item): + return all( + bool(self.bits[pos // 8] & (1 << (pos % 8))) + for pos in self._hash_positions(item) + ) + + @property + def bytes(self): + return bytes(self.bits) + + @classmethod + def from_bytes(cls, data, size=2048, num_hashes=3): + bf = cls(size=size, num_hashes=num_hashes) + bf.bits = bytearray(data) + return bf + + @staticmethod + def from_items(items, size=2048, num_hashes=3): + bf = BloomFilter(size=size, num_hashes=num_hashes) + for item in items: + bf.add(item) + return bf + + @staticmethod + def from_tags(tags, size=2048, num_hashes=3): + bf = BloomFilter(size=size, num_hashes=num_hashes) + for tag in tags: + bf.add(tag.strip().lower()) + return bf diff --git a/tinyweb_forum/db.py b/tinyweb_forum/db.py index 8a77715..8f396d7 100644 --- a/tinyweb_forum/db.py +++ b/tinyweb_forum/db.py @@ -1,8 +1,11 @@ import sqlite3 import os import threading +import time from datetime import datetime, timedelta +from tinyweb_forum.bloom import BloomFilter + FORUM_DB = "forum.db" @@ -88,6 +91,20 @@ class ForumDB: " PRIMARY KEY (content_id, content_type)" ")" ) + db.execute( + "CREATE TABLE IF NOT EXISTS peer_filters (" + " peer_hash TEXT PRIMARY KEY," + " bloom_bytes BLOB," + " bloom_size INTEGER DEFAULT 2048," + " bloom_hashes INTEGER DEFAULT 3," + " tag_count INTEGER DEFAULT 0," + " last_seen REAL NOT NULL" + ")" + ) + try: + db.execute("ALTER TABLE synced_instances ADD COLUMN consecutive_failures INTEGER DEFAULT 0") + except Exception: + pass db.commit() db.close() @@ -509,22 +526,190 @@ class ForumDB: db = self.get_db() try: cutoff = (datetime.utcnow() - timedelta(days=retention_days)).strftime("%Y-%m-%dT%H:%M:%S") - # Delete posts in old threads db.execute( "DELETE FROM posts WHERE thread_id IN " "(SELECT id FROM threads WHERE updated_at < ?)", (cutoff,), ) - # Delete orphaned posts (thread already deleted) db.execute( "DELETE FROM posts WHERE thread_id NOT IN (SELECT id FROM threads)" ) - # Delete old threads db.execute("DELETE FROM threads WHERE updated_at < ?", (cutoff,)) - # Clean up orphaned upvotes db.execute( "DELETE FROM upvotes WHERE thread_id NOT IN (SELECT id FROM threads)" ) db.commit() finally: self.return_db(db) + + def get_tag_cloud(self, limit=50): + db = self.get_db() + try: + rows = db.execute("SELECT tags FROM threads").fetchall() + counts = {} + for r in rows: + if r["tags"]: + for t in r["tags"].split(","): + tag = t.strip().lower() + if tag: + counts[tag] = counts.get(tag, 0) + 1 + return sorted(counts.items(), key=lambda x: -x[1])[:limit] + finally: + self.return_db(db) + + def get_threads_by_topics(self, topics, since="", limit=200): + db = self.get_db() + try: + params = [] + where = [] + if topics: + clauses = [] + for t in topics: + clauses.append("t.tags LIKE ?") + params.append(f"%{t}%") + where.append("(" + " OR ".join(clauses) + ")") + if since: + where.append("t.updated_at > ?") + params.append(since) + where_clause = (" WHERE " + " AND ".join(where)) if where else "" + return db.execute( + "SELECT t.*, (SELECT count(*) FROM posts p WHERE p.thread_id = t.id) AS reply_count " + f"FROM threads t{where_clause} ORDER BY t.updated_at DESC LIMIT ?", + params + [limit], + ).fetchall() + finally: + self.return_db(db) + + def get_posts_by_thread_ids(self, thread_ids): + if not thread_ids: + return [] + db = self.get_db() + try: + placeholders = ",".join("?" for _ in thread_ids) + return db.execute( + f"SELECT * FROM posts WHERE thread_id IN ({placeholders}) ORDER BY created_at ASC", + thread_ids, + ).fetchall() + finally: + self.return_db(db) + + def get_new_upvotes_since(self, since, thread_ids=None): + db = self.get_db() + try: + query = ( + "SELECT thread_id FROM upvotes u " + "WHERE NOT EXISTS (SELECT 1 FROM threads t WHERE t.id = u.thread_id AND t.updated_at > ?)" + ) + params = [since] + if thread_ids: + placeholders = ",".join("?" for _ in thread_ids) + query += f" AND u.thread_id IN ({placeholders})" + params.extend(thread_ids) + return [r["thread_id"] for r in db.execute(query, params).fetchall()] + finally: + self.return_db(db) + + # --- Filter Table (Architecture B: Bloom Gossip) --- + + def store_peer_filter(self, peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count): + db = self.get_db() + try: + db.execute( + "INSERT OR REPLACE INTO peer_filters " + "(peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count, last_seen) " + "VALUES (?, ?, ?, ?, ?, ?)", + (peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count, time.time()), + ) + db.commit() + finally: + self.return_db(db) + + def get_peer_filter(self, peer_hash): + db = self.get_db() + try: + row = db.execute( + "SELECT * FROM peer_filters WHERE peer_hash = ?", (peer_hash,) + ).fetchone() + if row: + return dict(row) + return None + finally: + self.return_db(db) + + def get_all_filters(self): + """Return all stored peer filters (for filter table gossip).""" + db = self.get_db() + try: + return [dict(r) for r in db.execute( + "SELECT * FROM peer_filters ORDER BY last_seen DESC" + ).fetchall()] + finally: + self.return_db(db) + + def get_filtered_peers_by_tag(self, tag): + """Return peer hashes whose bloom filter might contain the given tag.""" + tag = tag.strip().lower() + db = self.get_db() + try: + matches = [] + for r in db.execute("SELECT * FROM peer_filters").fetchall(): + bf = BloomFilter.from_bytes(r["bloom_bytes"], r["bloom_size"], r["bloom_hashes"]) + if bf.might_contain(tag): + matches.append(r["peer_hash"]) + return matches + finally: + self.return_db(db) + + def prune_peer_filters(self, max_age_days=7): + db = self.get_db() + try: + cutoff = time.time() - max_age_days * 86400 + db.execute("DELETE FROM peer_filters WHERE last_seen < ?", (cutoff,)) + db.commit() + finally: + self.return_db(db) + + def get_peer_filter_count(self): + db = self.get_db() + try: + return db.execute("SELECT count(*) FROM peer_filters").fetchone()[0] + finally: + self.return_db(db) + + # --- Peer Liveness --- + + def record_sync_result(self, peer_hash, success): + db = self.get_db() + try: + existing = db.execute( + "SELECT consecutive_failures FROM synced_instances WHERE instance_hash = ?", + (peer_hash,), + ).fetchone() + if existing is not None: + new_failures = 0 if success else (existing["consecutive_failures"] + 1) + db.execute( + "UPDATE synced_instances SET consecutive_failures = ? WHERE instance_hash = ?", + (new_failures, peer_hash), + ) + if success: + db.execute( + "UPDATE synced_instances SET status = 'active' WHERE instance_hash = ?", + (peer_hash,), + ) + db.commit() + finally: + self.return_db(db) + + def get_dead_peers(self, max_failures=5): + """Return list of peer hashes with too many consecutive failures.""" + db = self.get_db() + try: + return [ + r["instance_hash"] for r in db.execute( + "SELECT instance_hash FROM synced_instances " + "WHERE consecutive_failures >= ?", + (max_failures,), + ).fetchall() + ] + finally: + self.return_db(db) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 5ff8d3d..4572390 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -16,47 +16,85 @@ def esc(s): return html.escape(str(s)) -FORUM_CSS = """ +from tinyweb_forum.bloom import BloomFilter + + +FORUM_CSS_DEFAULT = """ """ + +FORUM_CSS_KODAMA2 = """ +""" @@ -100,11 +138,16 @@ class ForumHandlers: return "" return f' [block]' + def _forum_css(self): + theme = self.fdb.get_setting("forum_theme", "default") + css = FORUM_CSS_KODAMA2 if theme == "kodama2" else FORUM_CSS_DEFAULT + return css + def _respond(self, body_html, status=200): return { "status": status, "content_type": "text/html; charset=utf-8", - "body": FORUM_CSS + body_html, + "body": self._forum_css() + body_html, "headers": {}, } @@ -125,7 +168,7 @@ class ForumHandlers: } def _error(self, status): - return self._respond(f"

    {status}

    ", status) + return self._respond(f"

    {status}

    ", status) def _paginate(self, query): try: @@ -175,6 +218,14 @@ class ForumHandlers: return False return (datetime.now() - dt).total_seconds() < RECENT_SECONDS + def _get_subscribed_topics(self): + raw = self.fdb.get_setting("topic_subscriptions", "") + return [t.strip().lower() for t in raw.split(",") if t.strip()] + + def _get_subscribed_topics_str(self): + raw = self.fdb.get_setting("topic_subscriptions", "") + return raw + def _blocked_instances(self): raw = self.fdb.get_setting("blocked_instances", "") return set(h.strip() for h in raw.split(",") if h.strip()) @@ -204,6 +255,19 @@ class ForumHandlers: # --- Routes --- + def _status_bar(self): + topics = self._get_subscribed_topics() + topics_str = ", ".join(topics) if topics else "everything" + peer_count = len(self.fdb.get_synced_instances()) + filter_count = self.fdb.get_peer_filter_count() + return ( + f'
    ' + f'subscribed: {esc(topics_str)}' + f'{peer_count} peers' + f'{filter_count} filters' + f'
    ' + ) + def handle_list(self, query): page = self._paginate(query) tag = unquote(query.get("tag", [""])[0]).strip() @@ -224,12 +288,12 @@ class ForumHandlers: continue if self._is_new(r["created_at"]): new_count += 1 - badge = "[share]" if r["url"] else "[request]" - mute_badge = " [muted]" if is_muted else "" + badge = f'share' if r["url"] else f'request' + mute_label = " [muted]" if is_muted else "" tags_html = "" if r["tags"]: tag_links = " ".join( - f'[{esc(t.strip())}]' + f'{esc(t.strip())}' for t in r["tags"].split(",") if t.strip() ) tags_html = f' {tag_links}' @@ -237,7 +301,7 @@ class ForumHandlers: items += ( f'
  • ' f'
    ' - f'{badge}{mute_badge} ' + f'{badge}{mute_label} ' f'{esc(r["title"])}' f'{tags_html}' f'
    ' @@ -250,18 +314,19 @@ class ForumHandlers: f'
    • ' ) if not items: - items = "

    No threads yet.

    " + items = "

    no threads yet.

    " new_label = f" ({new_count} new)" if new_count else "" search_form = ( f'
    ' f'' f'
    ' ) - tag_label = f' — tag: {esc(tag)}' if tag else "" + tag_label = f' — {esc(tag)}' if tag else "" muted_link = f'show muted' if not show_muted else f'show all' page_url = f'/forum?q={esc(search)}&tag={esc(tag)}&muted=1' if show_muted else (f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum') return self._respond( - f"

    forum{tag_label}

    " + f"

    forum{tag_label}

    " + f'{self._status_bar()}' f'
    ' f'{search_form}' f'
    ' @@ -270,26 +335,30 @@ class ForumHandlers: f'sync now' f'{muted_link}' f'
    ' - f"

    {total} threads{new_label}

    " + f"

    {total} threads{new_label}

    " f'
      {items}
    ' f"{self._page_nav(page, total, page_url)}" ) def handle_new_form(self, msg=""): return self._respond( - f"

    new thread

    " + f"

    new thread

    " f'
    ' f'{self._csrf_field()}' + f'' f'' f"max {MAX_TITLE_LENGTH} characters" + f'' f'' + f'' f'' f"max {MAX_BODY_LENGTH} characters" - f'' + f'' + f'' f'' f"
    " f"

    {msg}

    " - f'back' + f'
    back
    ' ) def handle_new_submit(self, body): @@ -323,28 +392,25 @@ class ForumHandlers: instance_hash = self.identity.hash.hex() if self.identity else "local" has_upvoted = self.fdb.has_upvoted(thread_id, instance_hash) - badge = "[share]" if thread["url"] else "[request]" + badge = f'share' if thread["url"] else f'request' url_html = "" if thread["url"]: url_html = ( f'

    {esc(thread["url"])}' - f' (+ save to my index)

    ' + f' (+ save)

    ' ) tags_html = "" if thread["tags"]: tag_links = " ".join( - f'[{esc(t.strip())}]' + f'{esc(t.strip())}' for t in thread["tags"].split(",") if t.strip() ) - tags_html = f'

    {tag_links}

    ' + tags_html = f'

    {tag_links}

    ' body_html = f"

    {esc(thread['body'])}

    " if thread["body"] else "" - mute_btn = ( - f'unmute' - if is_muted else - f'mute' - ) + mute_label = "unmute" if is_muted else "mute" + mute_href = f'/forum/unmute/{thread["id"]}' if is_muted else f'/forum/mute/{thread["id"]}' posts_html = "" for p in posts: @@ -352,18 +418,18 @@ class ForumHandlers: for word in p["body"].split(): w = word.strip().strip(",.!?;:") if w.startswith(("http://", "https://")): - save_links += ( - f' + save' - ) + save_links += f' + save' parent_ref = "" if p["parent_id"]: - parent_ref = f' ↪ reply' + parent_ref = f' ↪ reply' posts_html += ( - f'
    ' - f'{esc(self._author_str(p["author_name"], p["author_instance"]))}' + f'
    ' + f'
    ' + f'{esc(self._author_str(p["author_name"], p["author_instance"]))}' f'{self._block_link(p["author_instance"])}' f' · {self._time_ago(p["created_at"])}{parent_ref}' - f'{" · " + self._post_retract_link(thread["id"], p["id"]) if p["author_instance"] == instance_hash else ""}' + f'{" · " + self._post_retract_link(thread["id"], p["id"]) if p["author_instance"] == instance_hash else ""}' + f'
    ' f'

    {esc(p["body"])}

    ' f'{save_links}' f'
    ' @@ -378,15 +444,16 @@ class ForumHandlers: f"" ) + upvote_label = "-1" if has_upvoted else "+1" return self._respond( - f"

    {badge} {esc(thread['title'])}

    " - f'

    ' + f"

    {badge} {esc(thread['title'])}

    " + f'

    ' f'by {esc(self._author_str(thread["author_name"], thread["author_instance"]))}' f'{self._block_link(thread["author_instance"])}' f' · {self._time_ago(thread["created_at"])}' f' · {thread["score"]} upvotes' - f' · {mute_btn}' - f' · {"-1" if has_upvoted else "+1"}' + f' · {mute_label}' + f' · {upvote_label}' f'{self._author_links(thread["id"], thread["author_instance"], instance_hash)}' f'

    ' f'{url_html}' @@ -428,7 +495,7 @@ class ForumHandlers: if thread["author_instance"] != instance_hash: return self._error(403) return self._respond( - f"

    edit thread

    " + f"

    edit thread

    " f'
    ' f'{self._csrf_field()}' f'' @@ -511,7 +578,7 @@ class ForumHandlers: def _peer_reports_html(self): counts = self.fdb.get_peer_block_counts() if not counts: - return "

    No peer reports yet.

    " + return "

    no peer reports yet

    " auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) blocked = self._blocked_instances() items = "" @@ -524,13 +591,24 @@ class ForumHandlers: blocked = self._blocked_instances() auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip()) peer_counts = self.fdb.get_peer_block_counts() + filters = self._keyword_filters() + filters_str = ", ".join(filters) if filters else "" + synced = self.fdb.get_synced_instances() + auto_discover = self.fdb.get_setting("forum_auto_discover", "1") + auto_discover_checked = " checked" if auto_discover == "1" else "" + auto_sync = self.fdb.get_setting("forum_auto_sync", "0") + auto_sync_checked = " checked" if auto_sync == "1" else "" + retention_days = self.fdb.get_setting("forum_retention_days", "30") + blocked_items = "" if blocked: for h in sorted(blocked): - label = "[auto] " if h in auto_blocked else "" - reports = f" ({peer_counts.get(h, 0)} peers)" if h in peer_counts else "" + label = "auto" if h in auto_blocked else "" + reports = f" ({peer_counts.get(h, 0)} reports)" if h in peer_counts else "" blocked_items += ( - f'
  • {label}{esc(h[:16])}...{reports} ' + f'
  • ' + f'{esc(h[:16])}...' + f'{" [" + label + "]" if label else ""}{reports} ' f'' f'{self._csrf_field()}' f'' @@ -539,12 +617,8 @@ class ForumHandlers: ) blocked_items = f"
      {blocked_items}
    " else: - blocked_items = "

    No instances blocked.

    " + blocked_items = "

    no instances blocked

    " - filters = self._keyword_filters() - filters_str = ", ".join(filters) if filters else "" - - synced = self.fdb.get_synced_instances() synced_items = "" for s in synced: synced_items += ( @@ -555,67 +629,78 @@ class ForumHandlers: f'
    • ' f'' ) - synced_items = f"
      {synced_items}
    " if synced_items else "

    No instances synced yet.

    " - - auto_discover = self.fdb.get_setting("forum_auto_discover", "1") - auto_discover_checked = " checked" if auto_discover == "1" else "" - auto_sync = self.fdb.get_setting("forum_auto_sync", "0") - auto_sync_checked = " checked" if auto_sync == "1" else "" - retention_days = self.fdb.get_setting("forum_retention_days", "30") + synced_items = f"
      {synced_items}
    " if synced_items else "

    no instances synced

    " return self._respond( - f"

    forum moderation

    " + f"

    moderation

    " f"

    {msg}

    " - f'

    sync now

    ' - f"

    auto-discovery

    " + f'{self._status_bar()}' + + f'
    ' + f'
    subscriptions
    ' + f'
    ' + f'{self._csrf_field()}' + f'' + f"only sync content matching these topics" + f'' + f"
    " + f'
    ' + + f'
    ' + f'
    settings
    ' + f'
    network behavior
    ' f'
    ' f'{self._csrf_field()}' f'" + f" auto-discover peers via announces" f'' f"
    " - f"

    auto-sync

    " f'
    ' f'{self._csrf_field()}' f'" + f" auto-sync every 5 minutes" f'' f"
    " - f"

    storage

    " f'
    ' f'{self._csrf_field()}' - f'' - f"Older threads are pruned automatically (default: 30). Set to 0 to keep everything." + f'' f'' f"
    " - f"

    blocked instances

    " - f"{blocked_items}" - f'
    ' - f'{self._csrf_field()}' - f'' - f'' - f"
    " - f"

    peer reports

    " - f"{self._peer_reports_html()}" - f"

    keyword filters

    " - f'
    ' - f'{self._csrf_field()}' - f'' - f'' - f"
    " - f"

    synced instances

    " - f"{synced_items}" - f"

    Instances are discovered automatically via mesh announces. " - f"You can also manually add a friend's instance hash to bootstrap.

    " + f'
    ' + + f'
    ' + f'
    network
    ' + f'
    {len(synced)} known peers
    ' + f'{synced_items}' f'
    ' f'{self._csrf_field()}' f'' f'' f'' f"
    " - f'
    ' - f'back to forum' + f'
    ' + + f'
    ' + f'
    moderation
    ' + f'{blocked_items}' + f'
    ' + f'{self._csrf_field()}' + f'' + f'' + f"
    " + f'
    peer reports
    ' + f'{self._peer_reports_html()}' + f'
    keyword filters
    ' + f'
    ' + f'{self._csrf_field()}' + f'' + f'' + f"
    " + f'
    ' + + f'
    back
    ' ) def handle_block(self, body): @@ -702,31 +787,122 @@ class ForumHandlers: self.fdb.remove_synced_instance(instance) return self.handle_moderation("Removed.") + def handle_topics(self, body): + topics = body.get("topics", [""])[0].strip() + self.fdb.set_setting("topic_subscriptions", topics) + return self.handle_moderation("Topic subscriptions saved.") + # --- Sync endpoint (called over RNS) --- def handle_sync_request(self, data): - """Handle incoming sync request from another forum instance.""" since = data.get("query", {}).get("since", [""])[0] if isinstance(data.get("query"), dict) else "" incoming_threads = data.get("threads", []) incoming_posts = data.get("posts", []) incoming_upvotes = data.get("upvotes", []) + peer_topics = data.get("my_topics", []) + peer_tag_cloud = data.get("my_tag_cloud", []) + peer_bloom_data = data.get("content_bloom") + peer_tag_bloom_data = data.get("tag_bloom") + peer_filter_table = data.get("filter_table", {}) + scoped_query_tag = data.get("scoped_query", "") blocked = self._blocked_instances() + my_topics = self._get_subscribed_topics() + from_hash = data.get("from_hash", "") + + # Store peer's tag bloom filter (Architecture B discovery) + if peer_tag_bloom_data and from_hash: + self.fdb.store_peer_filter( + peer_hash=from_hash, + bloom_bytes=bytes(peer_tag_bloom_data), + bloom_size=data.get("tag_bloom_size", 2048), + bloom_hashes=data.get("tag_bloom_hashes", 3), + tag_count=len(peer_topics), + ) + + # Merge filter table gossip (transitive peer discovery) + if peer_filter_table and from_hash: + for ph, entry in peer_filter_table.items(): + if isinstance(entry, dict): + bb = entry.get("bloom_bytes") + if bb: + self.fdb.store_peer_filter( + peer_hash=ph, + bloom_bytes=bytes(bb) if isinstance(bb, list) else bb, + bloom_size=entry.get("bloom_size", 2048), + bloom_hashes=entry.get("bloom_hashes", 3), + tag_count=entry.get("tag_count", 0), + ) + + # Handle scoped query: find peers whose bloom might contain the queried tag + scoped_query_results = [] + if scoped_query_tag and from_hash: + scoped_query_results = self.fdb.get_filtered_peers_by_tag(scoped_query_tag) + + # Store peer's topics for future routing (backward compat) + if peer_topics and from_hash: + self.fdb.set_setting(f"peer_topics_{from_hash}", ",".join(peer_topics)) + if peer_tag_cloud and from_hash: + self.fdb.set_setting(f"peer_tag_cloud_{from_hash}", json.dumps(peer_tag_cloud)) + + # Build our tag bloom filter to send back + peer_tag_bs = data.get("tag_bloom_size", 2048) + peer_tag_bh = data.get("tag_bloom_hashes", 3) + my_tag_bloom = BloomFilter.from_tags(my_topics or [], peer_tag_bs, peer_tag_bh) + + # Build our content bloom filter for dedup + our_existing = set() + for t in self.fdb.get_threads_by_topics(peer_topics) if peer_topics else []: + our_existing.add(t["id"]) + our_bloom = BloomFilter.from_items(list(our_existing), data.get("bloom_size", 2048) if data else 2048, data.get("bloom_hashes", 3) if data else 3) + + # Build filter table gossip from our stored filters + all_filters = self.fdb.get_all_filters() + filter_table_gossip = {} + for f in all_filters[:20]: + if f["peer_hash"] != from_hash: + filter_table_gossip[f["peer_hash"]] = { + "bloom_bytes": list(f["bloom_bytes"]), + "bloom_size": f["bloom_size"], + "bloom_hashes": f["bloom_hashes"], + "tag_count": f["tag_count"], + } + + # Parse peer's content bloom for dedup + peer_bloom = None + if peer_bloom_data: + peer_bloom = BloomFilter.from_bytes( + bytes(peer_bloom_data), + data.get("bloom_size", 2048), + data.get("bloom_hashes", 3), + ) + + # Merge incoming content, filtered by bloom and topics if incoming_threads: for t in incoming_threads: - if t.get("author_instance", "") not in blocked: + if t.get("author_instance", "") in blocked: + continue + if peer_bloom and peer_bloom.might_contain(t["id"]): + continue + if not my_topics: self.fdb.merge_thread(t) + else: + t_tags = [tag.strip().lower() for tag in t.get("tags", "").split(",") if tag.strip()] + if set(my_topics) & set(t_tags): + self.fdb.merge_thread(t) if incoming_posts: for p in incoming_posts: - if p.get("author_instance", "") not in blocked: - self.fdb.merge_post(p) + if p.get("author_instance", "") in blocked: + continue + if peer_bloom and peer_bloom.might_contain(p["id"]): + continue + self.fdb.merge_post(p) if incoming_upvotes: for uv in incoming_upvotes: self.fdb.merge_upvote(uv["thread_id"], uv["instance_hash"]) - # Record incoming peer blocks incoming_blocks = data.get("blocks", {}) - peer_hash = data.get("peer_hash", "") or data.get("from_hash", "") + peer_hash = data.get("peer_hash", "") or from_hash if incoming_blocks and peer_hash: for h in incoming_blocks.get("mine", []): if h and h not in blocked: @@ -735,13 +911,10 @@ class ForumHandlers: if h and h not in blocked: self.fdb.record_peer_block(peer_hash, h) - # Merge incoming retractions for r in data.get("retractions", []): if r.get("id") and r.get("type") and r.get("author") and r.get("at"): self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) - # Auto-discover the peer that synced with us and their known peers - from_hash = data.get("from_hash", "") if from_hash and from_hash not in blocked: self.fdb.add_known_peer(from_hash) for peer_hash in data.get("known_peers", []): @@ -750,18 +923,34 @@ class ForumHandlers: my_blocks = list(blocked) my_peer_blocks = self.fdb.get_peer_block_list() + my_tag_cloud = self.fdb.get_tag_cloud(50) + my_tag_list = [t for t, _ in my_tag_cloud] + threads, posts, upvote_threads = [], [], [] if since: - ts, posts_list, up_list = self.fdb.get_new_content(since) - threads = [dict(r) for r in ts] - posts = [dict(r) for r in posts_list] - upvote_threads = up_list + if peer_topics: + rows = self.fdb.get_threads_by_topics(peer_topics, since=since) + threads = [dict(r) for r in rows] + tids = [r["id"] for r in rows] + posts = [dict(p) for p in self.fdb.get_posts_by_thread_ids(tids)] + uv_rows = self.fdb.get_new_upvotes_since(since, tids) + upvote_threads = uv_rows + else: + ts, posts_list, up_list = self.fdb.get_new_content(since) + threads = [dict(r) for r in ts] + posts = [dict(r) for r in posts_list] + upvote_threads = up_list + + known_peers = [h for h in self.fdb.get_all_known_hashes() if h != from_hash] + peer_topics_map = {} + for ph in known_peers[:100]: + pt = self.fdb.get_setting(f"peer_topics_{ph}", "") + if pt: + peer_topics_map[ph] = [t.strip() for t in pt.split(",") if t.strip()] retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} for cid, ct, ai, ra in self.fdb.get_raw_retractions()] - known_peers = [h for h in self.fdb.get_all_known_hashes() if h != from_hash] - return { "status": 200, "content_type": "application/json", @@ -772,6 +961,18 @@ class ForumHandlers: "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, "retractions": retracted, "known_peers": known_peers, + "peer_topics": my_tag_list, + "peer_tag_cloud": my_tag_cloud, + "content_bloom": list(our_bloom.bytes), + "bloom_size": data.get("bloom_size", 2048), + "bloom_hashes": data.get("bloom_hashes", 3), + "peer_topics_map": peer_topics_map, + # Architecture B additions + "tag_bloom": list(my_tag_bloom.bytes), + "tag_bloom_size": peer_tag_bs, + "tag_bloom_hashes": peer_tag_bh, + "filter_table": filter_table_gossip, + "scoped_query_results": scoped_query_results, }), "headers": {}, } @@ -832,7 +1033,7 @@ class ForumHandlers: elif method == "POST": if not self._check_csrf(body): return self._with_csrf( - self._respond("

    403 Forbidden

    ", status=403), csrf_token + self._respond("

    403 Forbidden

    ", status=403), csrf_token ) if sub == "/new": return self._with_csrf(self.handle_new_submit(body), csrf_token) @@ -867,6 +1068,8 @@ class ForumHandlers: return self._with_csrf(self.handle_storage(body), csrf_token) elif sub == "/auto_sync": return self._with_csrf(self.handle_auto_sync(body), csrf_token) + elif sub == "/topics": + return self._with_csrf(self.handle_topics(body), csrf_token) return self._with_csrf(self._error(404), csrf_token) diff --git a/tinyweb_forum/sync.py b/tinyweb_forum/sync.py index 99e1985..a9d544d 100644 --- a/tinyweb_forum/sync.py +++ b/tinyweb_forum/sync.py @@ -4,15 +4,22 @@ import threading import time import RNS +from tinyweb_forum.bloom import BloomFilter + FORUM_APP = "tinyweb-forum" -SYNC_INTERVAL = 300 # 5 minutes +SYNC_INTERVAL = 300 REQUEST_TIMEOUT = 60 -GOSSIP_FANOUT = 20 # random peers to sync per cycle +GOSSIP_FANOUT = 20 +BLOOM_SIZE = 2048 +BLOOM_HASHES = 3 +TAG_BLOOM_SIZE = 2048 +TAG_BLOOM_HASHES = 3 +FILTER_TABLE_GOSSIP = 20 +MAX_PEER_FAILURES = 5 +FILTER_TABLE_TTL_DAYS = 7 class _ForumAnnounceHandler: - """Receives announces from other forum instances and auto-discovers them.""" - aspect_filter = FORUM_APP receive_path_responses = False @@ -39,6 +46,26 @@ class ForumSync: self._running = False self._thread = None + def _get_subscribed_topics(self): + raw = self.fdb.get_setting("topic_subscriptions", "") + return [t.strip().lower() for t in raw.split(",") if t.strip()] + + def _build_tag_bloom(self, topics=None): + if topics is None: + topics = self._get_subscribed_topics() + return BloomFilter.from_tags(topics or [], TAG_BLOOM_SIZE, TAG_BLOOM_HASHES) + + def _topics_overlap(self, my_topics, their_topics): + if not my_topics or not their_topics: + return True + return bool(set(my_topics) & set(their_topics)) + + def _topics_overlap_bloom(self, my_topics, peer_bloom): + """Check overlap using bloom filter instead of topic list.""" + if not my_topics or peer_bloom is None: + return True + return any(peer_bloom.might_contain(t) for t in my_topics) + def start(self): self.destination = RNS.Destination( self.identity, @@ -66,23 +93,48 @@ class ForumSync: self.fdb.set_setting("forum_auto_sync", "1" if enabled else "0") if enabled: self._start_sync_loop() - else: - pass # current cycle finishes, no new one starts def sync_now(self): - """Run one sync cycle immediately. Returns count of peers synced.""" instances = self.fdb.get_synced_instances() random.shuffle(instances) count = 0 + my_topics = self._get_subscribed_topics() + my_tag_bloom = self._build_tag_bloom(my_topics) + for inst in instances[:GOSSIP_FANOUT]: + if not self._running: + break + pfilter = self.fdb.get_peer_filter(inst["instance_hash"]) + if pfilter: + pbloom = BloomFilter.from_bytes(pfilter["bloom_bytes"], pfilter["bloom_size"], pfilter["bloom_hashes"]) + if not self._topics_overlap_bloom(my_topics, pbloom): + continue + else: + peer_topics = self._peer_tag_topics(inst["instance_hash"]) + if not self._topics_overlap(my_topics, peer_topics): + continue + try: + self._sync_with(inst["instance_hash"], my_topics, my_tag_bloom) + count += 1 + except Exception as e: + self.fdb.record_sync_result(inst["instance_hash"], False) + print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") + return count + + def scoped_query(self, tag): + """Ask known peers if they know anyone with the given tag.""" + tag = tag.strip().lower() + instances = self.fdb.get_synced_instances() + random.shuffle(instances) + results = set() for inst in instances[:GOSSIP_FANOUT]: if not self._running: break try: - self._sync_with(inst["instance_hash"]) - count += 1 - except Exception as e: - print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") - return count + matches = self._scoped_query_peer(inst["instance_hash"], tag) + results.update(matches) + except Exception: + continue + return list(results) def _start_sync_loop(self): if self._thread and self._thread.is_alive(): @@ -120,16 +172,25 @@ class ForumSync: while self._running: try: instances = self.fdb.get_synced_instances() - # Gossip: sync with random subset for scaling - # If <= GOSSIP_FANOUT peers, sync with all (current behavior) - # If more, sync with random FANOUT per cycle — content spreads epidemically + my_topics = self._get_subscribed_topics() + my_tag_bloom = self._build_tag_bloom(my_topics) random.shuffle(instances) for inst in instances[:GOSSIP_FANOUT]: if not self._running: break + pfilter = self.fdb.get_peer_filter(inst["instance_hash"]) + if pfilter: + pbloom = BloomFilter.from_bytes(pfilter["bloom_bytes"], pfilter["bloom_size"], pfilter["bloom_hashes"]) + if not self._topics_overlap_bloom(my_topics, pbloom): + continue + else: + peer_topics = self._peer_tag_topics(inst["instance_hash"]) + if not self._topics_overlap(my_topics, peer_topics): + continue try: - self._sync_with(inst["instance_hash"]) + self._sync_with(inst["instance_hash"], my_topics, my_tag_bloom) except Exception as e: + self.fdb.record_sync_result(inst["instance_hash"], False) print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}") except Exception: pass @@ -141,12 +202,99 @@ class ForumSync: self.fdb.prune_old_content(days) except Exception: pass + try: + self.fdb.prune_peer_filters(FILTER_TABLE_TTL_DAYS) + except Exception: + pass + try: + for dead in self.fdb.get_dead_peers(MAX_PEER_FAILURES): + self.fdb.remove_synced_instance(dead) + except Exception: + pass for _ in range(SYNC_INTERVAL): if not self._running: return time.sleep(1) - def _sync_with(self, instance_hash): + def _peer_tag_topics(self, instance_hash): + raw = self.fdb.get_setting(f"peer_topics_{instance_hash}", "") + return [t.strip().lower() for t in raw.split(",") if t.strip()] if raw else [] + + def _store_peer_filter(self, peer_hash, data): + tag_bloom_data = data.get("tag_bloom") + if tag_bloom_data: + self.fdb.store_peer_filter( + peer_hash=peer_hash, + bloom_bytes=bytes(tag_bloom_data), + bloom_size=data.get("tag_bloom_size", TAG_BLOOM_SIZE), + bloom_hashes=data.get("tag_bloom_hashes", TAG_BLOOM_HASHES), + tag_count=len(data.get("peer_topics", [])), + ) + + def _merge_filter_table_gossip(self, filter_table): + if not filter_table: + return + for ph, entry in filter_table.items(): + if isinstance(entry, dict): + bb = entry.get("bloom_bytes") + if bb: + self.fdb.store_peer_filter( + peer_hash=ph, + bloom_bytes=bytes(bb) if isinstance(bb, list) else bb, + bloom_size=entry.get("bloom_size", TAG_BLOOM_SIZE), + bloom_hashes=entry.get("bloom_hashes", TAG_BLOOM_HASHES), + tag_count=entry.get("tag_count", 0), + ) + + def _scoped_query_peer(self, peer_hash, tag): + dest_hash = bytes.fromhex(peer_hash) + if not RNS.Transport.has_path(dest_hash): + RNS.Transport.request_path(dest_hash) + elapsed = 0 + while not RNS.Transport.has_path(dest_hash) and elapsed < 15: + time.sleep(0.5) + elapsed += 0.5 + if not RNS.Transport.has_path(dest_hash): + return [] + server_identity = RNS.Identity.recall(dest_hash) + if server_identity is None: + return [] + destination = RNS.Destination( + server_identity, + RNS.Destination.OUT, + RNS.Destination.SINGLE, + FORUM_APP, + ) + link = RNS.Link(destination) + elapsed = 0 + while link.status == RNS.Link.PENDING and elapsed < 15: + time.sleep(0.25) + elapsed += 0.25 + if link.status != RNS.Link.ACTIVE: + return [] + try: + my_hash = self.identity.hash.hex() if self.identity else "local" + request_data = { + "scoped_query": tag, + "from_hash": my_hash, + } + receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) + elapsed = 0 + done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) + while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT: + time.sleep(0.1) + elapsed += 0.1 + if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): + resp = receipt.get_response() + if isinstance(resp, dict) and resp.get("status") == 200: + data = json.loads(resp["body"]) + return data.get("scoped_query_results", []) + return [] + finally: + link.teardown() + + def _sync_with(self, instance_hash, my_topics=None, my_tag_bloom=None): + success = False dest_hash = bytes.fromhex(instance_hash) if not RNS.Transport.has_path(dest_hash): RNS.Transport.request_path(dest_hash) @@ -155,10 +303,12 @@ class ForumSync: time.sleep(0.5) elapsed += 0.5 if not RNS.Transport.has_path(dest_hash): + self.fdb.record_sync_result(instance_hash, False) return server_identity = RNS.Identity.recall(dest_hash) if server_identity is None: + self.fdb.record_sync_result(instance_hash, False) return destination = RNS.Destination( @@ -175,6 +325,7 @@ class ForumSync: elapsed += 0.25 if link.status != RNS.Link.ACTIVE: + self.fdb.record_sync_result(instance_hash, False) return try: @@ -186,14 +337,50 @@ class ForumSync: break since = last_sync.replace(" ", "T") if last_sync else "" + my_hash = self.identity.hash.hex() if self.identity else "local" + + if my_topics is None: + my_topics = self._get_subscribed_topics() + if my_tag_bloom is None: + my_tag_bloom = self._build_tag_bloom(my_topics) + + # Build filter table gossip: share a subset of our known peer filters + all_filters = self.fdb.get_all_filters() + filter_table_gossip = {} + for f in all_filters[:FILTER_TABLE_GOSSIP]: + if f["peer_hash"] != instance_hash and f["peer_hash"] != my_hash: + filter_table_gossip[f["peer_hash"]] = { + "bloom_bytes": list(f["bloom_bytes"]), + "bloom_size": f["bloom_size"], + "bloom_hashes": f["bloom_hashes"], + "tag_count": f["tag_count"], + } threads, posts = [], [] upvotes = [] + my_tag_cloud = self.fdb.get_tag_cloud(50) + my_tag_list = [t for t, _ in my_tag_cloud] + + # Only build content if we have a since timestamp (incremental sync) if since: - ts, ps, uv = self.fdb.get_new_content(since) - threads = [dict(r) for r in ts] - posts = [dict(r) for r in ps] - upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv] + if my_topics: + rows = self.fdb.get_threads_by_topics(my_topics, since=since) + threads = [dict(r) for r in rows] + tids = [r["id"] for r in rows] + posts = [dict(p) for p in self.fdb.get_posts_by_thread_ids(tids)] + uv_rows = self.fdb.get_new_upvotes_since(since, tids) + upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv_rows] + else: + ts, ps, uv = self.fdb.get_new_content(since) + threads = [dict(r) for r in ts] + posts = [dict(r) for r in ps] + upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv] + + # Build content bloom for dedup + existing_ids = set(t["id"] for t in threads) + for t in self.fdb.get_threads_by_topics(my_topics) if my_topics else []: + existing_ids.add(t["id"]) + content_bloom = BloomFilter.from_items(list(existing_ids), BLOOM_SIZE, BLOOM_HASHES) my_blocks = [h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()] my_peer_blocks = self.fdb.get_peer_block_list() @@ -201,8 +388,12 @@ class ForumSync: retracted = [{"id": cid, "type": ct, "author": ai, "at": ra} for cid, ct, ai, ra in self.fdb.get_raw_retractions()] - my_hash = self.identity.hash.hex() if self.identity else "local" known_peers = [h for h in self.fdb.get_all_known_hashes() if h != instance_hash and h != my_hash] + peer_topics_map = {} + for ph in known_peers[:100]: + pt = self.fdb.get_setting(f"peer_topics_{ph}", "") + if pt: + peer_topics_map[ph] = [t.strip() for t in pt.split(",") if t.strip()] request_data = { "query": {"since": [since]} if since else {}, @@ -213,6 +404,17 @@ class ForumSync: "blocks": {"mine": my_blocks, "peers": my_peer_blocks}, "retractions": retracted, "known_peers": known_peers, + "my_topics": my_topics, + "my_tag_cloud": my_tag_cloud, + "content_bloom": list(content_bloom.bytes), + "bloom_size": BLOOM_SIZE, + "bloom_hashes": BLOOM_HASHES, + "peer_topics_map": peer_topics_map, + # Architecture B additions + "tag_bloom": list(my_tag_bloom.bytes), + "tag_bloom_size": TAG_BLOOM_SIZE, + "tag_bloom_hashes": TAG_BLOOM_HASHES, + "filter_table": filter_table_gossip, } receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT) @@ -229,16 +431,82 @@ class ForumSync: data = json.loads(resp["body"]) except (json.JSONDecodeError, KeyError): data = {} + success = True + + # Store peer's tag bloom filter + self._store_peer_filter(instance_hash, data) + + # Merge filter table gossip from peer + self._merge_filter_table_gossip(data.get("filter_table", {})) + + # Store peer topics (backward compat) + peer_topics = data.get("peer_topics", []) + if peer_topics: + self.fdb.set_setting(f"peer_topics_{instance_hash}", ",".join(peer_topics)) + peer_tag_cloud = data.get("peer_tag_cloud", []) + if peer_tag_cloud: + self.fdb.set_setting(f"peer_tag_cloud_{instance_hash}", json.dumps(peer_tag_cloud)) + + # Check tag overlap using bloom filter my_blocks = set(h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()) + + peer_tag_bloom_data = data.get("tag_bloom") + peer_tag_bloom = None + if peer_tag_bloom_data: + peer_tag_bloom = BloomFilter.from_bytes( + bytes(peer_tag_bloom_data), + data.get("tag_bloom_size", TAG_BLOOM_SIZE), + data.get("tag_bloom_hashes", TAG_BLOOM_HASHES), + ) + + # If no tag overlap, skip content sync but still store the filter + if my_topics and peer_tag_bloom is not None: + if not self._topics_overlap_bloom(my_topics, peer_tag_bloom): + success = True + now = time.strftime("%Y-%m-%dT%H:%M:%S") + self.fdb.set_last_sync(instance_hash, now) + self.fdb.record_sync_result(instance_hash, True) + return + + # Fall back to topic list overlap check if no bloom + if peer_tag_bloom is None: + incoming_topics = data.get("peer_topics", []) + if my_topics and incoming_topics: + if not self._topics_overlap(my_topics, incoming_topics): + success = True + now = time.strftime("%Y-%m-%dT%H:%M:%S") + self.fdb.set_last_sync(instance_hash, now) + self.fdb.record_sync_result(instance_hash, True) + return + + # Check content bloom for dedup + peer_bloom_data = data.get("content_bloom") + peer_bloom = None + if peer_bloom_data: + peer_bloom = BloomFilter.from_bytes( + bytes(peer_bloom_data), + data.get("bloom_size", BLOOM_SIZE), + data.get("bloom_hashes", BLOOM_HASHES), + ) + for t in data.get("threads", []): if t.get("author_instance", "") not in my_blocks: - self.fdb.merge_thread(t) + if peer_bloom and peer_bloom.might_contain(t["id"]): + continue + if not my_topics: + self.fdb.merge_thread(t) + else: + t_tags = [tag.strip().lower() for tag in t.get("tags", "").split(",") if tag.strip()] + if set(my_topics) & set(t_tags): + self.fdb.merge_thread(t) for p in data.get("posts", []): if p.get("author_instance", "") not in my_blocks: + if peer_bloom and peer_bloom.might_contain(p["id"]): + continue self.fdb.merge_post(p) for tid in data.get("upvote_threads", []): self.fdb.merge_upvote(tid, instance_hash) - # Gossip blocks from peer + peer_blocks = data.get("blocks", {}) for h in peer_blocks.get("mine", []): if h and h not in my_blocks and instance_hash: @@ -247,18 +515,25 @@ class ForumSync: if h and h not in my_blocks and instance_hash: self.fdb.record_peer_block(instance_hash, h) self._apply_peer_blocks() - # Merge incoming retractions + for r in data.get("retractions", []): if r.get("id") and r.get("type") and r.get("author") and r.get("at"): self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"]) - # Discover new peers from gossip + for peer_hash in data.get("known_peers", []): if peer_hash and peer_hash != my_hash and peer_hash != instance_hash: self.fdb.add_known_peer(peer_hash) + + my_topics_set = set(my_topics) + for ph, pt in data.get("peer_topics_map", {}).items(): + if ph and ph != my_hash and ph != instance_hash: + pt_set = set(t.lower() for t in pt) + if not my_topics or my_topics_set & pt_set: + self.fdb.add_known_peer(ph) + now = time.strftime("%Y-%m-%dT%H:%M:%S") self.fdb.set_last_sync(instance_hash, now) - else: - pass + self.fdb.record_sync_result(instance_hash, success) finally: link.teardown() From 7f05df826f47744422e1809365c532e53ccb6a42 Mon Sep 17 00:00:00 2001 From: blankie Date: Sat, 6 Jun 2026 01:41:41 +0000 Subject: [PATCH 53/56] strip forum CSS, defer all styling to site template --- tinyweb_forum/handlers.py | 86 +-------------------------------------- 1 file changed, 1 insertion(+), 85 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 4572390..f6a4f5b 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -19,85 +19,6 @@ def esc(s): from tinyweb_forum.bloom import BloomFilter -FORUM_CSS_DEFAULT = """ -""" - -FORUM_CSS_KODAMA2 = """ -""" - - class ForumHandlers: def __init__(self, fdb, sync, identity, reticulum, site_name="me"): self.fdb = fdb @@ -138,16 +59,11 @@ class ForumHandlers: return "" return f' [block]' - def _forum_css(self): - theme = self.fdb.get_setting("forum_theme", "default") - css = FORUM_CSS_KODAMA2 if theme == "kodama2" else FORUM_CSS_DEFAULT - return css - def _respond(self, body_html, status=200): return { "status": status, "content_type": "text/html; charset=utf-8", - "body": self._forum_css() + body_html, + "body": body_html, "headers": {}, } From 5dce1323c6c377094f1f3e0fa6a926182268922d Mon Sep 17 00:00:00 2001 From: user Date: Sat, 6 Jun 2026 01:41:41 +0000 Subject: [PATCH 54/56] strip forum CSS, defer all styling to site template --- tinyweb_forum/handlers.py | 86 +-------------------------------------- 1 file changed, 1 insertion(+), 85 deletions(-) diff --git a/tinyweb_forum/handlers.py b/tinyweb_forum/handlers.py index 4572390..f6a4f5b 100644 --- a/tinyweb_forum/handlers.py +++ b/tinyweb_forum/handlers.py @@ -19,85 +19,6 @@ def esc(s): from tinyweb_forum.bloom import BloomFilter -FORUM_CSS_DEFAULT = """ -""" - -FORUM_CSS_KODAMA2 = """ -""" - - class ForumHandlers: def __init__(self, fdb, sync, identity, reticulum, site_name="me"): self.fdb = fdb @@ -138,16 +59,11 @@ class ForumHandlers: return "" return f' [block]' - def _forum_css(self): - theme = self.fdb.get_setting("forum_theme", "default") - css = FORUM_CSS_KODAMA2 if theme == "kodama2" else FORUM_CSS_DEFAULT - return css - def _respond(self, body_html, status=200): return { "status": status, "content_type": "text/html; charset=utf-8", - "body": self._forum_css() + body_html, + "body": body_html, "headers": {}, } From bf1cb0b0a4ae19c7b52c6ad7e23c399dce320605 Mon Sep 17 00:00:00 2001 From: blankie Date: Tue, 9 Jun 2026 01:10:04 +0000 Subject: [PATCH 55/56] add AGPLv3 license --- LICENSE | 574 +++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 575 insertions(+), 1 deletion(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5445774 --- /dev/null +++ b/LICENSE @@ -0,0 +1,574 @@ +GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License giving you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +are made publicly available of their being a derivative work, need not +be distributed to others. + + For example, if you modify a part of a free program, you are +not required to distribute the object code for the modified version +itself; however, the GNU Affero General Public License requires you +to provide source code for any version of the program that you use +or modify. This requirement is similar to the requirement that +the user can receive the source code if they distribute a copy. + + Also, if you link or combine the program with any other software +that contains code covered by this License (or any work based on the +program), you must provide the source code for that combined work +as well. The GNU Affero General Public License normally requires +that any work that you distribute or publish that in whole or in +part contains or is derived from the program or any part thereof, +to be licensed as a whole at no charge to all third parties under +the terms of this License. This is known as "providing source code" +or "making available" the work. + + An "aggregated" or "combined" work is not covered by this License +if you do not meet these conditions, and you must provide the source +code as above. Additionally, aggregating works does not exempt you +from the requirements of this License. + + Specifically, if you make an "aggregate" or "combined" work by +combining this program with other software (or any work based on this +program) on a volume of a storage or distribution medium, you must +provide the source code for the combined work as above. This +requirement is intended to ensure that any user of the combined work +gets the source code that you made available, and can exercise the +right to modify and re-distribute the combined work. + + This License is specifically intended to limit any attempt to +place your modifications under a license that would restrict re-use +or further modification by others. This is to ensure that any +derivative work you create will be available under the same license +as the original, so that any derivative work can be re-distributed +under the same conditions as the original. + + Finally, this License is not intended to limit your rights under +fair use or other limitations on exclusive rights, such as patents +or trademarks. This License does not grant you any rights to the +names of the authors or copyright holders, nor to trade names, +trademarks, or service marks, except as needed for the normal and +customary use in describing the origin of the work and reproducing +the content of the notice file. + + The source code for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided in copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must contain prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must contain prominent notices stating that it is + released under this License and any conditions added under section 7. + This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the legal rights of the compilation's users beyond +what the individual works permit. Inclusion of a covered work in an +aggregate does not cause this License to apply to the other parts of +the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in + accord with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A +product is a consumer product regardless of whether the product has +substantial commercial, industrial or non-consumer uses, unless such +uses represent the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions applicable to the entire Program shall be treated +as though they were included in this License, to the extent that they +are valid under applicable law. If additional permissions apply only to +part of the Program, then that part may be used separately under those +permissions, but the entire Program remains governed by this License +without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as +you received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, or selling the work, +or by making, using, or selling the work. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available to anyone +in the United States, you may not convey the work under this License. +This is done by providing access to copy the corresponding source code +from a network server at no charge. + + If, during the execution of the Program, the Program is transmitted +to a user or a computer, either the source code or object code, you +must meet the requirements of this License regarding the +Corresponding Source of the work. You must make sure that the source +code or object code (as applicable) is available for such users to +copy and modify, and to run, for their own use, the corresponding +source in accordance with this License. This requirement applies +both to the work as stand-alone and to the work as part of an +aggregate. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user +through a computer network, with no transfer of a copy, is not +conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 12. No Warranty + + THE PROGRAM IS PROVIDED WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK +AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD +THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY +SERVICING, REPAIR OR CORRECTION. + + 13. Disclaimer of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR +THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + 14. Interpretation of Sections 12 and 13. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + + The hypothetical commands `show w' and `show c' should show the appropriate + parts of the General Public License. Of course, your program's commands + might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, + if any, to sign a "copyright disclaimer" for the program, if necessary. + For more information on this, and how to apply and follow the GNU AGPL, + see . diff --git a/pyproject.toml b/pyproject.toml index 19a0bb7..c164407 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.backends._legacy:_Backend" name = "tinyweb-forum" version = "0.1.0" description = "Decentralized link-sharing forum for TinyWeb" -license = {text = "MIT"} +license = {text = "AGPL-3.0-only"} requires-python = ">=3.9" dependencies = [ "rns", From 0e7e6d841327e8372aaba5adc5326689abb645c6 Mon Sep 17 00:00:00 2001 From: user Date: Tue, 9 Jun 2026 01:10:04 +0000 Subject: [PATCH 56/56] add AGPLv3 license --- LICENSE | 574 +++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 575 insertions(+), 1 deletion(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5445774 --- /dev/null +++ b/LICENSE @@ -0,0 +1,574 @@ +GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License giving you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +are made publicly available of their being a derivative work, need not +be distributed to others. + + For example, if you modify a part of a free program, you are +not required to distribute the object code for the modified version +itself; however, the GNU Affero General Public License requires you +to provide source code for any version of the program that you use +or modify. This requirement is similar to the requirement that +the user can receive the source code if they distribute a copy. + + Also, if you link or combine the program with any other software +that contains code covered by this License (or any work based on the +program), you must provide the source code for that combined work +as well. The GNU Affero General Public License normally requires +that any work that you distribute or publish that in whole or in +part contains or is derived from the program or any part thereof, +to be licensed as a whole at no charge to all third parties under +the terms of this License. This is known as "providing source code" +or "making available" the work. + + An "aggregated" or "combined" work is not covered by this License +if you do not meet these conditions, and you must provide the source +code as above. Additionally, aggregating works does not exempt you +from the requirements of this License. + + Specifically, if you make an "aggregate" or "combined" work by +combining this program with other software (or any work based on this +program) on a volume of a storage or distribution medium, you must +provide the source code for the combined work as above. This +requirement is intended to ensure that any user of the combined work +gets the source code that you made available, and can exercise the +right to modify and re-distribute the combined work. + + This License is specifically intended to limit any attempt to +place your modifications under a license that would restrict re-use +or further modification by others. This is to ensure that any +derivative work you create will be available under the same license +as the original, so that any derivative work can be re-distributed +under the same conditions as the original. + + Finally, this License is not intended to limit your rights under +fair use or other limitations on exclusive rights, such as patents +or trademarks. This License does not grant you any rights to the +names of the authors or copyright holders, nor to trade names, +trademarks, or service marks, except as needed for the normal and +customary use in describing the origin of the work and reproducing +the content of the notice file. + + The source code for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided in copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must contain prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must contain prominent notices stating that it is + released under this License and any conditions added under section 7. + This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the legal rights of the compilation's users beyond +what the individual works permit. Inclusion of a covered work in an +aggregate does not cause this License to apply to the other parts of +the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in + accord with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A +product is a consumer product regardless of whether the product has +substantial commercial, industrial or non-consumer uses, unless such +uses represent the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions applicable to the entire Program shall be treated +as though they were included in this License, to the extent that they +are valid under applicable law. If additional permissions apply only to +part of the Program, then that part may be used separately under those +permissions, but the entire Program remains governed by this License +without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as +you received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, or selling the work, +or by making, using, or selling the work. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available to anyone +in the United States, you may not convey the work under this License. +This is done by providing access to copy the corresponding source code +from a network server at no charge. + + If, during the execution of the Program, the Program is transmitted +to a user or a computer, either the source code or object code, you +must meet the requirements of this License regarding the +Corresponding Source of the work. You must make sure that the source +code or object code (as applicable) is available for such users to +copy and modify, and to run, for their own use, the corresponding +source in accordance with this License. This requirement applies +both to the work as stand-alone and to the work as part of an +aggregate. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user +through a computer network, with no transfer of a copy, is not +conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 12. No Warranty + + THE PROGRAM IS PROVIDED WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK +AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD +THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY +SERVICING, REPAIR OR CORRECTION. + + 13. Disclaimer of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR +THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + 14. Interpretation of Sections 12 and 13. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + + The hypothetical commands `show w' and `show c' should show the appropriate + parts of the General Public License. Of course, your program's commands + might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, + if any, to sign a "copyright disclaimer" for the program, if necessary. + For more information on this, and how to apply and follow the GNU AGPL, + see . diff --git a/pyproject.toml b/pyproject.toml index 19a0bb7..c164407 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.backends._legacy:_Backend" name = "tinyweb-forum" version = "0.1.0" description = "Decentralized link-sharing forum for TinyWeb" -license = {text = "MIT"} +license = {text = "AGPL-3.0-only"} requires-python = ">=3.9" dependencies = [ "rns",