Architecture B: Bloom Gossip + Implicit Replication
- Tag bloom filter as primary peer discovery (2048 bits x 3 hashes) - Filter table gossip for transitive peer discovery - Scoped queries for on-demand tag lookup - Liveness tracking + peer eviction (5 failures, 7-day TTL) - Topic subscriptions filter content sync at both ends - Two-theme CSS system (default minimal + kodama2) - Status bar on all pages (topics, peers, filters) - Bracketless tags, grouped moderation page, cleaner forms - Match main site heading level (h1 -> h2)
This commit is contained in:
blankie
parent
ce6d31c357
commit
497079c7a7
4 changed files with 875 additions and 164 deletions
48
tinyweb_forum/bloom.py
Normal file
48
tinyweb_forum/bloom.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import hashlib
|
||||
|
||||
|
||||
class BloomFilter:
|
||||
def __init__(self, size=2048, num_hashes=3):
|
||||
self.size = size
|
||||
self.num_hashes = num_hashes
|
||||
self.bits = bytearray(size // 8 + 1)
|
||||
|
||||
def _hash_positions(self, item):
|
||||
h = hashlib.sha256(item.encode("utf-8")).digest()
|
||||
for i in range(self.num_hashes):
|
||||
val = int.from_bytes(h[i*4:(i+1)*4], "big") % self.size
|
||||
yield val
|
||||
|
||||
def add(self, item):
|
||||
for pos in self._hash_positions(item):
|
||||
self.bits[pos // 8] |= 1 << (pos % 8)
|
||||
|
||||
def might_contain(self, item):
|
||||
return all(
|
||||
bool(self.bits[pos // 8] & (1 << (pos % 8)))
|
||||
for pos in self._hash_positions(item)
|
||||
)
|
||||
|
||||
@property
|
||||
def bytes(self):
|
||||
return bytes(self.bits)
|
||||
|
||||
@classmethod
|
||||
def from_bytes(cls, data, size=2048, num_hashes=3):
|
||||
bf = cls(size=size, num_hashes=num_hashes)
|
||||
bf.bits = bytearray(data)
|
||||
return bf
|
||||
|
||||
@staticmethod
|
||||
def from_items(items, size=2048, num_hashes=3):
|
||||
bf = BloomFilter(size=size, num_hashes=num_hashes)
|
||||
for item in items:
|
||||
bf.add(item)
|
||||
return bf
|
||||
|
||||
@staticmethod
|
||||
def from_tags(tags, size=2048, num_hashes=3):
|
||||
bf = BloomFilter(size=size, num_hashes=num_hashes)
|
||||
for tag in tags:
|
||||
bf.add(tag.strip().lower())
|
||||
return bf
|
||||
|
|
@ -1,8 +1,11 @@
|
|||
import sqlite3
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from tinyweb_forum.bloom import BloomFilter
|
||||
|
||||
FORUM_DB = "forum.db"
|
||||
|
||||
|
||||
|
|
@ -88,6 +91,20 @@ class ForumDB:
|
|||
" PRIMARY KEY (content_id, content_type)"
|
||||
")"
|
||||
)
|
||||
db.execute(
|
||||
"CREATE TABLE IF NOT EXISTS peer_filters ("
|
||||
" peer_hash TEXT PRIMARY KEY,"
|
||||
" bloom_bytes BLOB,"
|
||||
" bloom_size INTEGER DEFAULT 2048,"
|
||||
" bloom_hashes INTEGER DEFAULT 3,"
|
||||
" tag_count INTEGER DEFAULT 0,"
|
||||
" last_seen REAL NOT NULL"
|
||||
")"
|
||||
)
|
||||
try:
|
||||
db.execute("ALTER TABLE synced_instances ADD COLUMN consecutive_failures INTEGER DEFAULT 0")
|
||||
except Exception:
|
||||
pass
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
|
@ -509,22 +526,190 @@ class ForumDB:
|
|||
db = self.get_db()
|
||||
try:
|
||||
cutoff = (datetime.utcnow() - timedelta(days=retention_days)).strftime("%Y-%m-%dT%H:%M:%S")
|
||||
# Delete posts in old threads
|
||||
db.execute(
|
||||
"DELETE FROM posts WHERE thread_id IN "
|
||||
"(SELECT id FROM threads WHERE updated_at < ?)",
|
||||
(cutoff,),
|
||||
)
|
||||
# Delete orphaned posts (thread already deleted)
|
||||
db.execute(
|
||||
"DELETE FROM posts WHERE thread_id NOT IN (SELECT id FROM threads)"
|
||||
)
|
||||
# Delete old threads
|
||||
db.execute("DELETE FROM threads WHERE updated_at < ?", (cutoff,))
|
||||
# Clean up orphaned upvotes
|
||||
db.execute(
|
||||
"DELETE FROM upvotes WHERE thread_id NOT IN (SELECT id FROM threads)"
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_tag_cloud(self, limit=50):
|
||||
db = self.get_db()
|
||||
try:
|
||||
rows = db.execute("SELECT tags FROM threads").fetchall()
|
||||
counts = {}
|
||||
for r in rows:
|
||||
if r["tags"]:
|
||||
for t in r["tags"].split(","):
|
||||
tag = t.strip().lower()
|
||||
if tag:
|
||||
counts[tag] = counts.get(tag, 0) + 1
|
||||
return sorted(counts.items(), key=lambda x: -x[1])[:limit]
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_threads_by_topics(self, topics, since="", limit=200):
|
||||
db = self.get_db()
|
||||
try:
|
||||
params = []
|
||||
where = []
|
||||
if topics:
|
||||
clauses = []
|
||||
for t in topics:
|
||||
clauses.append("t.tags LIKE ?")
|
||||
params.append(f"%{t}%")
|
||||
where.append("(" + " OR ".join(clauses) + ")")
|
||||
if since:
|
||||
where.append("t.updated_at > ?")
|
||||
params.append(since)
|
||||
where_clause = (" WHERE " + " AND ".join(where)) if where else ""
|
||||
return db.execute(
|
||||
"SELECT t.*, (SELECT count(*) FROM posts p WHERE p.thread_id = t.id) AS reply_count "
|
||||
f"FROM threads t{where_clause} ORDER BY t.updated_at DESC LIMIT ?",
|
||||
params + [limit],
|
||||
).fetchall()
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_posts_by_thread_ids(self, thread_ids):
|
||||
if not thread_ids:
|
||||
return []
|
||||
db = self.get_db()
|
||||
try:
|
||||
placeholders = ",".join("?" for _ in thread_ids)
|
||||
return db.execute(
|
||||
f"SELECT * FROM posts WHERE thread_id IN ({placeholders}) ORDER BY created_at ASC",
|
||||
thread_ids,
|
||||
).fetchall()
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_new_upvotes_since(self, since, thread_ids=None):
|
||||
db = self.get_db()
|
||||
try:
|
||||
query = (
|
||||
"SELECT thread_id FROM upvotes u "
|
||||
"WHERE NOT EXISTS (SELECT 1 FROM threads t WHERE t.id = u.thread_id AND t.updated_at > ?)"
|
||||
)
|
||||
params = [since]
|
||||
if thread_ids:
|
||||
placeholders = ",".join("?" for _ in thread_ids)
|
||||
query += f" AND u.thread_id IN ({placeholders})"
|
||||
params.extend(thread_ids)
|
||||
return [r["thread_id"] for r in db.execute(query, params).fetchall()]
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
# --- Filter Table (Architecture B: Bloom Gossip) ---
|
||||
|
||||
def store_peer_filter(self, peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count):
|
||||
db = self.get_db()
|
||||
try:
|
||||
db.execute(
|
||||
"INSERT OR REPLACE INTO peer_filters "
|
||||
"(peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count, last_seen) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(peer_hash, bloom_bytes, bloom_size, bloom_hashes, tag_count, time.time()),
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_peer_filter(self, peer_hash):
|
||||
db = self.get_db()
|
||||
try:
|
||||
row = db.execute(
|
||||
"SELECT * FROM peer_filters WHERE peer_hash = ?", (peer_hash,)
|
||||
).fetchone()
|
||||
if row:
|
||||
return dict(row)
|
||||
return None
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_all_filters(self):
|
||||
"""Return all stored peer filters (for filter table gossip)."""
|
||||
db = self.get_db()
|
||||
try:
|
||||
return [dict(r) for r in db.execute(
|
||||
"SELECT * FROM peer_filters ORDER BY last_seen DESC"
|
||||
).fetchall()]
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_filtered_peers_by_tag(self, tag):
|
||||
"""Return peer hashes whose bloom filter might contain the given tag."""
|
||||
tag = tag.strip().lower()
|
||||
db = self.get_db()
|
||||
try:
|
||||
matches = []
|
||||
for r in db.execute("SELECT * FROM peer_filters").fetchall():
|
||||
bf = BloomFilter.from_bytes(r["bloom_bytes"], r["bloom_size"], r["bloom_hashes"])
|
||||
if bf.might_contain(tag):
|
||||
matches.append(r["peer_hash"])
|
||||
return matches
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def prune_peer_filters(self, max_age_days=7):
|
||||
db = self.get_db()
|
||||
try:
|
||||
cutoff = time.time() - max_age_days * 86400
|
||||
db.execute("DELETE FROM peer_filters WHERE last_seen < ?", (cutoff,))
|
||||
db.commit()
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_peer_filter_count(self):
|
||||
db = self.get_db()
|
||||
try:
|
||||
return db.execute("SELECT count(*) FROM peer_filters").fetchone()[0]
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
# --- Peer Liveness ---
|
||||
|
||||
def record_sync_result(self, peer_hash, success):
|
||||
db = self.get_db()
|
||||
try:
|
||||
existing = db.execute(
|
||||
"SELECT consecutive_failures FROM synced_instances WHERE instance_hash = ?",
|
||||
(peer_hash,),
|
||||
).fetchone()
|
||||
if existing is not None:
|
||||
new_failures = 0 if success else (existing["consecutive_failures"] + 1)
|
||||
db.execute(
|
||||
"UPDATE synced_instances SET consecutive_failures = ? WHERE instance_hash = ?",
|
||||
(new_failures, peer_hash),
|
||||
)
|
||||
if success:
|
||||
db.execute(
|
||||
"UPDATE synced_instances SET status = 'active' WHERE instance_hash = ?",
|
||||
(peer_hash,),
|
||||
)
|
||||
db.commit()
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
||||
def get_dead_peers(self, max_failures=5):
|
||||
"""Return list of peer hashes with too many consecutive failures."""
|
||||
db = self.get_db()
|
||||
try:
|
||||
return [
|
||||
r["instance_hash"] for r in db.execute(
|
||||
"SELECT instance_hash FROM synced_instances "
|
||||
"WHERE consecutive_failures >= ?",
|
||||
(max_failures,),
|
||||
).fetchall()
|
||||
]
|
||||
finally:
|
||||
self.return_db(db)
|
||||
|
|
|
|||
|
|
@ -16,47 +16,85 @@ def esc(s):
|
|||
return html.escape(str(s))
|
||||
|
||||
|
||||
FORUM_CSS = """
|
||||
from tinyweb_forum.bloom import BloomFilter
|
||||
|
||||
|
||||
FORUM_CSS_DEFAULT = """
|
||||
<style>
|
||||
.forum-form { max-width: 500px; }
|
||||
.forum-form input:not([type=checkbox]):not([type=radio]), .forum-form textarea {
|
||||
width: 100%; box-sizing: border-box; padding: 10px 12px; margin-bottom: 12px;
|
||||
font-size: 0.95rem;
|
||||
}
|
||||
.forum-form input:not([type=checkbox]):not([type=radio]):focus, .forum-form textarea:focus {
|
||||
outline: none;
|
||||
}
|
||||
width: 100%; box-sizing: border-box; margin-bottom: 10px; }
|
||||
.forum-form input[type=checkbox] { width: auto; margin: 0; }
|
||||
.forum-form button {
|
||||
padding: 10px 20px; margin-bottom: 12px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.forum-form textarea { font-family: ui-monospace, 'SF Mono', 'Cascadia Code', 'Segoe UI Mono', Menlo, Consolas, monospace; font-size: 0.85rem; line-height: 1.6; resize: vertical; }
|
||||
.forum-form label.inline-label input { width: 60px; display: inline; }
|
||||
.forum-form label:not(.checkbox-label):not(.inline-label) { display: block; margin-bottom: 8px; }
|
||||
.forum-form button { cursor: pointer; padding: 1px 6px; }
|
||||
.forum-form textarea { line-height: 1.5; resize: vertical; }
|
||||
.forum-form small { display: block; margin-bottom: 6px; font-size: 0.8rem; opacity: 0.7; }
|
||||
.forum-form label { display: block; margin-bottom: 6px; }
|
||||
.forum-form + .forum-form { margin-top: 0.8rem; }
|
||||
.forum-form label.checkbox-label { display: flex; align-items: center; gap: 6px; margin-bottom: 8px; }
|
||||
.forum-form label.inline-label { display: inline-flex; align-items: center; gap: 4px; margin-bottom: 8px; white-space: nowrap; }
|
||||
.forum-form small { display: block; margin-bottom: 8px; }
|
||||
.forum-toolbar { display: flex; flex-wrap: wrap; gap: 10px; align-items: center; margin: 0.5rem 0; }
|
||||
.forum-toolbar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 0.5rem 0; }
|
||||
.forum-toolbar form { flex: 1; min-width: 160px; margin: 0; }
|
||||
.forum-toolbar input[name=q] { width: 100%; box-sizing: border-box; }
|
||||
.forum-toolbar-actions { display: flex; flex-wrap: wrap; gap: 4px; }
|
||||
.forum-status { font-size: 0.82rem; opacity: 0.65; margin: 0 0 0.8rem 0; }
|
||||
.forum-status span { margin-right: 1.2rem; }
|
||||
.forum-nav { margin: 0.5rem 0; }
|
||||
a.forum-action { text-decoration: none; padding: 1px 4px; }
|
||||
a.forum-action:hover, .tag:hover { text-decoration: underline; }
|
||||
.tag { text-decoration: none; }
|
||||
.forum-list { list-style: none; padding: 0; }
|
||||
.forum-list li { padding: 0.6rem 0; }
|
||||
.forum-list li + li { border-top: 1px solid; opacity: 0.25; }
|
||||
.forum-list .thread-title { font-size: 1.05rem; margin-bottom: 0.1rem; }
|
||||
.forum-list .thread-meta { font-size: 0.8rem; opacity: 0.7; }
|
||||
.forum-list .thread-badge { font-size: 0.78rem; opacity: 0.6; }
|
||||
.post { margin-bottom: 1rem; padding-left: 1rem; border-left: 1px solid; opacity: 0.85; }
|
||||
.post-meta { font-size: 0.82rem; opacity: 0.7; }
|
||||
.section { margin: 1.5rem 0; }
|
||||
.section-title { font-weight: bold; margin-bottom: 0.3rem; }
|
||||
.section-desc { font-size: 0.85rem; opacity: 0.7; margin-bottom: 0.5rem; }
|
||||
.section ul { margin: 0.3rem 0; }
|
||||
.section .forum-form { margin-bottom: 0; }
|
||||
.pagination { font-size: 0.85rem; }
|
||||
</style>"""
|
||||
|
||||
FORUM_CSS_KODAMA2 = """
|
||||
<style>
|
||||
.forum-form { max-width: 500px; }
|
||||
.forum-form input:not([type=checkbox]):not([type=radio]), .forum-form textarea {
|
||||
width: 100%; box-sizing: border-box; padding: 10px 12px; margin-bottom: 12px; font-size: 0.95rem;
|
||||
border: 1px solid; border-radius: 6px; }
|
||||
.forum-form input[type=checkbox] { width: auto; margin: 0; }
|
||||
.forum-form button { padding: 10px 20px; cursor: pointer; border: 1px solid; border-radius: 6px; }
|
||||
.forum-form button:hover { opacity: 0.8; }
|
||||
.forum-form textarea { font-size: 0.85rem; line-height: 1.5; resize: vertical; }
|
||||
.forum-form small { display: block; margin-bottom: 6px; font-size: 0.8rem; opacity: 0.7; }
|
||||
.forum-form label { display: block; margin-bottom: 6px; }
|
||||
.forum-form + .forum-form { margin-top: 0.8rem; }
|
||||
.forum-form label.checkbox-label { display: flex; align-items: center; gap: 6px; margin-bottom: 8px; }
|
||||
.forum-toolbar { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 0.5rem 0; padding: 0.6rem 0.8rem; border: 1px solid; border-radius: 8px; opacity: 0.85; }
|
||||
.forum-toolbar form { flex: 1; min-width: 160px; }
|
||||
.forum-toolbar input[name=q] { width: 100%; box-sizing: border-box; padding: 8px 12px; font-size: 0.9rem; }
|
||||
.forum-toolbar input[name=q]:focus { outline: none; }
|
||||
.forum-toolbar-actions { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||
.forum-actions { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; margin: 0.5rem 0; }
|
||||
a.forum-action, a.forum-action-inline {
|
||||
text-decoration: none; border-bottom: none; font-size: 0.88rem;
|
||||
padding: 8px 14px;
|
||||
transition: background 0.2s, color 0.2s, border-color 0.2s;
|
||||
}
|
||||
a.forum-action-inline { padding: 2px 6px; border: none; }
|
||||
a.forum-action-inline:hover { border: none; }
|
||||
p.meta { font-size: 0.85rem; }
|
||||
.forum-list { list-style: none; padding-left: 0; }
|
||||
.forum-list li { padding: 0.8rem 0; border-bottom: 1px solid; }
|
||||
.forum-list li:last-child { border-bottom: none; }
|
||||
.forum-list .thread-title { margin-bottom: 0.15rem; }
|
||||
.forum-list .thread-meta { font-size: 0.78rem; }
|
||||
.forum-list a { border-bottom: none; }
|
||||
.forum-toolbar input[name=q] { width: 100%; box-sizing: border-box; padding: 8px 10px; font-size: 0.9rem; border: 1px solid; border-radius: 6px; }
|
||||
.forum-toolbar-actions { display: flex; flex-wrap: wrap; gap: 4px; }
|
||||
.forum-status { font-size: 0.82rem; opacity: 0.65; margin: 0 0 0.8rem 0; }
|
||||
.forum-status span { margin-right: 1.2rem; }
|
||||
.forum-nav { margin: 0.5rem 0; display: flex; gap: 8px; align-items: center; }
|
||||
a.forum-action { text-decoration: none; font-size: 0.88rem; padding: 6px 14px; border-radius: 6px; }
|
||||
a.forum-action:hover { text-decoration: none; opacity: 0.8; }
|
||||
.tag { text-decoration: none; display: inline-block; padding: 1px 8px; border-radius: 4px; font-size: 0.82rem; }
|
||||
a.tag:hover { opacity: 0.8; }
|
||||
.forum-list { list-style: none; padding: 0; }
|
||||
.forum-list li { padding: 0.8rem; margin-bottom: 0.5rem; border: 1px solid; border-radius: 8px; }
|
||||
.forum-list .thread-title { font-size: 1.05rem; margin-bottom: 0.15rem; }
|
||||
.forum-list .thread-meta { font-size: 0.8rem; opacity: 0.7; }
|
||||
.forum-list .thread-badge { display: inline-block; padding: 1px 6px; border-radius: 4px; font-size: 0.75rem; font-weight: bold; }
|
||||
.post { margin-bottom: 1rem; padding: 0.6rem 0.8rem; border: 1px solid; border-radius: 6px; }
|
||||
.post-meta { font-size: 0.82rem; opacity: 0.7; }
|
||||
.section { margin: 1.5rem 0; padding: 1rem; border: 1px solid; border-radius: 8px; }
|
||||
.section-title { font-weight: bold; font-size: 1.05rem; margin-bottom: 0.5rem; }
|
||||
.section-desc { font-size: 0.85rem; opacity: 0.7; margin-bottom: 0.5rem; }
|
||||
.section ul { margin: 0.3rem 0; }
|
||||
.section .forum-form { margin-bottom: 0; }
|
||||
.pagination { font-size: 0.85rem; }
|
||||
</style>"""
|
||||
|
||||
|
||||
|
|
@ -100,11 +138,16 @@ class ForumHandlers:
|
|||
return ""
|
||||
return f' <a class="forum-action-inline" href="/forum/blockhash/{instance}">[block]</a>'
|
||||
|
||||
def _forum_css(self):
|
||||
theme = self.fdb.get_setting("forum_theme", "default")
|
||||
css = FORUM_CSS_KODAMA2 if theme == "kodama2" else FORUM_CSS_DEFAULT
|
||||
return css
|
||||
|
||||
def _respond(self, body_html, status=200):
|
||||
return {
|
||||
"status": status,
|
||||
"content_type": "text/html; charset=utf-8",
|
||||
"body": FORUM_CSS + body_html,
|
||||
"body": self._forum_css() + body_html,
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
|
|
@ -125,7 +168,7 @@ class ForumHandlers:
|
|||
}
|
||||
|
||||
def _error(self, status):
|
||||
return self._respond(f"<h1>{status}</h1>", status)
|
||||
return self._respond(f"<h2>{status}</h2>", status)
|
||||
|
||||
def _paginate(self, query):
|
||||
try:
|
||||
|
|
@ -175,6 +218,14 @@ class ForumHandlers:
|
|||
return False
|
||||
return (datetime.now() - dt).total_seconds() < RECENT_SECONDS
|
||||
|
||||
def _get_subscribed_topics(self):
|
||||
raw = self.fdb.get_setting("topic_subscriptions", "")
|
||||
return [t.strip().lower() for t in raw.split(",") if t.strip()]
|
||||
|
||||
def _get_subscribed_topics_str(self):
|
||||
raw = self.fdb.get_setting("topic_subscriptions", "")
|
||||
return raw
|
||||
|
||||
def _blocked_instances(self):
|
||||
raw = self.fdb.get_setting("blocked_instances", "")
|
||||
return set(h.strip() for h in raw.split(",") if h.strip())
|
||||
|
|
@ -204,6 +255,19 @@ class ForumHandlers:
|
|||
|
||||
# --- Routes ---
|
||||
|
||||
def _status_bar(self):
|
||||
topics = self._get_subscribed_topics()
|
||||
topics_str = ", ".join(topics) if topics else "everything"
|
||||
peer_count = len(self.fdb.get_synced_instances())
|
||||
filter_count = self.fdb.get_peer_filter_count()
|
||||
return (
|
||||
f'<div class="forum-status">'
|
||||
f'<span>subscribed: {esc(topics_str)}</span>'
|
||||
f'<span>{peer_count} peers</span>'
|
||||
f'<span>{filter_count} filters</span>'
|
||||
f'</div>'
|
||||
)
|
||||
|
||||
def handle_list(self, query):
|
||||
page = self._paginate(query)
|
||||
tag = unquote(query.get("tag", [""])[0]).strip()
|
||||
|
|
@ -224,12 +288,12 @@ class ForumHandlers:
|
|||
continue
|
||||
if self._is_new(r["created_at"]):
|
||||
new_count += 1
|
||||
badge = "[share]" if r["url"] else "[request]"
|
||||
mute_badge = " [muted]" if is_muted else ""
|
||||
badge = f'<span class="thread-badge">share</span>' if r["url"] else f'<span class="thread-badge">request</span>'
|
||||
mute_label = " [muted]" if is_muted else ""
|
||||
tags_html = ""
|
||||
if r["tags"]:
|
||||
tag_links = " ".join(
|
||||
f'<a href="/forum?tag={esc(t.strip())}" class="tag">[{esc(t.strip())}]</a>'
|
||||
f'<a href="/forum?tag={esc(t.strip())}" class="tag">{esc(t.strip())}</a>'
|
||||
for t in r["tags"].split(",") if t.strip()
|
||||
)
|
||||
tags_html = f' {tag_links}'
|
||||
|
|
@ -237,7 +301,7 @@ class ForumHandlers:
|
|||
items += (
|
||||
f'<li>'
|
||||
f'<div class="thread-title">'
|
||||
f'<small>{badge}{mute_badge}</small> '
|
||||
f'{badge}{mute_label} '
|
||||
f'<a href="/forum/t/{esc(r["id"])}">{esc(r["title"])}</a>'
|
||||
f'{tags_html}'
|
||||
f'</div>'
|
||||
|
|
@ -250,18 +314,19 @@ class ForumHandlers:
|
|||
f'</li>'
|
||||
)
|
||||
if not items:
|
||||
items = "<p>No threads yet.</p>"
|
||||
items = "<p>no threads yet.</p>"
|
||||
new_label = f" ({new_count} new)" if new_count else ""
|
||||
search_form = (
|
||||
f'<form method="get" action="/forum">'
|
||||
f'<input name="q" placeholder="search" value="{esc(search)}">'
|
||||
f'</form>'
|
||||
)
|
||||
tag_label = f' — tag: {esc(tag)}' if tag else ""
|
||||
tag_label = f' — {esc(tag)}' if tag else ""
|
||||
muted_link = f'<a class="forum-action" href="/forum?muted=1">show muted</a>' if not show_muted else f'<a class="forum-action" href="/forum">show all</a>'
|
||||
page_url = f'/forum?q={esc(search)}&tag={esc(tag)}&muted=1' if show_muted else (f'/forum?q={esc(search)}&tag={esc(tag)}' if search or tag else '/forum')
|
||||
return self._respond(
|
||||
f"<h1>forum{tag_label}</h1>"
|
||||
f"<h2>forum{tag_label}</h2>"
|
||||
f'{self._status_bar()}'
|
||||
f'<div class="forum-toolbar">'
|
||||
f'{search_form}'
|
||||
f'<div class="forum-toolbar-actions">'
|
||||
|
|
@ -270,26 +335,30 @@ class ForumHandlers:
|
|||
f'<a class="forum-action" href="/forum/sync/now">sync now</a>'
|
||||
f'{muted_link}'
|
||||
f'</div></div>'
|
||||
f"<p class=\"meta\">{total} threads{new_label}</p>"
|
||||
f"<p>{total} threads{new_label}</p>"
|
||||
f'<ul class="forum-list">{items}</ul>'
|
||||
f"{self._page_nav(page, total, page_url)}"
|
||||
)
|
||||
|
||||
def handle_new_form(self, msg=""):
|
||||
return self._respond(
|
||||
f"<h1>new thread</h1>"
|
||||
f"<h2>new thread</h2>"
|
||||
f'<form class="forum-form" method="post" action="/forum/new">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<label>title</label>'
|
||||
f'<input name="title" placeholder="title" required>'
|
||||
f"<small>max {MAX_TITLE_LENGTH} characters</small>"
|
||||
f'<label>URL</label>'
|
||||
f'<input name="url" placeholder="URL you want to share (optional)">'
|
||||
f'<label>body</label>'
|
||||
f'<textarea name="body" rows="6" placeholder="details or context (optional)"></textarea>'
|
||||
f"<small>max {MAX_BODY_LENGTH} characters</small>"
|
||||
f'<input name="tags" placeholder="tags, comma-separated (optional)">'
|
||||
f'<label>tags</label>'
|
||||
f'<input name="tags" placeholder="comma-separated (optional)">'
|
||||
f'<button type="submit">post</button>'
|
||||
f"</form>"
|
||||
f"<p>{msg}</p>"
|
||||
f'<a href="/forum">back</a>'
|
||||
f'<div class="forum-nav"><a href="/forum">back</a></div>'
|
||||
)
|
||||
|
||||
def handle_new_submit(self, body):
|
||||
|
|
@ -323,28 +392,25 @@ class ForumHandlers:
|
|||
instance_hash = self.identity.hash.hex() if self.identity else "local"
|
||||
has_upvoted = self.fdb.has_upvoted(thread_id, instance_hash)
|
||||
|
||||
badge = "[share]" if thread["url"] else "[request]"
|
||||
badge = f'<span class="thread-badge">share</span>' if thread["url"] else f'<span class="thread-badge">request</span>'
|
||||
url_html = ""
|
||||
if thread["url"]:
|
||||
url_html = (
|
||||
f'<p><a href="{esc(thread["url"])}" rel="noreferrer noopener">{esc(thread["url"])}</a>'
|
||||
f' (<a href="/add?url={esc(thread["url"])}">+ save to my index</a>)</p>'
|
||||
f' (<a href="/add?url={esc(thread["url"])}">+ save</a>)</p>'
|
||||
)
|
||||
tags_html = ""
|
||||
if thread["tags"]:
|
||||
tag_links = " ".join(
|
||||
f'<a href="/forum?tag={esc(t.strip())}" class="tag">[{esc(t.strip())}]</a>'
|
||||
f'<a href="/forum?tag={esc(t.strip())}" class="tag">{esc(t.strip())}</a>'
|
||||
for t in thread["tags"].split(",") if t.strip()
|
||||
)
|
||||
tags_html = f'<p class="tags">{tag_links}</p>'
|
||||
tags_html = f'<p>{tag_links}</p>'
|
||||
|
||||
body_html = f"<p>{esc(thread['body'])}</p>" if thread["body"] else ""
|
||||
|
||||
mute_btn = (
|
||||
f'<a class="forum-action-inline" href="/forum/unmute/{thread["id"]}">unmute</a>'
|
||||
if is_muted else
|
||||
f'<a class="forum-action-inline" href="/forum/mute/{thread["id"]}">mute</a>'
|
||||
)
|
||||
mute_label = "unmute" if is_muted else "mute"
|
||||
mute_href = f'/forum/unmute/{thread["id"]}' if is_muted else f'/forum/mute/{thread["id"]}'
|
||||
|
||||
posts_html = ""
|
||||
for p in posts:
|
||||
|
|
@ -352,18 +418,18 @@ class ForumHandlers:
|
|||
for word in p["body"].split():
|
||||
w = word.strip().strip(",.!?;:")
|
||||
if w.startswith(("http://", "https://")):
|
||||
save_links += (
|
||||
f' <a href="/add?url={esc(w)}">+ save</a>'
|
||||
)
|
||||
save_links += f' <a href="/add?url={esc(w)}">+ save</a>'
|
||||
parent_ref = ""
|
||||
if p["parent_id"]:
|
||||
parent_ref = f' <small><a href="#post-{esc(p["parent_id"])}">↪ reply</a></small>'
|
||||
parent_ref = f' <a href="#post-{esc(p["parent_id"])}">↪ reply</a>'
|
||||
posts_html += (
|
||||
f'<div class="post" id="post-{esc(p["id"])}" style="margin-bottom:1rem;padding-left:1rem;border-left:2px solid">'
|
||||
f'<small><b>{esc(self._author_str(p["author_name"], p["author_instance"]))}</b>'
|
||||
f'<div class="post" id="post-{esc(p["id"])}">'
|
||||
f'<div class="post-meta">'
|
||||
f'<b>{esc(self._author_str(p["author_name"], p["author_instance"]))}</b>'
|
||||
f'{self._block_link(p["author_instance"])}'
|
||||
f' · {self._time_ago(p["created_at"])}{parent_ref}'
|
||||
f'{" · " + self._post_retract_link(thread["id"], p["id"]) if p["author_instance"] == instance_hash else ""}</small>'
|
||||
f'{" · " + self._post_retract_link(thread["id"], p["id"]) if p["author_instance"] == instance_hash else ""}'
|
||||
f'</div>'
|
||||
f'<p>{esc(p["body"])}</p>'
|
||||
f'{save_links}'
|
||||
f'</div>'
|
||||
|
|
@ -378,15 +444,16 @@ class ForumHandlers:
|
|||
f"</form>"
|
||||
)
|
||||
|
||||
upvote_label = "-1" if has_upvoted else "+1"
|
||||
return self._respond(
|
||||
f"<h1>{badge} {esc(thread['title'])}</h1>"
|
||||
f'<p class="meta">'
|
||||
f"<h2>{badge} {esc(thread['title'])}</h2>"
|
||||
f'<p>'
|
||||
f'by {esc(self._author_str(thread["author_name"], thread["author_instance"]))}'
|
||||
f'{self._block_link(thread["author_instance"])}'
|
||||
f' · {self._time_ago(thread["created_at"])}'
|
||||
f' · {thread["score"]} upvotes'
|
||||
f' · {mute_btn}'
|
||||
f' · <a class="forum-action-inline" href="/forum/t/{thread["id"]}/upvote">{"-1" if has_upvoted else "+1"}</a>'
|
||||
f' · <a href="{mute_href}">{mute_label}</a>'
|
||||
f' · <a href="/forum/t/{thread["id"]}/upvote">{upvote_label}</a>'
|
||||
f'{self._author_links(thread["id"], thread["author_instance"], instance_hash)}'
|
||||
f'</p>'
|
||||
f'{url_html}'
|
||||
|
|
@ -428,7 +495,7 @@ class ForumHandlers:
|
|||
if thread["author_instance"] != instance_hash:
|
||||
return self._error(403)
|
||||
return self._respond(
|
||||
f"<h1>edit thread</h1>"
|
||||
f"<h2>edit thread</h2>"
|
||||
f'<form class="forum-form" method="post" action="/forum/t/{thread_id}/edit">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<input name="title" value="{esc(thread["title"])}" required>'
|
||||
|
|
@ -511,7 +578,7 @@ class ForumHandlers:
|
|||
def _peer_reports_html(self):
|
||||
counts = self.fdb.get_peer_block_counts()
|
||||
if not counts:
|
||||
return "<p>No peer reports yet.</p>"
|
||||
return "<p>no peer reports yet</p>"
|
||||
auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip())
|
||||
blocked = self._blocked_instances()
|
||||
items = ""
|
||||
|
|
@ -524,13 +591,24 @@ class ForumHandlers:
|
|||
blocked = self._blocked_instances()
|
||||
auto_blocked = set(h.strip() for h in self.fdb.get_setting("auto_blocked_instances", "").split(",") if h.strip())
|
||||
peer_counts = self.fdb.get_peer_block_counts()
|
||||
filters = self._keyword_filters()
|
||||
filters_str = ", ".join(filters) if filters else ""
|
||||
synced = self.fdb.get_synced_instances()
|
||||
auto_discover = self.fdb.get_setting("forum_auto_discover", "1")
|
||||
auto_discover_checked = " checked" if auto_discover == "1" else ""
|
||||
auto_sync = self.fdb.get_setting("forum_auto_sync", "0")
|
||||
auto_sync_checked = " checked" if auto_sync == "1" else ""
|
||||
retention_days = self.fdb.get_setting("forum_retention_days", "30")
|
||||
|
||||
blocked_items = ""
|
||||
if blocked:
|
||||
for h in sorted(blocked):
|
||||
label = "[auto] " if h in auto_blocked else ""
|
||||
reports = f" ({peer_counts.get(h, 0)} peers)" if h in peer_counts else ""
|
||||
label = "auto" if h in auto_blocked else ""
|
||||
reports = f" ({peer_counts.get(h, 0)} reports)" if h in peer_counts else ""
|
||||
blocked_items += (
|
||||
f'<li>{label}{esc(h[:16])}...{reports} '
|
||||
f'<li>'
|
||||
f'{esc(h[:16])}...'
|
||||
f'{" [" + label + "]" if label else ""}{reports} '
|
||||
f'<form method="post" action="/forum/unblock" style="display:inline">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<input type="hidden" name="instance" value="{esc(h)}">'
|
||||
|
|
@ -539,12 +617,8 @@ class ForumHandlers:
|
|||
)
|
||||
blocked_items = f"<ul>{blocked_items}</ul>"
|
||||
else:
|
||||
blocked_items = "<p>No instances blocked.</p>"
|
||||
blocked_items = "<p>no instances blocked</p>"
|
||||
|
||||
filters = self._keyword_filters()
|
||||
filters_str = ", ".join(filters) if filters else ""
|
||||
|
||||
synced = self.fdb.get_synced_instances()
|
||||
synced_items = ""
|
||||
for s in synced:
|
||||
synced_items += (
|
||||
|
|
@ -555,67 +629,78 @@ class ForumHandlers:
|
|||
f'<button>remove</button></form>'
|
||||
f'</li>'
|
||||
)
|
||||
synced_items = f"<ul>{synced_items}</ul>" if synced_items else "<p>No instances synced yet.</p>"
|
||||
|
||||
auto_discover = self.fdb.get_setting("forum_auto_discover", "1")
|
||||
auto_discover_checked = " checked" if auto_discover == "1" else ""
|
||||
auto_sync = self.fdb.get_setting("forum_auto_sync", "0")
|
||||
auto_sync_checked = " checked" if auto_sync == "1" else ""
|
||||
retention_days = self.fdb.get_setting("forum_retention_days", "30")
|
||||
synced_items = f"<ul>{synced_items}</ul>" if synced_items else "<p>no instances synced</p>"
|
||||
|
||||
return self._respond(
|
||||
f"<h1>forum moderation</h1>"
|
||||
f"<h2>moderation</h2>"
|
||||
f"<p>{msg}</p>"
|
||||
f'<p><a class="forum-action" href="/forum/sync/now">sync now</a></p>'
|
||||
f"<h2>auto-discovery</h2>"
|
||||
f'{self._status_bar()}'
|
||||
|
||||
f'<div class="section">'
|
||||
f'<div class="section-title">subscriptions</div>'
|
||||
f'<form class="forum-form" method="post" action="/forum/topics">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<input name="topics" value="{esc(self._get_subscribed_topics_str())}" placeholder="comma-separated topics (leave empty for all)">'
|
||||
f"<small>only sync content matching these topics</small>"
|
||||
f'<button>save</button>'
|
||||
f"</form>"
|
||||
f'</div>'
|
||||
|
||||
f'<div class="section">'
|
||||
f'<div class="section-title">settings</div>'
|
||||
f'<div class="section-desc">network behavior</div>'
|
||||
f'<form class="forum-form" method="post" action="/forum/auto_discover">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<label class="checkbox-label"><input type="checkbox" name="enabled" value="1"{auto_discover_checked}>'
|
||||
f" automatically discover other forum instances on the mesh</label>"
|
||||
f" auto-discover peers via announces</label>"
|
||||
f'<button>save</button>'
|
||||
f"</form>"
|
||||
f"<h2>auto-sync</h2>"
|
||||
f'<form class="forum-form" method="post" action="/forum/auto_sync">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<label class="checkbox-label"><input type="checkbox" name="enabled" value="1"{auto_sync_checked}>'
|
||||
f" automatically sync content every 5 minutes</label>"
|
||||
f" auto-sync every 5 minutes</label>"
|
||||
f'<button>save</button>'
|
||||
f"</form>"
|
||||
f"<h2>storage</h2>"
|
||||
f'<form class="forum-form" method="post" action="/forum/storage">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<label class="inline-label">Keep threads for '
|
||||
f'<input name="retention_days" value="{esc(retention_days)}" size="5"> days</label>'
|
||||
f"<small>Older threads are pruned automatically (default: 30). Set to 0 to keep everything.</small>"
|
||||
f'<label>keep threads for '
|
||||
f'<input name="retention_days" value="{esc(retention_days)}" size="5" style="width:50px">'
|
||||
f' days (0 = keep everything)</label>'
|
||||
f'<button>save</button>'
|
||||
f"</form>"
|
||||
f"<h2>blocked instances</h2>"
|
||||
f"{blocked_items}"
|
||||
f'<form class="forum-form" method="post" action="/forum/block">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<input name="instance" placeholder="instance hash (32 hex chars)">'
|
||||
f'<button>block</button>'
|
||||
f"</form>"
|
||||
f"<h2>peer reports</h2>"
|
||||
f"{self._peer_reports_html()}"
|
||||
f"<h2>keyword filters</h2>"
|
||||
f'<form class="forum-form" method="post" action="/forum/filters">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<input name="keywords" value="{esc(filters_str)}" placeholder="comma-separated keywords">'
|
||||
f'<button>save</button>'
|
||||
f"</form>"
|
||||
f"<h2>synced instances</h2>"
|
||||
f"{synced_items}"
|
||||
f"<p><small>Instances are discovered automatically via mesh announces. "
|
||||
f"You can also manually add a friend's instance hash to bootstrap.</small></p>"
|
||||
f'</div>'
|
||||
|
||||
f'<div class="section">'
|
||||
f'<div class="section-title">network</div>'
|
||||
f'<div class="section-desc">{len(synced)} known peers</div>'
|
||||
f'{synced_items}'
|
||||
f'<form class="forum-form" method="post" action="/forum/sync/add">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<input name="instance" placeholder="instance hash">'
|
||||
f'<input name="name" placeholder="label (optional)">'
|
||||
f'<button>add</button>'
|
||||
f"</form>"
|
||||
f'<hr>'
|
||||
f'<a href="/forum">back to forum</a>'
|
||||
f'</div>'
|
||||
|
||||
f'<div class="section">'
|
||||
f'<div class="section-title">moderation</div>'
|
||||
f'{blocked_items}'
|
||||
f'<form class="forum-form" method="post" action="/forum/block">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<input name="instance" placeholder="instance hash (32 hex chars)">'
|
||||
f'<button>block</button>'
|
||||
f"</form>"
|
||||
f'<div class="section-title">peer reports</div>'
|
||||
f'{self._peer_reports_html()}'
|
||||
f'<div class="section-title">keyword filters</div>'
|
||||
f'<form class="forum-form" method="post" action="/forum/filters">'
|
||||
f'{self._csrf_field()}'
|
||||
f'<input name="keywords" value="{esc(filters_str)}" placeholder="comma-separated keywords">'
|
||||
f'<button>save</button>'
|
||||
f"</form>"
|
||||
f'</div>'
|
||||
|
||||
f'<div class="forum-nav"><a href="/forum">back</a></div>'
|
||||
)
|
||||
|
||||
def handle_block(self, body):
|
||||
|
|
@ -702,31 +787,122 @@ class ForumHandlers:
|
|||
self.fdb.remove_synced_instance(instance)
|
||||
return self.handle_moderation("Removed.")
|
||||
|
||||
def handle_topics(self, body):
|
||||
topics = body.get("topics", [""])[0].strip()
|
||||
self.fdb.set_setting("topic_subscriptions", topics)
|
||||
return self.handle_moderation("Topic subscriptions saved.")
|
||||
|
||||
# --- Sync endpoint (called over RNS) ---
|
||||
|
||||
def handle_sync_request(self, data):
|
||||
"""Handle incoming sync request from another forum instance."""
|
||||
since = data.get("query", {}).get("since", [""])[0] if isinstance(data.get("query"), dict) else ""
|
||||
incoming_threads = data.get("threads", [])
|
||||
incoming_posts = data.get("posts", [])
|
||||
incoming_upvotes = data.get("upvotes", [])
|
||||
peer_topics = data.get("my_topics", [])
|
||||
peer_tag_cloud = data.get("my_tag_cloud", [])
|
||||
peer_bloom_data = data.get("content_bloom")
|
||||
peer_tag_bloom_data = data.get("tag_bloom")
|
||||
peer_filter_table = data.get("filter_table", {})
|
||||
scoped_query_tag = data.get("scoped_query", "")
|
||||
|
||||
blocked = self._blocked_instances()
|
||||
my_topics = self._get_subscribed_topics()
|
||||
from_hash = data.get("from_hash", "")
|
||||
|
||||
# Store peer's tag bloom filter (Architecture B discovery)
|
||||
if peer_tag_bloom_data and from_hash:
|
||||
self.fdb.store_peer_filter(
|
||||
peer_hash=from_hash,
|
||||
bloom_bytes=bytes(peer_tag_bloom_data),
|
||||
bloom_size=data.get("tag_bloom_size", 2048),
|
||||
bloom_hashes=data.get("tag_bloom_hashes", 3),
|
||||
tag_count=len(peer_topics),
|
||||
)
|
||||
|
||||
# Merge filter table gossip (transitive peer discovery)
|
||||
if peer_filter_table and from_hash:
|
||||
for ph, entry in peer_filter_table.items():
|
||||
if isinstance(entry, dict):
|
||||
bb = entry.get("bloom_bytes")
|
||||
if bb:
|
||||
self.fdb.store_peer_filter(
|
||||
peer_hash=ph,
|
||||
bloom_bytes=bytes(bb) if isinstance(bb, list) else bb,
|
||||
bloom_size=entry.get("bloom_size", 2048),
|
||||
bloom_hashes=entry.get("bloom_hashes", 3),
|
||||
tag_count=entry.get("tag_count", 0),
|
||||
)
|
||||
|
||||
# Handle scoped query: find peers whose bloom might contain the queried tag
|
||||
scoped_query_results = []
|
||||
if scoped_query_tag and from_hash:
|
||||
scoped_query_results = self.fdb.get_filtered_peers_by_tag(scoped_query_tag)
|
||||
|
||||
# Store peer's topics for future routing (backward compat)
|
||||
if peer_topics and from_hash:
|
||||
self.fdb.set_setting(f"peer_topics_{from_hash}", ",".join(peer_topics))
|
||||
if peer_tag_cloud and from_hash:
|
||||
self.fdb.set_setting(f"peer_tag_cloud_{from_hash}", json.dumps(peer_tag_cloud))
|
||||
|
||||
# Build our tag bloom filter to send back
|
||||
peer_tag_bs = data.get("tag_bloom_size", 2048)
|
||||
peer_tag_bh = data.get("tag_bloom_hashes", 3)
|
||||
my_tag_bloom = BloomFilter.from_tags(my_topics or [], peer_tag_bs, peer_tag_bh)
|
||||
|
||||
# Build our content bloom filter for dedup
|
||||
our_existing = set()
|
||||
for t in self.fdb.get_threads_by_topics(peer_topics) if peer_topics else []:
|
||||
our_existing.add(t["id"])
|
||||
our_bloom = BloomFilter.from_items(list(our_existing), data.get("bloom_size", 2048) if data else 2048, data.get("bloom_hashes", 3) if data else 3)
|
||||
|
||||
# Build filter table gossip from our stored filters
|
||||
all_filters = self.fdb.get_all_filters()
|
||||
filter_table_gossip = {}
|
||||
for f in all_filters[:20]:
|
||||
if f["peer_hash"] != from_hash:
|
||||
filter_table_gossip[f["peer_hash"]] = {
|
||||
"bloom_bytes": list(f["bloom_bytes"]),
|
||||
"bloom_size": f["bloom_size"],
|
||||
"bloom_hashes": f["bloom_hashes"],
|
||||
"tag_count": f["tag_count"],
|
||||
}
|
||||
|
||||
# Parse peer's content bloom for dedup
|
||||
peer_bloom = None
|
||||
if peer_bloom_data:
|
||||
peer_bloom = BloomFilter.from_bytes(
|
||||
bytes(peer_bloom_data),
|
||||
data.get("bloom_size", 2048),
|
||||
data.get("bloom_hashes", 3),
|
||||
)
|
||||
|
||||
# Merge incoming content, filtered by bloom and topics
|
||||
if incoming_threads:
|
||||
for t in incoming_threads:
|
||||
if t.get("author_instance", "") not in blocked:
|
||||
if t.get("author_instance", "") in blocked:
|
||||
continue
|
||||
if peer_bloom and peer_bloom.might_contain(t["id"]):
|
||||
continue
|
||||
if not my_topics:
|
||||
self.fdb.merge_thread(t)
|
||||
else:
|
||||
t_tags = [tag.strip().lower() for tag in t.get("tags", "").split(",") if tag.strip()]
|
||||
if set(my_topics) & set(t_tags):
|
||||
self.fdb.merge_thread(t)
|
||||
if incoming_posts:
|
||||
for p in incoming_posts:
|
||||
if p.get("author_instance", "") not in blocked:
|
||||
if p.get("author_instance", "") in blocked:
|
||||
continue
|
||||
if peer_bloom and peer_bloom.might_contain(p["id"]):
|
||||
continue
|
||||
self.fdb.merge_post(p)
|
||||
if incoming_upvotes:
|
||||
for uv in incoming_upvotes:
|
||||
self.fdb.merge_upvote(uv["thread_id"], uv["instance_hash"])
|
||||
|
||||
# Record incoming peer blocks
|
||||
incoming_blocks = data.get("blocks", {})
|
||||
peer_hash = data.get("peer_hash", "") or data.get("from_hash", "")
|
||||
peer_hash = data.get("peer_hash", "") or from_hash
|
||||
if incoming_blocks and peer_hash:
|
||||
for h in incoming_blocks.get("mine", []):
|
||||
if h and h not in blocked:
|
||||
|
|
@ -735,13 +911,10 @@ class ForumHandlers:
|
|||
if h and h not in blocked:
|
||||
self.fdb.record_peer_block(peer_hash, h)
|
||||
|
||||
# Merge incoming retractions
|
||||
for r in data.get("retractions", []):
|
||||
if r.get("id") and r.get("type") and r.get("author") and r.get("at"):
|
||||
self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"])
|
||||
|
||||
# Auto-discover the peer that synced with us and their known peers
|
||||
from_hash = data.get("from_hash", "")
|
||||
if from_hash and from_hash not in blocked:
|
||||
self.fdb.add_known_peer(from_hash)
|
||||
for peer_hash in data.get("known_peers", []):
|
||||
|
|
@ -750,18 +923,34 @@ class ForumHandlers:
|
|||
|
||||
my_blocks = list(blocked)
|
||||
my_peer_blocks = self.fdb.get_peer_block_list()
|
||||
my_tag_cloud = self.fdb.get_tag_cloud(50)
|
||||
my_tag_list = [t for t, _ in my_tag_cloud]
|
||||
|
||||
threads, posts, upvote_threads = [], [], []
|
||||
if since:
|
||||
if peer_topics:
|
||||
rows = self.fdb.get_threads_by_topics(peer_topics, since=since)
|
||||
threads = [dict(r) for r in rows]
|
||||
tids = [r["id"] for r in rows]
|
||||
posts = [dict(p) for p in self.fdb.get_posts_by_thread_ids(tids)]
|
||||
uv_rows = self.fdb.get_new_upvotes_since(since, tids)
|
||||
upvote_threads = uv_rows
|
||||
else:
|
||||
ts, posts_list, up_list = self.fdb.get_new_content(since)
|
||||
threads = [dict(r) for r in ts]
|
||||
posts = [dict(r) for r in posts_list]
|
||||
upvote_threads = up_list
|
||||
|
||||
known_peers = [h for h in self.fdb.get_all_known_hashes() if h != from_hash]
|
||||
peer_topics_map = {}
|
||||
for ph in known_peers[:100]:
|
||||
pt = self.fdb.get_setting(f"peer_topics_{ph}", "")
|
||||
if pt:
|
||||
peer_topics_map[ph] = [t.strip() for t in pt.split(",") if t.strip()]
|
||||
|
||||
retracted = [{"id": cid, "type": ct, "author": ai, "at": ra}
|
||||
for cid, ct, ai, ra in self.fdb.get_raw_retractions()]
|
||||
|
||||
known_peers = [h for h in self.fdb.get_all_known_hashes() if h != from_hash]
|
||||
|
||||
return {
|
||||
"status": 200,
|
||||
"content_type": "application/json",
|
||||
|
|
@ -772,6 +961,18 @@ class ForumHandlers:
|
|||
"blocks": {"mine": my_blocks, "peers": my_peer_blocks},
|
||||
"retractions": retracted,
|
||||
"known_peers": known_peers,
|
||||
"peer_topics": my_tag_list,
|
||||
"peer_tag_cloud": my_tag_cloud,
|
||||
"content_bloom": list(our_bloom.bytes),
|
||||
"bloom_size": data.get("bloom_size", 2048),
|
||||
"bloom_hashes": data.get("bloom_hashes", 3),
|
||||
"peer_topics_map": peer_topics_map,
|
||||
# Architecture B additions
|
||||
"tag_bloom": list(my_tag_bloom.bytes),
|
||||
"tag_bloom_size": peer_tag_bs,
|
||||
"tag_bloom_hashes": peer_tag_bh,
|
||||
"filter_table": filter_table_gossip,
|
||||
"scoped_query_results": scoped_query_results,
|
||||
}),
|
||||
"headers": {},
|
||||
}
|
||||
|
|
@ -832,7 +1033,7 @@ class ForumHandlers:
|
|||
elif method == "POST":
|
||||
if not self._check_csrf(body):
|
||||
return self._with_csrf(
|
||||
self._respond("<h1>403 Forbidden</h1>", status=403), csrf_token
|
||||
self._respond("<h2>403 Forbidden</h2>", status=403), csrf_token
|
||||
)
|
||||
if sub == "/new":
|
||||
return self._with_csrf(self.handle_new_submit(body), csrf_token)
|
||||
|
|
@ -867,6 +1068,8 @@ class ForumHandlers:
|
|||
return self._with_csrf(self.handle_storage(body), csrf_token)
|
||||
elif sub == "/auto_sync":
|
||||
return self._with_csrf(self.handle_auto_sync(body), csrf_token)
|
||||
elif sub == "/topics":
|
||||
return self._with_csrf(self.handle_topics(body), csrf_token)
|
||||
|
||||
return self._with_csrf(self._error(404), csrf_token)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,15 +4,22 @@ import threading
|
|||
import time
|
||||
import RNS
|
||||
|
||||
from tinyweb_forum.bloom import BloomFilter
|
||||
|
||||
FORUM_APP = "tinyweb-forum"
|
||||
SYNC_INTERVAL = 300 # 5 minutes
|
||||
SYNC_INTERVAL = 300
|
||||
REQUEST_TIMEOUT = 60
|
||||
GOSSIP_FANOUT = 20 # random peers to sync per cycle
|
||||
GOSSIP_FANOUT = 20
|
||||
BLOOM_SIZE = 2048
|
||||
BLOOM_HASHES = 3
|
||||
TAG_BLOOM_SIZE = 2048
|
||||
TAG_BLOOM_HASHES = 3
|
||||
FILTER_TABLE_GOSSIP = 20
|
||||
MAX_PEER_FAILURES = 5
|
||||
FILTER_TABLE_TTL_DAYS = 7
|
||||
|
||||
|
||||
class _ForumAnnounceHandler:
|
||||
"""Receives announces from other forum instances and auto-discovers them."""
|
||||
|
||||
aspect_filter = FORUM_APP
|
||||
receive_path_responses = False
|
||||
|
||||
|
|
@ -39,6 +46,26 @@ class ForumSync:
|
|||
self._running = False
|
||||
self._thread = None
|
||||
|
||||
def _get_subscribed_topics(self):
|
||||
raw = self.fdb.get_setting("topic_subscriptions", "")
|
||||
return [t.strip().lower() for t in raw.split(",") if t.strip()]
|
||||
|
||||
def _build_tag_bloom(self, topics=None):
|
||||
if topics is None:
|
||||
topics = self._get_subscribed_topics()
|
||||
return BloomFilter.from_tags(topics or [], TAG_BLOOM_SIZE, TAG_BLOOM_HASHES)
|
||||
|
||||
def _topics_overlap(self, my_topics, their_topics):
|
||||
if not my_topics or not their_topics:
|
||||
return True
|
||||
return bool(set(my_topics) & set(their_topics))
|
||||
|
||||
def _topics_overlap_bloom(self, my_topics, peer_bloom):
|
||||
"""Check overlap using bloom filter instead of topic list."""
|
||||
if not my_topics or peer_bloom is None:
|
||||
return True
|
||||
return any(peer_bloom.might_contain(t) for t in my_topics)
|
||||
|
||||
def start(self):
|
||||
self.destination = RNS.Destination(
|
||||
self.identity,
|
||||
|
|
@ -66,23 +93,48 @@ class ForumSync:
|
|||
self.fdb.set_setting("forum_auto_sync", "1" if enabled else "0")
|
||||
if enabled:
|
||||
self._start_sync_loop()
|
||||
else:
|
||||
pass # current cycle finishes, no new one starts
|
||||
|
||||
def sync_now(self):
|
||||
"""Run one sync cycle immediately. Returns count of peers synced."""
|
||||
instances = self.fdb.get_synced_instances()
|
||||
random.shuffle(instances)
|
||||
count = 0
|
||||
my_topics = self._get_subscribed_topics()
|
||||
my_tag_bloom = self._build_tag_bloom(my_topics)
|
||||
for inst in instances[:GOSSIP_FANOUT]:
|
||||
if not self._running:
|
||||
break
|
||||
pfilter = self.fdb.get_peer_filter(inst["instance_hash"])
|
||||
if pfilter:
|
||||
pbloom = BloomFilter.from_bytes(pfilter["bloom_bytes"], pfilter["bloom_size"], pfilter["bloom_hashes"])
|
||||
if not self._topics_overlap_bloom(my_topics, pbloom):
|
||||
continue
|
||||
else:
|
||||
peer_topics = self._peer_tag_topics(inst["instance_hash"])
|
||||
if not self._topics_overlap(my_topics, peer_topics):
|
||||
continue
|
||||
try:
|
||||
self._sync_with(inst["instance_hash"], my_topics, my_tag_bloom)
|
||||
count += 1
|
||||
except Exception as e:
|
||||
self.fdb.record_sync_result(inst["instance_hash"], False)
|
||||
print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}")
|
||||
return count
|
||||
|
||||
def scoped_query(self, tag):
|
||||
"""Ask known peers if they know anyone with the given tag."""
|
||||
tag = tag.strip().lower()
|
||||
instances = self.fdb.get_synced_instances()
|
||||
random.shuffle(instances)
|
||||
results = set()
|
||||
for inst in instances[:GOSSIP_FANOUT]:
|
||||
if not self._running:
|
||||
break
|
||||
try:
|
||||
self._sync_with(inst["instance_hash"])
|
||||
count += 1
|
||||
except Exception as e:
|
||||
print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}")
|
||||
return count
|
||||
matches = self._scoped_query_peer(inst["instance_hash"], tag)
|
||||
results.update(matches)
|
||||
except Exception:
|
||||
continue
|
||||
return list(results)
|
||||
|
||||
def _start_sync_loop(self):
|
||||
if self._thread and self._thread.is_alive():
|
||||
|
|
@ -120,16 +172,25 @@ class ForumSync:
|
|||
while self._running:
|
||||
try:
|
||||
instances = self.fdb.get_synced_instances()
|
||||
# Gossip: sync with random subset for scaling
|
||||
# If <= GOSSIP_FANOUT peers, sync with all (current behavior)
|
||||
# If more, sync with random FANOUT per cycle — content spreads epidemically
|
||||
my_topics = self._get_subscribed_topics()
|
||||
my_tag_bloom = self._build_tag_bloom(my_topics)
|
||||
random.shuffle(instances)
|
||||
for inst in instances[:GOSSIP_FANOUT]:
|
||||
if not self._running:
|
||||
break
|
||||
pfilter = self.fdb.get_peer_filter(inst["instance_hash"])
|
||||
if pfilter:
|
||||
pbloom = BloomFilter.from_bytes(pfilter["bloom_bytes"], pfilter["bloom_size"], pfilter["bloom_hashes"])
|
||||
if not self._topics_overlap_bloom(my_topics, pbloom):
|
||||
continue
|
||||
else:
|
||||
peer_topics = self._peer_tag_topics(inst["instance_hash"])
|
||||
if not self._topics_overlap(my_topics, peer_topics):
|
||||
continue
|
||||
try:
|
||||
self._sync_with(inst["instance_hash"])
|
||||
self._sync_with(inst["instance_hash"], my_topics, my_tag_bloom)
|
||||
except Exception as e:
|
||||
self.fdb.record_sync_result(inst["instance_hash"], False)
|
||||
print(f"[forum] sync error with {inst['instance_hash'][:16]}: {e}")
|
||||
except Exception:
|
||||
pass
|
||||
|
|
@ -141,12 +202,99 @@ class ForumSync:
|
|||
self.fdb.prune_old_content(days)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.fdb.prune_peer_filters(FILTER_TABLE_TTL_DAYS)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
for dead in self.fdb.get_dead_peers(MAX_PEER_FAILURES):
|
||||
self.fdb.remove_synced_instance(dead)
|
||||
except Exception:
|
||||
pass
|
||||
for _ in range(SYNC_INTERVAL):
|
||||
if not self._running:
|
||||
return
|
||||
time.sleep(1)
|
||||
|
||||
def _sync_with(self, instance_hash):
|
||||
def _peer_tag_topics(self, instance_hash):
|
||||
raw = self.fdb.get_setting(f"peer_topics_{instance_hash}", "")
|
||||
return [t.strip().lower() for t in raw.split(",") if t.strip()] if raw else []
|
||||
|
||||
def _store_peer_filter(self, peer_hash, data):
|
||||
tag_bloom_data = data.get("tag_bloom")
|
||||
if tag_bloom_data:
|
||||
self.fdb.store_peer_filter(
|
||||
peer_hash=peer_hash,
|
||||
bloom_bytes=bytes(tag_bloom_data),
|
||||
bloom_size=data.get("tag_bloom_size", TAG_BLOOM_SIZE),
|
||||
bloom_hashes=data.get("tag_bloom_hashes", TAG_BLOOM_HASHES),
|
||||
tag_count=len(data.get("peer_topics", [])),
|
||||
)
|
||||
|
||||
def _merge_filter_table_gossip(self, filter_table):
|
||||
if not filter_table:
|
||||
return
|
||||
for ph, entry in filter_table.items():
|
||||
if isinstance(entry, dict):
|
||||
bb = entry.get("bloom_bytes")
|
||||
if bb:
|
||||
self.fdb.store_peer_filter(
|
||||
peer_hash=ph,
|
||||
bloom_bytes=bytes(bb) if isinstance(bb, list) else bb,
|
||||
bloom_size=entry.get("bloom_size", TAG_BLOOM_SIZE),
|
||||
bloom_hashes=entry.get("bloom_hashes", TAG_BLOOM_HASHES),
|
||||
tag_count=entry.get("tag_count", 0),
|
||||
)
|
||||
|
||||
def _scoped_query_peer(self, peer_hash, tag):
|
||||
dest_hash = bytes.fromhex(peer_hash)
|
||||
if not RNS.Transport.has_path(dest_hash):
|
||||
RNS.Transport.request_path(dest_hash)
|
||||
elapsed = 0
|
||||
while not RNS.Transport.has_path(dest_hash) and elapsed < 15:
|
||||
time.sleep(0.5)
|
||||
elapsed += 0.5
|
||||
if not RNS.Transport.has_path(dest_hash):
|
||||
return []
|
||||
server_identity = RNS.Identity.recall(dest_hash)
|
||||
if server_identity is None:
|
||||
return []
|
||||
destination = RNS.Destination(
|
||||
server_identity,
|
||||
RNS.Destination.OUT,
|
||||
RNS.Destination.SINGLE,
|
||||
FORUM_APP,
|
||||
)
|
||||
link = RNS.Link(destination)
|
||||
elapsed = 0
|
||||
while link.status == RNS.Link.PENDING and elapsed < 15:
|
||||
time.sleep(0.25)
|
||||
elapsed += 0.25
|
||||
if link.status != RNS.Link.ACTIVE:
|
||||
return []
|
||||
try:
|
||||
my_hash = self.identity.hash.hex() if self.identity else "local"
|
||||
request_data = {
|
||||
"scoped_query": tag,
|
||||
"from_hash": my_hash,
|
||||
}
|
||||
receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT)
|
||||
elapsed = 0
|
||||
done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED)
|
||||
while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT:
|
||||
time.sleep(0.1)
|
||||
elapsed += 0.1
|
||||
if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED):
|
||||
resp = receipt.get_response()
|
||||
if isinstance(resp, dict) and resp.get("status") == 200:
|
||||
data = json.loads(resp["body"])
|
||||
return data.get("scoped_query_results", [])
|
||||
return []
|
||||
finally:
|
||||
link.teardown()
|
||||
|
||||
def _sync_with(self, instance_hash, my_topics=None, my_tag_bloom=None):
|
||||
success = False
|
||||
dest_hash = bytes.fromhex(instance_hash)
|
||||
if not RNS.Transport.has_path(dest_hash):
|
||||
RNS.Transport.request_path(dest_hash)
|
||||
|
|
@ -155,10 +303,12 @@ class ForumSync:
|
|||
time.sleep(0.5)
|
||||
elapsed += 0.5
|
||||
if not RNS.Transport.has_path(dest_hash):
|
||||
self.fdb.record_sync_result(instance_hash, False)
|
||||
return
|
||||
|
||||
server_identity = RNS.Identity.recall(dest_hash)
|
||||
if server_identity is None:
|
||||
self.fdb.record_sync_result(instance_hash, False)
|
||||
return
|
||||
|
||||
destination = RNS.Destination(
|
||||
|
|
@ -175,6 +325,7 @@ class ForumSync:
|
|||
elapsed += 0.25
|
||||
|
||||
if link.status != RNS.Link.ACTIVE:
|
||||
self.fdb.record_sync_result(instance_hash, False)
|
||||
return
|
||||
|
||||
try:
|
||||
|
|
@ -186,23 +337,63 @@ class ForumSync:
|
|||
break
|
||||
|
||||
since = last_sync.replace(" ", "T") if last_sync else ""
|
||||
my_hash = self.identity.hash.hex() if self.identity else "local"
|
||||
|
||||
if my_topics is None:
|
||||
my_topics = self._get_subscribed_topics()
|
||||
if my_tag_bloom is None:
|
||||
my_tag_bloom = self._build_tag_bloom(my_topics)
|
||||
|
||||
# Build filter table gossip: share a subset of our known peer filters
|
||||
all_filters = self.fdb.get_all_filters()
|
||||
filter_table_gossip = {}
|
||||
for f in all_filters[:FILTER_TABLE_GOSSIP]:
|
||||
if f["peer_hash"] != instance_hash and f["peer_hash"] != my_hash:
|
||||
filter_table_gossip[f["peer_hash"]] = {
|
||||
"bloom_bytes": list(f["bloom_bytes"]),
|
||||
"bloom_size": f["bloom_size"],
|
||||
"bloom_hashes": f["bloom_hashes"],
|
||||
"tag_count": f["tag_count"],
|
||||
}
|
||||
|
||||
threads, posts = [], []
|
||||
upvotes = []
|
||||
my_tag_cloud = self.fdb.get_tag_cloud(50)
|
||||
my_tag_list = [t for t, _ in my_tag_cloud]
|
||||
|
||||
# Only build content if we have a since timestamp (incremental sync)
|
||||
if since:
|
||||
if my_topics:
|
||||
rows = self.fdb.get_threads_by_topics(my_topics, since=since)
|
||||
threads = [dict(r) for r in rows]
|
||||
tids = [r["id"] for r in rows]
|
||||
posts = [dict(p) for p in self.fdb.get_posts_by_thread_ids(tids)]
|
||||
uv_rows = self.fdb.get_new_upvotes_since(since, tids)
|
||||
upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv_rows]
|
||||
else:
|
||||
ts, ps, uv = self.fdb.get_new_content(since)
|
||||
threads = [dict(r) for r in ts]
|
||||
posts = [dict(r) for r in ps]
|
||||
upvotes = [{"thread_id": tid, "instance_hash": instance_hash} for tid in uv]
|
||||
|
||||
# Build content bloom for dedup
|
||||
existing_ids = set(t["id"] for t in threads)
|
||||
for t in self.fdb.get_threads_by_topics(my_topics) if my_topics else []:
|
||||
existing_ids.add(t["id"])
|
||||
content_bloom = BloomFilter.from_items(list(existing_ids), BLOOM_SIZE, BLOOM_HASHES)
|
||||
|
||||
my_blocks = [h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip()]
|
||||
my_peer_blocks = self.fdb.get_peer_block_list()
|
||||
|
||||
retracted = [{"id": cid, "type": ct, "author": ai, "at": ra}
|
||||
for cid, ct, ai, ra in self.fdb.get_raw_retractions()]
|
||||
|
||||
my_hash = self.identity.hash.hex() if self.identity else "local"
|
||||
known_peers = [h for h in self.fdb.get_all_known_hashes() if h != instance_hash and h != my_hash]
|
||||
peer_topics_map = {}
|
||||
for ph in known_peers[:100]:
|
||||
pt = self.fdb.get_setting(f"peer_topics_{ph}", "")
|
||||
if pt:
|
||||
peer_topics_map[ph] = [t.strip() for t in pt.split(",") if t.strip()]
|
||||
|
||||
request_data = {
|
||||
"query": {"since": [since]} if since else {},
|
||||
|
|
@ -213,6 +404,17 @@ class ForumSync:
|
|||
"blocks": {"mine": my_blocks, "peers": my_peer_blocks},
|
||||
"retractions": retracted,
|
||||
"known_peers": known_peers,
|
||||
"my_topics": my_topics,
|
||||
"my_tag_cloud": my_tag_cloud,
|
||||
"content_bloom": list(content_bloom.bytes),
|
||||
"bloom_size": BLOOM_SIZE,
|
||||
"bloom_hashes": BLOOM_HASHES,
|
||||
"peer_topics_map": peer_topics_map,
|
||||
# Architecture B additions
|
||||
"tag_bloom": list(my_tag_bloom.bytes),
|
||||
"tag_bloom_size": TAG_BLOOM_SIZE,
|
||||
"tag_bloom_hashes": TAG_BLOOM_HASHES,
|
||||
"filter_table": filter_table_gossip,
|
||||
}
|
||||
|
||||
receipt = link.request("/forum", data=request_data, timeout=REQUEST_TIMEOUT)
|
||||
|
|
@ -229,16 +431,82 @@ class ForumSync:
|
|||
data = json.loads(resp["body"])
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
data = {}
|
||||
success = True
|
||||
|
||||
# Store peer's tag bloom filter
|
||||
self._store_peer_filter(instance_hash, data)
|
||||
|
||||
# Merge filter table gossip from peer
|
||||
self._merge_filter_table_gossip(data.get("filter_table", {}))
|
||||
|
||||
# Store peer topics (backward compat)
|
||||
peer_topics = data.get("peer_topics", [])
|
||||
if peer_topics:
|
||||
self.fdb.set_setting(f"peer_topics_{instance_hash}", ",".join(peer_topics))
|
||||
peer_tag_cloud = data.get("peer_tag_cloud", [])
|
||||
if peer_tag_cloud:
|
||||
self.fdb.set_setting(f"peer_tag_cloud_{instance_hash}", json.dumps(peer_tag_cloud))
|
||||
|
||||
# Check tag overlap using bloom filter
|
||||
my_blocks = set(h.strip() for h in self.fdb.get_setting("blocked_instances", "").split(",") if h.strip())
|
||||
|
||||
peer_tag_bloom_data = data.get("tag_bloom")
|
||||
peer_tag_bloom = None
|
||||
if peer_tag_bloom_data:
|
||||
peer_tag_bloom = BloomFilter.from_bytes(
|
||||
bytes(peer_tag_bloom_data),
|
||||
data.get("tag_bloom_size", TAG_BLOOM_SIZE),
|
||||
data.get("tag_bloom_hashes", TAG_BLOOM_HASHES),
|
||||
)
|
||||
|
||||
# If no tag overlap, skip content sync but still store the filter
|
||||
if my_topics and peer_tag_bloom is not None:
|
||||
if not self._topics_overlap_bloom(my_topics, peer_tag_bloom):
|
||||
success = True
|
||||
now = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
self.fdb.set_last_sync(instance_hash, now)
|
||||
self.fdb.record_sync_result(instance_hash, True)
|
||||
return
|
||||
|
||||
# Fall back to topic list overlap check if no bloom
|
||||
if peer_tag_bloom is None:
|
||||
incoming_topics = data.get("peer_topics", [])
|
||||
if my_topics and incoming_topics:
|
||||
if not self._topics_overlap(my_topics, incoming_topics):
|
||||
success = True
|
||||
now = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
self.fdb.set_last_sync(instance_hash, now)
|
||||
self.fdb.record_sync_result(instance_hash, True)
|
||||
return
|
||||
|
||||
# Check content bloom for dedup
|
||||
peer_bloom_data = data.get("content_bloom")
|
||||
peer_bloom = None
|
||||
if peer_bloom_data:
|
||||
peer_bloom = BloomFilter.from_bytes(
|
||||
bytes(peer_bloom_data),
|
||||
data.get("bloom_size", BLOOM_SIZE),
|
||||
data.get("bloom_hashes", BLOOM_HASHES),
|
||||
)
|
||||
|
||||
for t in data.get("threads", []):
|
||||
if t.get("author_instance", "") not in my_blocks:
|
||||
if peer_bloom and peer_bloom.might_contain(t["id"]):
|
||||
continue
|
||||
if not my_topics:
|
||||
self.fdb.merge_thread(t)
|
||||
else:
|
||||
t_tags = [tag.strip().lower() for tag in t.get("tags", "").split(",") if tag.strip()]
|
||||
if set(my_topics) & set(t_tags):
|
||||
self.fdb.merge_thread(t)
|
||||
for p in data.get("posts", []):
|
||||
if p.get("author_instance", "") not in my_blocks:
|
||||
if peer_bloom and peer_bloom.might_contain(p["id"]):
|
||||
continue
|
||||
self.fdb.merge_post(p)
|
||||
for tid in data.get("upvote_threads", []):
|
||||
self.fdb.merge_upvote(tid, instance_hash)
|
||||
# Gossip blocks from peer
|
||||
|
||||
peer_blocks = data.get("blocks", {})
|
||||
for h in peer_blocks.get("mine", []):
|
||||
if h and h not in my_blocks and instance_hash:
|
||||
|
|
@ -247,18 +515,25 @@ class ForumSync:
|
|||
if h and h not in my_blocks and instance_hash:
|
||||
self.fdb.record_peer_block(instance_hash, h)
|
||||
self._apply_peer_blocks()
|
||||
# Merge incoming retractions
|
||||
|
||||
for r in data.get("retractions", []):
|
||||
if r.get("id") and r.get("type") and r.get("author") and r.get("at"):
|
||||
self.fdb.merge_retraction(r["id"], r["type"], r["author"], r["at"])
|
||||
# Discover new peers from gossip
|
||||
|
||||
for peer_hash in data.get("known_peers", []):
|
||||
if peer_hash and peer_hash != my_hash and peer_hash != instance_hash:
|
||||
self.fdb.add_known_peer(peer_hash)
|
||||
|
||||
my_topics_set = set(my_topics)
|
||||
for ph, pt in data.get("peer_topics_map", {}).items():
|
||||
if ph and ph != my_hash and ph != instance_hash:
|
||||
pt_set = set(t.lower() for t in pt)
|
||||
if not my_topics or my_topics_set & pt_set:
|
||||
self.fdb.add_known_peer(ph)
|
||||
|
||||
now = time.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
self.fdb.set_last_sync(instance_hash, now)
|
||||
else:
|
||||
pass
|
||||
self.fdb.record_sync_result(instance_hash, success)
|
||||
finally:
|
||||
link.teardown()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue