lichenblankie/tinyweb
1
0
Fork 0
Code Issues Pull requests Projects 1 Releases 1 Packages Wiki Activity Actions

added LoRa sync with settings UI

Browse source 
- Progressive retry in rns_client.py: fast timeout (15s) then slow (60s+)
  for LoRa/multi-hop links, with automatic fallback
- Background sync threads so subscriptions page returns immediately
  with syncing/error status indicators per subscription
- LoRa RNode configuration in settings page with serial port and
  expandable advanced radio settings (frequency, bandwidth, etc.)
- Internet transport now toggleable alongside LoRa — users can
  enable one, the other, or both
- Reticulum config auto-generated from settings on startup
This commit is contained in:
lichenblankie 2026-04-22 08:47:09 -07:00
parent 2dbbc5a538
commit e3aadf3947
3 changed files with 227 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

190
handlers.py
View file

@ -758,10 +758,23 @@ def handle_style_form(msg=""):
reranker_checked = " checked" if reranker == "1" else ""
disabled = "" if semantic == "1" else " disabled"
dimmed = ' style="opacity:0.4"' if semantic != "1" else ""
tcp_enabled = get_setting("tcp_enabled", "1")
tcp_checked = " checked" if tcp_enabled == "1" else ""
tcp_disabled = "" if tcp_enabled == "1" else " disabled"
transport_host = get_setting("transport_host", "reticulum.derickphan.com")
transport_port = get_setting("transport_port", "4242")
compress = get_setting("compress_embeddings", "0")
compress_checked = " checked" if compress == "1" else ""
lora_enabled = get_setting("lora_enabled", "0")
lora_checked = " checked" if lora_enabled == "1" else ""
lora_disabled = "" if lora_enabled == "1" else " disabled"
lora_dimmed = ' style="opacity:0.4"' if lora_enabled != "1" else ""
lora_port = get_setting("lora_port", "")
lora_frequency = get_setting("lora_frequency", "867200000")
lora_bandwidth = get_setting("lora_bandwidth", "125000")
lora_txpower = get_setting("lora_txpower", "7")
lora_sf = get_setting("lora_sf", "8")
lora_cr = get_setting("lora_cr", "5")
return _respond(
f"<h1>customize</h1>"
f"<h2>name your search engine</h2>"
@ -773,11 +786,43 @@ def handle_style_form(msg=""):
f" share your site list publicly at /api/sites</label><br>"
f"<small>Note: pages tagged: private will not be shared.</small><br><br>"
f"<h2>mesh network</h2>"
f"<p>Connect to a Reticulum transport node to reach other peers.</p>"
f"<p>Choose how to connect to the mesh. You can enable both for maximum reach.</p>"
f"<h3>internet</h3>"
f'<label><input type="checkbox" name="tcp_enabled" value="1"{tcp_checked} '
f'onchange="var d=!this.checked;'
f'for(var e of document.querySelectorAll(\'#tcp-fields input\'))e.disabled=d;'
f'document.getElementById(\'tcp-fields\').style.opacity=d?\'0.4\':\'1\'">'
f" connect via internet transport node</label><br>"
f"<small>Reach peers anywhere online.</small><br>"
f'<div id="tcp-fields" style="margin-top:0.5rem{";opacity:0.4" if tcp_enabled != "1" else ""}">'
f"<small>Default: reticulum.derickphan.com:4242</small><br>"
f'<input name="transport_host" value="{esc(transport_host)}" placeholder="hostname" size="30">'
f' <input name="transport_port" value="{esc(transport_port)}" placeholder="port" size="6"><br>'
f'<p><a href="https://rmap.world/" target="_blank" rel="noreferrer noopener">discover more nodes</a></p><br>'
f'<input name="transport_host" value="{esc(transport_host)}" placeholder="hostname" size="30"{tcp_disabled}>'
f' <input name="transport_port" value="{esc(transport_port)}" placeholder="port" size="6"{tcp_disabled}><br>'
f'<p><a href="https://rmap.world/" target="_blank" rel="noreferrer noopener">discover more nodes</a></p>'
f'</div><br>'
f"<h3>LoRa</h3>"
f'<label><input type="checkbox" name="lora_enabled" value="1"{lora_checked} '
f'onchange="var d=!this.checked;document.getElementById(\'lora-port\').disabled=d;'
f'document.getElementById(\'lora-extras\').style.opacity=d?\'0.4\':\'1\';'
f'for(var e of document.querySelectorAll(\'#lora-extras input\'))e.disabled=d">'
f" connect via LoRa radio</label><br>"
f"<small>Reach nearby peers off-grid with an <a href=\"https://unsigned.io/rnode/\" target=\"_blank\" rel=\"noreferrer noopener\">RNode</a>.</small><br><br>"
f'<div id="lora-fields" style="{";opacity:0.4" if lora_enabled != "1" else ""}">'
f'<label>Serial port: <input id="lora-port" name="lora_port" value="{esc(lora_port)}" '
f'placeholder="/dev/ttyUSB0" size="20"{lora_disabled}></label><br><br>'
f'<details><summary>advanced radio settings</summary>'
f'<div id="lora-extras" style="margin-top:0.5rem">'
f'<label>Frequency (Hz): <input name="lora_frequency" value="{esc(lora_frequency)}" size="12"{lora_disabled}></label><br>'
f"<small>ISM band frequency. Default: 867200000 (868 MHz EU). US: 915000000.</small><br><br>"
f'<label>Bandwidth (Hz): <input name="lora_bandwidth" value="{esc(lora_bandwidth)}" size="8"{lora_disabled}></label><br>'
f"<small>Default: 125000</small><br><br>"
f'<label>TX Power (dBm): <input name="lora_txpower" value="{esc(lora_txpower)}" size="4"{lora_disabled}></label><br>'
f"<small>0-17 typical. Check local regulations.</small><br><br>"
f'<label>Spreading Factor: <input name="lora_sf" value="{esc(lora_sf)}" size="4"{lora_disabled}></label><br>'
f"<small>5-12. Higher = longer range, slower speed.</small><br><br>"
f'<label>Coding Rate: <input name="lora_cr" value="{esc(lora_cr)}" size="4"{lora_disabled}></label><br>'
f"<small>5-8. Higher = more error correction.</small><br>"
f'</div></details></div><br>'
f"<h2>search</h2>"
f"<h3>ai</h3>"
f'<label><input type="checkbox" name="semantic_search" value="1"{semantic_checked} '
@ -826,6 +871,7 @@ def handle_style_submit(body):
semantic = "1" if body.get("semantic_search") else "0"
reranker = "1" if body.get("use_reranker") else "0"
compress = "1" if body.get("compress_embeddings") else "0"
tcp_enabled = "1" if body.get("tcp_enabled") else "0"
transport_host = body.get("transport_host", [""])[0].strip()
transport_port = body.get("transport_port", [""])[0].strip()
set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "")
@ -834,10 +880,19 @@ def handle_style_submit(body):
set_setting("semantic_search", semantic)
set_setting("use_reranker", reranker)
set_setting("compress_embeddings", compress)
set_setting("tcp_enabled", tcp_enabled)
if transport_host:
set_setting("transport_host", transport_host)
if transport_port:
set_setting("transport_port", transport_port)
lora_enabled = "1" if body.get("lora_enabled") else "0"
set_setting("lora_enabled", lora_enabled)
set_setting("lora_port", body.get("lora_port", [""])[0].strip())
set_setting("lora_frequency", body.get("lora_frequency", ["867200000"])[0].strip())
set_setting("lora_bandwidth", body.get("lora_bandwidth", ["125000"])[0].strip())
set_setting("lora_txpower", body.get("lora_txpower", ["7"])[0].strip())
set_setting("lora_sf", body.get("lora_sf", ["8"])[0].strip())
set_setting("lora_cr", body.get("lora_cr", ["5"])[0].strip())
return handle_style_form("Saved. Restart TinyWeb for mesh network changes to take effect.")
@ -998,6 +1053,9 @@ def handle_api_sites(query=None):
return _json_response(data, headers={"Access-Control-Allow-Origin": "*"})
_sync_threads = {}
def handle_subscriptions(msg=""):
db = get_db()
try:
@ -1006,30 +1064,54 @@ def handle_subscriptions(msg=""):
return_db(db)
cards = ""
for s in subs:
sub_id = s["id"]
auto_label = "on" if s["auto_sync"] else "off"
last = s["last_sync"] or "never"
sync_status = get_setting(f"sync_status_{sub_id}", "")
is_syncing = sub_id in _sync_threads and _sync_threads[sub_id].is_alive()
# Status line: show syncing indicator or last result
if is_syncing:
status_html = '<div style="margin-top:0.4rem;font-size:0.85rem;color:#2070c0">syncing...</div>'
elif sync_status.startswith("error:"):
err_msg = sync_status[6:]
status_html = f'<div style="margin-top:0.4rem;font-size:0.85rem;color:#c03030">{esc(err_msg)}</div>'
else:
status_html = ""
# Disable sync button while syncing
if is_syncing:
sync_btn = '<button disabled>syncing...</button>'
else:
sync_btn = (
f'<form method="post" action="/subscriptions/sync/{sub_id}" style="display:inline">'
f'{_csrf_field()}<button>sync now</button></form>'
)
cards += (
f'<div style="border:1px solid #ddd;border-radius:4px;padding:0.9rem 1rem;margin-bottom:0.75rem">'
f'<div style="margin-bottom:0.4rem"><b>{esc(s["name"] or "unknown")}</b></div>'
f'<div><small>{esc(s["dest_hash"])}</small></div>'
f'<div style="margin-top:0.4rem;font-size:0.85rem;color:#606060">last sync: {esc(last)}</div>'
f'{status_html}'
f'<div style="display:flex;gap:0.5rem;align-items:center;flex-wrap:wrap;margin-top:0.7rem">'
f'<a href="/subscriptions/browse/{s["id"]}">browse</a>'
f'<form method="post" action="/subscriptions/sync/{s["id"]}" style="display:inline">'
f'{_csrf_field()}<button>sync now</button></form>'
f'<form method="post" action="/subscriptions/autosync/{s["id"]}" style="display:inline">'
f'<a href="/subscriptions/browse/{sub_id}">browse</a>'
f'{sync_btn}'
f'<form method="post" action="/subscriptions/autosync/{sub_id}" style="display:inline">'
f'{_csrf_field()}<button>auto-sync: {auto_label}</button></form>'
f'<form method="post" action="/subscriptions/delete/{s["id"]}" style="display:inline">'
f'<form method="post" action="/subscriptions/delete/{sub_id}" style="display:inline">'
f'{_csrf_field()}<button>remove</button></form>'
f'</div>'
f'</div>'
)
listing = ""
if subs:
any_syncing = any(sid in _sync_threads and _sync_threads[sid].is_alive() for sid in [s["id"] for s in subs])
syncall_btn = '<button disabled>syncing...</button>' if any_syncing else '<button>sync all</button>'
listing = (
f'{cards}'
f'<form method="post" action="/subscriptions/syncall">'
f'{_csrf_field()}<button>sync all</button></form>'
f'{_csrf_field()}{syncall_btn}</form>'
)
return _respond(
f"<h1>subscriptions</h1>"
@ -1188,13 +1270,15 @@ def handle_subscription_pick(body):
return handle_subscriptions(f"Imported {imported} page(s). {errors} error(s).")
def handle_subscription_sync(sub_id):
def _sync_subscription(sub_id):
"""Run a single subscription sync. Designed to run in a background thread."""
set_setting(f"sync_status_{sub_id}", "syncing")
db = get_db()
try:
sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone()
if not sub:
return handle_subscriptions("Subscription not found.")
# Use last_sync for delta sync if available
set_setting(f"sync_status_{sub_id}", "error:Subscription not found.")
return
since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else ""
try:
data = fetch_remote_sites(sub["dest_hash"], since=since)
@ -1202,11 +1286,12 @@ def handle_subscription_sync(sub_id):
all_urls = data.get("all_urls")
remote_name = data.get("name", sub["name"])
except PermissionError:
return handle_subscriptions("That instance has sharing disabled.")
except Exception:
return handle_subscriptions("Could not sync with that instance.")
set_setting(f"sync_status_{sub_id}", "error:That instance has sharing disabled.")
return
except Exception as e:
set_setting(f"sync_status_{sub_id}", f"error:Could not sync \u2014 {e}")
return
# If full sync (all_urls provided), remove pages no longer on remote
if all_urls is not None:
existing = db.execute(
"SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub_id,)
@ -1216,7 +1301,6 @@ def handle_subscription_sync(sub_id):
if row["url"] not in remote_url_set:
db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],))
# Upsert changed/new pages
synced = 0
for s in sites:
try:
@ -1226,7 +1310,6 @@ def handle_subscription_sync(sub_id):
"ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags",
(sub_id, s["url"], s["title"], s.get("note", ""), tags_str),
)
# Embed remote page for semantic search
if get_setting("semantic_search", "0") == "1":
try:
from embeddings import store_remote_embeddings
@ -1243,9 +1326,22 @@ def handle_subscription_sync(sub_id):
now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id))
db.commit()
set_setting(f"sync_status_{sub_id}", f"done:{synced}")
except Exception as e:
set_setting(f"sync_status_{sub_id}", f"error:{e}")
finally:
return_db(db)
return handle_subscriptions(f"Synced {synced} site(s) from {esc(remote_name)}.")
def handle_subscription_sync(sub_id):
if sub_id in _sync_threads and _sync_threads[sub_id].is_alive():
return _redirect("/subscriptions")
# Clear previous status
set_setting(f"sync_status_{sub_id}", "syncing")
t = threading.Thread(target=_sync_subscription, args=(sub_id,), daemon=True)
_sync_threads[sub_id] = t
t.start()
return _redirect("/subscriptions")
def handle_subscription_autosync(sub_id):
@ -1277,53 +1373,15 @@ def handle_subscription_syncall():
return_db(db)
if not subs:
return handle_subscriptions("No subscriptions have auto-sync enabled.")
total = 0
for sub in subs:
try:
since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else ""
data = fetch_remote_sites(sub["dest_hash"], since=since)
sites = data.get("sites", [])
all_urls = data.get("all_urls")
remote_name = data.get("name", sub["name"])
db = get_db()
try:
if all_urls is not None:
existing = db.execute(
"SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub["id"],)
).fetchall()
remote_url_set = set(all_urls)
for row in existing:
if row["url"] not in remote_url_set:
db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],))
for s in sites:
try:
tags_str = ",".join(s.get("tags", []))
db.execute(
"INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?) "
"ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags",
(sub["id"], s["url"], s["title"], s.get("note", ""), tags_str),
)
if get_setting("semantic_search", "0") == "1":
try:
from embeddings import store_remote_embeddings
rp_id = db.execute(
"SELECT id FROM remote_pages WHERE subscription_id = ? AND url = ?",
(sub["id"], s["url"]),
).fetchone()["id"]
store_remote_embeddings(rp_id, s["title"], s.get("note", ""), db)
except Exception:
pass
except Exception:
pass
now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub["id"]))
db.commit()
finally:
return_db(db)
total += 1
except Exception:
pass
return handle_subscriptions(f"Synced {total} subscription(s).")
sub_id = sub["id"]
if sub_id in _sync_threads and _sync_threads[sub_id].is_alive():
continue
set_setting(f"sync_status_{sub_id}", "syncing")
t = threading.Thread(target=_sync_subscription, args=(sub_id,), daemon=True)
_sync_threads[sub_id] = t
t.start()
return _redirect("/subscriptions")
# --- Reindex (semantic search) ---