added LoRa sync with settings UI

- Progressive retry in rns_client.py: fast timeout (15s) then slow (60s+)
  for LoRa/multi-hop links, with automatic fallback
- Background sync threads so subscriptions page returns immediately
  with syncing/error status indicators per subscription
- LoRa RNode configuration in settings page with serial port and
  expandable advanced radio settings (frequency, bandwidth, etc.)
- Internet transport now toggleable alongside LoRa — users can
  enable one, the other, or both
- Reticulum config auto-generated from settings on startup
This commit is contained in:
lichenblankie 2026-04-22 08:47:09 -07:00
parent 2dbbc5a538
commit e3aadf3947
3 changed files with 227 additions and 96 deletions

70
app.py
View file

@ -85,16 +85,32 @@ def start_gateway(reticulum):
thread.start() thread.start()
def _transport_settings_match(config_file, desired_host, desired_port): def _config_settings_match(config_file, desired_host, desired_port):
"""Check if existing config transport settings match desired values.""" """Check if existing config transport and LoRa settings match desired values."""
import configparser import configparser
try: try:
config = configparser.ConfigParser() config = configparser.ConfigParser()
config.read(config_file) config.read(config_file)
if config.has_section("TCP Transport"): # Check TCP transport
existing_host = config.get("TCP Transport", "target_host") tcp_enabled = get_setting("tcp_enabled", "1") == "1"
existing_port = config.get("TCP Transport", "target_port") has_tcp = config.has_section("TCP Transport")
return existing_host == desired_host and existing_port == str(desired_port) if tcp_enabled != has_tcp:
return False
if tcp_enabled and has_tcp:
if (config.get("TCP Transport", "target_host") != desired_host or
config.get("TCP Transport", "target_port") != str(desired_port)):
return False
# Check LoRa
lora_enabled = get_setting("lora_enabled", "0") == "1"
has_lora = config.has_section("RNode LoRa")
if lora_enabled != has_lora:
return False
if lora_enabled and has_lora:
if config.get("RNode LoRa", "port", fallback="") != get_setting("lora_port", ""):
return False
if config.get("RNode LoRa", "frequency", fallback="") != get_setting("lora_frequency", "867200000"):
return False
return True
except Exception: except Exception:
pass pass
return False return False
@ -111,9 +127,41 @@ def ensure_rns_config(config_dir, transport_host=None, transport_port=None):
transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))) transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT)))
if os.path.exists(config_file): if os.path.exists(config_file):
if _transport_settings_match(config_file, transport_host, transport_port): if _config_settings_match(config_file, transport_host, transport_port):
return return
# Build optional interface blocks
tcp_block = ""
if get_setting("tcp_enabled", "1") == "1":
tcp_block = f"""
[[TCP Transport]]
type = TCPClientInterface
enabled = yes
target_host = {transport_host}
target_port = {transport_port}
"""
lora_block = ""
if get_setting("lora_enabled", "0") == "1":
lora_port = get_setting("lora_port", "")
if lora_port:
lora_frequency = get_setting("lora_frequency", "867200000")
lora_bandwidth = get_setting("lora_bandwidth", "125000")
lora_txpower = get_setting("lora_txpower", "7")
lora_sf = get_setting("lora_sf", "8")
lora_cr = get_setting("lora_cr", "5")
lora_block = f"""
[[RNode LoRa]]
type = RNodeInterface
enabled = yes
port = {lora_port}
frequency = {lora_frequency}
bandwidth = {lora_bandwidth}
txpower = {lora_txpower}
spreadingfactor = {lora_sf}
codingrate = {lora_cr}
"""
os.makedirs(config_dir, exist_ok=True) os.makedirs(config_dir, exist_ok=True)
with open(config_file, "w") as f: with open(config_file, "w") as f:
f.write(f"""[reticulum] f.write(f"""[reticulum]
@ -127,13 +175,7 @@ def ensure_rns_config(config_dir, transport_host=None, transport_port=None):
[[Default Interface]] [[Default Interface]]
type = AutoInterface type = AutoInterface
enabled = Yes enabled = Yes
{tcp_block}{lora_block}""")
[[TCP Transport]]
type = TCPClientInterface
enabled = yes
target_host = {transport_host}
target_port = {transport_port}
""")
print(f"Created Reticulum config at {config_file}") print(f"Created Reticulum config at {config_file}")

View file

@ -758,10 +758,23 @@ def handle_style_form(msg=""):
reranker_checked = " checked" if reranker == "1" else "" reranker_checked = " checked" if reranker == "1" else ""
disabled = "" if semantic == "1" else " disabled" disabled = "" if semantic == "1" else " disabled"
dimmed = ' style="opacity:0.4"' if semantic != "1" else "" dimmed = ' style="opacity:0.4"' if semantic != "1" else ""
tcp_enabled = get_setting("tcp_enabled", "1")
tcp_checked = " checked" if tcp_enabled == "1" else ""
tcp_disabled = "" if tcp_enabled == "1" else " disabled"
transport_host = get_setting("transport_host", "reticulum.derickphan.com") transport_host = get_setting("transport_host", "reticulum.derickphan.com")
transport_port = get_setting("transport_port", "4242") transport_port = get_setting("transport_port", "4242")
compress = get_setting("compress_embeddings", "0") compress = get_setting("compress_embeddings", "0")
compress_checked = " checked" if compress == "1" else "" compress_checked = " checked" if compress == "1" else ""
lora_enabled = get_setting("lora_enabled", "0")
lora_checked = " checked" if lora_enabled == "1" else ""
lora_disabled = "" if lora_enabled == "1" else " disabled"
lora_dimmed = ' style="opacity:0.4"' if lora_enabled != "1" else ""
lora_port = get_setting("lora_port", "")
lora_frequency = get_setting("lora_frequency", "867200000")
lora_bandwidth = get_setting("lora_bandwidth", "125000")
lora_txpower = get_setting("lora_txpower", "7")
lora_sf = get_setting("lora_sf", "8")
lora_cr = get_setting("lora_cr", "5")
return _respond( return _respond(
f"<h1>customize</h1>" f"<h1>customize</h1>"
f"<h2>name your search engine</h2>" f"<h2>name your search engine</h2>"
@ -773,11 +786,43 @@ def handle_style_form(msg=""):
f" share your site list publicly at /api/sites</label><br>" f" share your site list publicly at /api/sites</label><br>"
f"<small>Note: pages tagged: private will not be shared.</small><br><br>" f"<small>Note: pages tagged: private will not be shared.</small><br><br>"
f"<h2>mesh network</h2>" f"<h2>mesh network</h2>"
f"<p>Connect to a Reticulum transport node to reach other peers.</p>" f"<p>Choose how to connect to the mesh. You can enable both for maximum reach.</p>"
f"<h3>internet</h3>"
f'<label><input type="checkbox" name="tcp_enabled" value="1"{tcp_checked} '
f'onchange="var d=!this.checked;'
f'for(var e of document.querySelectorAll(\'#tcp-fields input\'))e.disabled=d;'
f'document.getElementById(\'tcp-fields\').style.opacity=d?\'0.4\':\'1\'">'
f" connect via internet transport node</label><br>"
f"<small>Reach peers anywhere online.</small><br>"
f'<div id="tcp-fields" style="margin-top:0.5rem{";opacity:0.4" if tcp_enabled != "1" else ""}">'
f"<small>Default: reticulum.derickphan.com:4242</small><br>" f"<small>Default: reticulum.derickphan.com:4242</small><br>"
f'<input name="transport_host" value="{esc(transport_host)}" placeholder="hostname" size="30">' f'<input name="transport_host" value="{esc(transport_host)}" placeholder="hostname" size="30"{tcp_disabled}>'
f' <input name="transport_port" value="{esc(transport_port)}" placeholder="port" size="6"><br>' f' <input name="transport_port" value="{esc(transport_port)}" placeholder="port" size="6"{tcp_disabled}><br>'
f'<p><a href="https://rmap.world/" target="_blank" rel="noreferrer noopener">discover more nodes</a></p><br>' f'<p><a href="https://rmap.world/" target="_blank" rel="noreferrer noopener">discover more nodes</a></p>'
f'</div><br>'
f"<h3>LoRa</h3>"
f'<label><input type="checkbox" name="lora_enabled" value="1"{lora_checked} '
f'onchange="var d=!this.checked;document.getElementById(\'lora-port\').disabled=d;'
f'document.getElementById(\'lora-extras\').style.opacity=d?\'0.4\':\'1\';'
f'for(var e of document.querySelectorAll(\'#lora-extras input\'))e.disabled=d">'
f" connect via LoRa radio</label><br>"
f"<small>Reach nearby peers off-grid with an <a href=\"https://unsigned.io/rnode/\" target=\"_blank\" rel=\"noreferrer noopener\">RNode</a>.</small><br><br>"
f'<div id="lora-fields" style="{";opacity:0.4" if lora_enabled != "1" else ""}">'
f'<label>Serial port: <input id="lora-port" name="lora_port" value="{esc(lora_port)}" '
f'placeholder="/dev/ttyUSB0" size="20"{lora_disabled}></label><br><br>'
f'<details><summary>advanced radio settings</summary>'
f'<div id="lora-extras" style="margin-top:0.5rem">'
f'<label>Frequency (Hz): <input name="lora_frequency" value="{esc(lora_frequency)}" size="12"{lora_disabled}></label><br>'
f"<small>ISM band frequency. Default: 867200000 (868 MHz EU). US: 915000000.</small><br><br>"
f'<label>Bandwidth (Hz): <input name="lora_bandwidth" value="{esc(lora_bandwidth)}" size="8"{lora_disabled}></label><br>'
f"<small>Default: 125000</small><br><br>"
f'<label>TX Power (dBm): <input name="lora_txpower" value="{esc(lora_txpower)}" size="4"{lora_disabled}></label><br>'
f"<small>0-17 typical. Check local regulations.</small><br><br>"
f'<label>Spreading Factor: <input name="lora_sf" value="{esc(lora_sf)}" size="4"{lora_disabled}></label><br>'
f"<small>5-12. Higher = longer range, slower speed.</small><br><br>"
f'<label>Coding Rate: <input name="lora_cr" value="{esc(lora_cr)}" size="4"{lora_disabled}></label><br>'
f"<small>5-8. Higher = more error correction.</small><br>"
f'</div></details></div><br>'
f"<h2>search</h2>" f"<h2>search</h2>"
f"<h3>ai</h3>" f"<h3>ai</h3>"
f'<label><input type="checkbox" name="semantic_search" value="1"{semantic_checked} ' f'<label><input type="checkbox" name="semantic_search" value="1"{semantic_checked} '
@ -826,6 +871,7 @@ def handle_style_submit(body):
semantic = "1" if body.get("semantic_search") else "0" semantic = "1" if body.get("semantic_search") else "0"
reranker = "1" if body.get("use_reranker") else "0" reranker = "1" if body.get("use_reranker") else "0"
compress = "1" if body.get("compress_embeddings") else "0" compress = "1" if body.get("compress_embeddings") else "0"
tcp_enabled = "1" if body.get("tcp_enabled") else "0"
transport_host = body.get("transport_host", [""])[0].strip() transport_host = body.get("transport_host", [""])[0].strip()
transport_port = body.get("transport_port", [""])[0].strip() transport_port = body.get("transport_port", [""])[0].strip()
set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "") set_setting("custom_template", template if template.strip() != DEFAULT_TEMPLATE.strip() else "")
@ -834,10 +880,19 @@ def handle_style_submit(body):
set_setting("semantic_search", semantic) set_setting("semantic_search", semantic)
set_setting("use_reranker", reranker) set_setting("use_reranker", reranker)
set_setting("compress_embeddings", compress) set_setting("compress_embeddings", compress)
set_setting("tcp_enabled", tcp_enabled)
if transport_host: if transport_host:
set_setting("transport_host", transport_host) set_setting("transport_host", transport_host)
if transport_port: if transport_port:
set_setting("transport_port", transport_port) set_setting("transport_port", transport_port)
lora_enabled = "1" if body.get("lora_enabled") else "0"
set_setting("lora_enabled", lora_enabled)
set_setting("lora_port", body.get("lora_port", [""])[0].strip())
set_setting("lora_frequency", body.get("lora_frequency", ["867200000"])[0].strip())
set_setting("lora_bandwidth", body.get("lora_bandwidth", ["125000"])[0].strip())
set_setting("lora_txpower", body.get("lora_txpower", ["7"])[0].strip())
set_setting("lora_sf", body.get("lora_sf", ["8"])[0].strip())
set_setting("lora_cr", body.get("lora_cr", ["5"])[0].strip())
return handle_style_form("Saved. Restart TinyWeb for mesh network changes to take effect.") return handle_style_form("Saved. Restart TinyWeb for mesh network changes to take effect.")
@ -998,6 +1053,9 @@ def handle_api_sites(query=None):
return _json_response(data, headers={"Access-Control-Allow-Origin": "*"}) return _json_response(data, headers={"Access-Control-Allow-Origin": "*"})
_sync_threads = {}
def handle_subscriptions(msg=""): def handle_subscriptions(msg=""):
db = get_db() db = get_db()
try: try:
@ -1006,30 +1064,54 @@ def handle_subscriptions(msg=""):
return_db(db) return_db(db)
cards = "" cards = ""
for s in subs: for s in subs:
sub_id = s["id"]
auto_label = "on" if s["auto_sync"] else "off" auto_label = "on" if s["auto_sync"] else "off"
last = s["last_sync"] or "never" last = s["last_sync"] or "never"
sync_status = get_setting(f"sync_status_{sub_id}", "")
is_syncing = sub_id in _sync_threads and _sync_threads[sub_id].is_alive()
# Status line: show syncing indicator or last result
if is_syncing:
status_html = '<div style="margin-top:0.4rem;font-size:0.85rem;color:#2070c0">syncing...</div>'
elif sync_status.startswith("error:"):
err_msg = sync_status[6:]
status_html = f'<div style="margin-top:0.4rem;font-size:0.85rem;color:#c03030">{esc(err_msg)}</div>'
else:
status_html = ""
# Disable sync button while syncing
if is_syncing:
sync_btn = '<button disabled>syncing...</button>'
else:
sync_btn = (
f'<form method="post" action="/subscriptions/sync/{sub_id}" style="display:inline">'
f'{_csrf_field()}<button>sync now</button></form>'
)
cards += ( cards += (
f'<div style="border:1px solid #ddd;border-radius:4px;padding:0.9rem 1rem;margin-bottom:0.75rem">' f'<div style="border:1px solid #ddd;border-radius:4px;padding:0.9rem 1rem;margin-bottom:0.75rem">'
f'<div style="margin-bottom:0.4rem"><b>{esc(s["name"] or "unknown")}</b></div>' f'<div style="margin-bottom:0.4rem"><b>{esc(s["name"] or "unknown")}</b></div>'
f'<div><small>{esc(s["dest_hash"])}</small></div>' f'<div><small>{esc(s["dest_hash"])}</small></div>'
f'<div style="margin-top:0.4rem;font-size:0.85rem;color:#606060">last sync: {esc(last)}</div>' f'<div style="margin-top:0.4rem;font-size:0.85rem;color:#606060">last sync: {esc(last)}</div>'
f'{status_html}'
f'<div style="display:flex;gap:0.5rem;align-items:center;flex-wrap:wrap;margin-top:0.7rem">' f'<div style="display:flex;gap:0.5rem;align-items:center;flex-wrap:wrap;margin-top:0.7rem">'
f'<a href="/subscriptions/browse/{s["id"]}">browse</a>' f'<a href="/subscriptions/browse/{sub_id}">browse</a>'
f'<form method="post" action="/subscriptions/sync/{s["id"]}" style="display:inline">' f'{sync_btn}'
f'{_csrf_field()}<button>sync now</button></form>' f'<form method="post" action="/subscriptions/autosync/{sub_id}" style="display:inline">'
f'<form method="post" action="/subscriptions/autosync/{s["id"]}" style="display:inline">'
f'{_csrf_field()}<button>auto-sync: {auto_label}</button></form>' f'{_csrf_field()}<button>auto-sync: {auto_label}</button></form>'
f'<form method="post" action="/subscriptions/delete/{s["id"]}" style="display:inline">' f'<form method="post" action="/subscriptions/delete/{sub_id}" style="display:inline">'
f'{_csrf_field()}<button>remove</button></form>' f'{_csrf_field()}<button>remove</button></form>'
f'</div>' f'</div>'
f'</div>' f'</div>'
) )
listing = "" listing = ""
if subs: if subs:
any_syncing = any(sid in _sync_threads and _sync_threads[sid].is_alive() for sid in [s["id"] for s in subs])
syncall_btn = '<button disabled>syncing...</button>' if any_syncing else '<button>sync all</button>'
listing = ( listing = (
f'{cards}' f'{cards}'
f'<form method="post" action="/subscriptions/syncall">' f'<form method="post" action="/subscriptions/syncall">'
f'{_csrf_field()}<button>sync all</button></form>' f'{_csrf_field()}{syncall_btn}</form>'
) )
return _respond( return _respond(
f"<h1>subscriptions</h1>" f"<h1>subscriptions</h1>"
@ -1188,13 +1270,15 @@ def handle_subscription_pick(body):
return handle_subscriptions(f"Imported {imported} page(s). {errors} error(s).") return handle_subscriptions(f"Imported {imported} page(s). {errors} error(s).")
def handle_subscription_sync(sub_id): def _sync_subscription(sub_id):
"""Run a single subscription sync. Designed to run in a background thread."""
set_setting(f"sync_status_{sub_id}", "syncing")
db = get_db() db = get_db()
try: try:
sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone() sub = db.execute("SELECT * FROM subscriptions WHERE id = ?", (sub_id,)).fetchone()
if not sub: if not sub:
return handle_subscriptions("Subscription not found.") set_setting(f"sync_status_{sub_id}", "error:Subscription not found.")
# Use last_sync for delta sync if available return
since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else "" since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else ""
try: try:
data = fetch_remote_sites(sub["dest_hash"], since=since) data = fetch_remote_sites(sub["dest_hash"], since=since)
@ -1202,11 +1286,12 @@ def handle_subscription_sync(sub_id):
all_urls = data.get("all_urls") all_urls = data.get("all_urls")
remote_name = data.get("name", sub["name"]) remote_name = data.get("name", sub["name"])
except PermissionError: except PermissionError:
return handle_subscriptions("That instance has sharing disabled.") set_setting(f"sync_status_{sub_id}", "error:That instance has sharing disabled.")
except Exception: return
return handle_subscriptions("Could not sync with that instance.") except Exception as e:
set_setting(f"sync_status_{sub_id}", f"error:Could not sync \u2014 {e}")
return
# If full sync (all_urls provided), remove pages no longer on remote
if all_urls is not None: if all_urls is not None:
existing = db.execute( existing = db.execute(
"SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub_id,) "SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub_id,)
@ -1216,7 +1301,6 @@ def handle_subscription_sync(sub_id):
if row["url"] not in remote_url_set: if row["url"] not in remote_url_set:
db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],)) db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],))
# Upsert changed/new pages
synced = 0 synced = 0
for s in sites: for s in sites:
try: try:
@ -1226,7 +1310,6 @@ def handle_subscription_sync(sub_id):
"ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags", "ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags",
(sub_id, s["url"], s["title"], s.get("note", ""), tags_str), (sub_id, s["url"], s["title"], s.get("note", ""), tags_str),
) )
# Embed remote page for semantic search
if get_setting("semantic_search", "0") == "1": if get_setting("semantic_search", "0") == "1":
try: try:
from embeddings import store_remote_embeddings from embeddings import store_remote_embeddings
@ -1243,9 +1326,22 @@ def handle_subscription_sync(sub_id):
now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id)) db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub_id))
db.commit() db.commit()
set_setting(f"sync_status_{sub_id}", f"done:{synced}")
except Exception as e:
set_setting(f"sync_status_{sub_id}", f"error:{e}")
finally: finally:
return_db(db) return_db(db)
return handle_subscriptions(f"Synced {synced} site(s) from {esc(remote_name)}.")
def handle_subscription_sync(sub_id):
if sub_id in _sync_threads and _sync_threads[sub_id].is_alive():
return _redirect("/subscriptions")
# Clear previous status
set_setting(f"sync_status_{sub_id}", "syncing")
t = threading.Thread(target=_sync_subscription, args=(sub_id,), daemon=True)
_sync_threads[sub_id] = t
t.start()
return _redirect("/subscriptions")
def handle_subscription_autosync(sub_id): def handle_subscription_autosync(sub_id):
@ -1277,53 +1373,15 @@ def handle_subscription_syncall():
return_db(db) return_db(db)
if not subs: if not subs:
return handle_subscriptions("No subscriptions have auto-sync enabled.") return handle_subscriptions("No subscriptions have auto-sync enabled.")
total = 0
for sub in subs: for sub in subs:
try: sub_id = sub["id"]
since = sub["last_sync"].replace(" ", "T") if sub["last_sync"] else "" if sub_id in _sync_threads and _sync_threads[sub_id].is_alive():
data = fetch_remote_sites(sub["dest_hash"], since=since) continue
sites = data.get("sites", []) set_setting(f"sync_status_{sub_id}", "syncing")
all_urls = data.get("all_urls") t = threading.Thread(target=_sync_subscription, args=(sub_id,), daemon=True)
remote_name = data.get("name", sub["name"]) _sync_threads[sub_id] = t
db = get_db() t.start()
try: return _redirect("/subscriptions")
if all_urls is not None:
existing = db.execute(
"SELECT id, url FROM remote_pages WHERE subscription_id = ?", (sub["id"],)
).fetchall()
remote_url_set = set(all_urls)
for row in existing:
if row["url"] not in remote_url_set:
db.execute("DELETE FROM remote_pages WHERE id = ?", (row["id"],))
for s in sites:
try:
tags_str = ",".join(s.get("tags", []))
db.execute(
"INSERT INTO remote_pages (subscription_id, url, title, note, tags) VALUES (?, ?, ?, ?, ?) "
"ON CONFLICT(subscription_id, url) DO UPDATE SET title=excluded.title, note=excluded.note, tags=excluded.tags",
(sub["id"], s["url"], s["title"], s.get("note", ""), tags_str),
)
if get_setting("semantic_search", "0") == "1":
try:
from embeddings import store_remote_embeddings
rp_id = db.execute(
"SELECT id FROM remote_pages WHERE subscription_id = ? AND url = ?",
(sub["id"], s["url"]),
).fetchone()["id"]
store_remote_embeddings(rp_id, s["title"], s.get("note", ""), db)
except Exception:
pass
except Exception:
pass
now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
db.execute("UPDATE subscriptions SET last_sync = ?, name = ? WHERE id = ?", (now, remote_name, sub["id"]))
db.commit()
finally:
return_db(db)
total += 1
except Exception:
pass
return handle_subscriptions(f"Synced {total} subscription(s).")
# --- Reindex (semantic search) --- # --- Reindex (semantic search) ---

View file

@ -1,9 +1,15 @@
import json
import time import time
import RNS import RNS
APP_NAME = "tinyweb" APP_NAME = "tinyweb"
ASPECTS = ["server"] ASPECTS = ["server"]
REQUEST_TIMEOUT = 30
# Two-tier timeout profiles: fast first, then slow for LoRa/multi-hop links
_TIMEOUT_TIERS = [
{"path": 15, "link": 15, "request": 30, "poll": 0.25},
{"path": 60, "link": 60, "request": 120, "poll": 1.0},
]
def fetch_remote_sites(dest_hash_hex, since=""): def fetch_remote_sites(dest_hash_hex, since=""):
@ -11,18 +17,40 @@ def fetch_remote_sites(dest_hash_hex, since=""):
Connect to a remote TinyWeb instance over Reticulum and fetch its Connect to a remote TinyWeb instance over Reticulum and fetch its
shared sites. Returns the response dict from /api/sites, or raises shared sites. Returns the response dict from /api/sites, or raises
an exception on failure. Pass `since` as ISO timestamp for delta sync. an exception on failure. Pass `since` as ISO timestamp for delta sync.
Uses progressive timeouts: tries fast first, then retries with longer
timeouts for slow links (LoRa, multi-hop).
""" """
last_error = None
for tier in _TIMEOUT_TIERS:
try:
return _fetch(dest_hash_hex, since, tier)
except PermissionError:
raise # Don't retry permission errors
except Exception as e:
last_error = e
continue
raise ConnectionError(
f"Could not reach {dest_hash_hex} after {len(_TIMEOUT_TIERS)} attempts: {last_error}"
)
def _fetch(dest_hash_hex, since, timeouts):
"""Single fetch attempt with the given timeout profile."""
dest_hash = bytes.fromhex(dest_hash_hex) dest_hash = bytes.fromhex(dest_hash_hex)
poll = timeouts["poll"]
# Resolve path if needed # Resolve path if needed
if not RNS.Transport.has_path(dest_hash): if not RNS.Transport.has_path(dest_hash):
RNS.Transport.request_path(dest_hash) RNS.Transport.request_path(dest_hash)
elapsed = 0 elapsed = 0
while not RNS.Transport.has_path(dest_hash) and elapsed < 15: while not RNS.Transport.has_path(dest_hash) and elapsed < timeouts["path"]:
time.sleep(0.5) time.sleep(poll)
elapsed += 0.5 elapsed += poll
if not RNS.Transport.has_path(dest_hash): if not RNS.Transport.has_path(dest_hash):
raise ConnectionError(f"Could not find path to {dest_hash_hex}") raise ConnectionError(
f"Could not find path to {dest_hash_hex} ({timeouts['path']}s timeout)"
)
server_identity = RNS.Identity.recall(dest_hash) server_identity = RNS.Identity.recall(dest_hash)
if server_identity is None: if server_identity is None:
@ -39,15 +67,16 @@ def fetch_remote_sites(dest_hash_hex, since=""):
# Establish link # Establish link
link = RNS.Link(destination) link = RNS.Link(destination)
elapsed = 0 elapsed = 0
while link.status == RNS.Link.PENDING and elapsed < 15: while link.status == RNS.Link.PENDING and elapsed < timeouts["link"]:
time.sleep(0.25) time.sleep(poll)
elapsed += 0.25 elapsed += poll
if link.status != RNS.Link.ACTIVE: if link.status != RNS.Link.ACTIVE:
raise ConnectionError(f"Could not establish link to {dest_hash_hex}") raise ConnectionError(
f"Could not establish link to {dest_hash_hex} ({timeouts['link']}s timeout)"
)
try: try:
# Request /api/sites
query = {"since": [since]} if since else {} query = {"since": [since]} if since else {}
request_data = { request_data = {
"method": "GET", "method": "GET",
@ -57,13 +86,14 @@ def fetch_remote_sites(dest_hash_hex, since=""):
"gateway_host": "", "gateway_host": "",
} }
receipt = link.request("/tinyweb", data=request_data, timeout=REQUEST_TIMEOUT) req_timeout = timeouts["request"]
receipt = link.request("/tinyweb", data=request_data, timeout=req_timeout)
elapsed = 0 elapsed = 0
done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED) done = (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED, RNS.RequestReceipt.FAILED)
while receipt.get_status() not in done and elapsed < REQUEST_TIMEOUT: while receipt.get_status() not in done and elapsed < req_timeout:
time.sleep(0.5) time.sleep(poll)
elapsed += 0.5 elapsed += poll
if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED): if receipt.get_status() in (RNS.RequestReceipt.READY, RNS.RequestReceipt.DELIVERED):
resp = receipt.get_response() resp = receipt.get_response()
@ -71,9 +101,10 @@ def fetch_remote_sites(dest_hash_hex, since=""):
raise PermissionError("That instance has sharing disabled.") raise PermissionError("That instance has sharing disabled.")
if resp["status"] != 200: if resp["status"] != 200:
raise ConnectionError(f"Remote returned status {resp['status']}") raise ConnectionError(f"Remote returned status {resp['status']}")
import json
return json.loads(resp["body"]) return json.loads(resp["body"])
else: else:
raise ConnectionError(f"Request failed or timed out") raise ConnectionError(
f"Request failed or timed out ({req_timeout}s timeout)"
)
finally: finally:
link.teardown() link.teardown()