diff --git a/Dockerfile b/Dockerfile index de57fda..3f73263 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,6 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . . -RUN pip install -e /app - RUN mkdir -p /data ENV PYTHONUNBUFFERED=1 diff --git a/README.md b/README.md index b38ae0e..991c981 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ The `/export` page produces a JSON dump of your pages. It's a migration aid — To browse a remote TinyWeb instance without running your own index: ```bash -python -m tinyweb.gateway +python gateway.py ``` This connects over Reticulum and serves the remote instance at `http://localhost:8080`. @@ -215,26 +215,21 @@ For full feature docs, see the [tinyweb-forum README](https://codeberg.org/tinyw ## Project structure ``` -app.py — Entry point (shim, imports from tinyweb.app) -pyproject.toml — Package configuration (src layout) -src/tinyweb/ - __init__.py — Package marker - app.py — Boots Reticulum, starts HTTP gateway - db.py — SQLite database, FTS5, URL fetching, SSRF protection - gateway.py — HTTP-to-RNS bridge (local or remote dispatch) - templates.py — HTML template rendering and escaping - embeddings.py — Semantic search: ONNX, HNSW, reranking - rns_client.py — Reticulum client for fetching remote site lists - handlers/ - __init__.py — Dispatch logic + re-exports - _helpers.py — CSRF, FTS sanitizer, pagination, response builders - search.py — Search (BM25, hybrid, trusted/remote results) - pages.py — Add/edit/delete/bulk/bookmark handlers - subscriptions.py — Sync, sharing, API, subscription CRUD - customize.py — Settings form, about page - tags.py — Tag list and browse - data.py — Export, import, semantic reindex -themes/ — Saved HTML templates (e.g. default.html, junimo.html) +app.py — Entry point: boots Reticulum, starts HTTP gateway +gateway.py — HTTP-to-RNS bridge (local or remote dispatch) +handlers/ — Route dispatcher and request handlers + __init__.py — Dispatch logic + re-exports + _helpers.py — CSRF, FTS sanitizer, pagination, response builders + search.py — Search (BM25, hybrid, trusted/remote results) + pages.py — Add/edit/delete/bulk/bookmark handlers + subscriptions.py — Sync, sharing, API, subscription CRUD + customize.py — Settings form, about page + tags.py — Tag list and browse + data.py — Export, import, semantic reindex +db.py — SQLite database, FTS5, URL fetching, SSRF protection +templates.py — HTML template rendering and escaping +rns_client.py — Reticulum client for fetching remote site lists +themes/ — Saved HTML templates (e.g. kodama.html) ``` ## Security diff --git a/app.py b/app.py index a5bf38b..035eca0 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,312 @@ +import os import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent / "src")) -from tinyweb.app import main -main() +import time +import threading +import argparse +import RNS +from http.server import HTTPServer, ThreadingHTTPServer + +from db import init_db, get_setting, set_setting +from handlers import dispatch_request +import handlers as handlers_mod +import templates as templates_mod +import gateway +from gateway import GatewayState, GatewayHandler + +IDENTITY_FILE = "tinyweb_identity" +DEFAULT_TRANSPORT_HOST = "rnode.bre.land" +DEFAULT_TRANSPORT_PORT = 4242 +DATA_DIR = os.environ.get("TINYWEB_DATA_DIR") or os.path.expanduser("~/.tinyweb") + + +def get_transport_config(): + host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) + port = get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT)) + return host, int(port) + + +def find_available_port(start=8080, max_attempts=20, host="127.0.0.1"): + """Find an available port starting from start.""" + import socket + for port in range(start, start + max_attempts): + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + return port + except OSError: + continue + return start + + +def get_version(): + """Get version from git tag or VERSION file.""" + try: + import subprocess + tag = subprocess.check_output( + ["git", "describe", "--tags", "--abbrev=0"], + stderr=subprocess.DEVNULL, + text=True + ).strip() + if tag.startswith("v"): + return tag[1:] + return tag + except Exception: + version_file = os.path.join(os.path.dirname(__file__), "VERSION") + if os.path.exists(version_file): + with open(version_file) as f: + return f.read().strip() + return "0.0.0" + + +def load_or_create_identity(): + os.makedirs(DATA_DIR, exist_ok=True) + identity_path = os.path.join(DATA_DIR, IDENTITY_FILE) + if os.path.isfile(identity_path): + current = os.stat(identity_path).st_mode & 0o777 + if current != 0o600: + os.chmod(identity_path, 0o600) + return RNS.Identity.from_file(identity_path) + identity = RNS.Identity() + identity.to_file(identity_path) + os.chmod(identity_path, 0o600) + return identity + + +# Remote peers on the Reticulum mesh can only reach a narrow, read-only surface. +# Any other method/path is rejected here — CSRF cannot authenticate mesh callers +# (the attacker controls both the "cookie" and the "form" side of the check), so +# gating by whitelist is the only safe option. +_RNS_ALLOWED = {("GET", "/api/sites")} + + +def rns_request_handler(path, data, request_id, link_id, remote_identity, requested_at): + if data is None: + data = {"method": "GET", "path": "/", "query": {}, "body": {}, "gateway_host": ""} + method = data.get("method", "GET") + req_path = data.get("path", "/") + if (method, req_path) not in _RNS_ALLOWED: + return { + "status": 403, + "content_type": "text/plain; charset=utf-8", + "body": "Forbidden: this endpoint is not available over Reticulum.", + "headers": {}, + } + return dispatch_request(data) + + +def start_gateway(reticulum, bind_host="127.0.0.1"): + GatewayState.reticulum = reticulum + GatewayState.local_dispatch = dispatch_request + HTTPServer.allow_reuse_address = True + server = ThreadingHTTPServer((bind_host, gateway.GATEWAY_PORT), GatewayHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + +def _config_settings_match(config_file, desired_host, desired_port): + """Check if existing config transport and LoRa settings match desired values.""" + import configparser + try: + config = configparser.ConfigParser() + config.read(config_file) + # Check TCP transport + tcp_enabled = get_setting("tcp_enabled", "1") == "1" + has_tcp = config.has_section("TCP Transport") + if tcp_enabled != has_tcp: + return False + if tcp_enabled and has_tcp: + if (config.get("TCP Transport", "target_host") != desired_host or + config.get("TCP Transport", "target_port") != str(desired_port)): + return False + # Check LoRa + lora_enabled = get_setting("lora_enabled", "0") == "1" + has_lora = config.has_section("RNode LoRa") + if lora_enabled != has_lora: + return False + if lora_enabled and has_lora: + if config.get("RNode LoRa", "port", fallback="") != get_setting("lora_port", ""): + return False + if config.get("RNode LoRa", "frequency", fallback="") != get_setting("lora_frequency", "867200000"): + return False + return True + except Exception: + pass + return False + + +def ensure_rns_config(config_dir, transport_host=None, transport_port=None): + """Generate a default Reticulum config with internet transport if none exists.""" + if config_dir is None: + config_dir = os.path.expanduser("~/.reticulum") + config_file = os.path.join(config_dir, "config") + if transport_host is None: + transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) + if transport_port is None: + transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))) + + managed_sentinel = "# managed by tinyweb" + if os.path.exists(config_file): + try: + with open(config_file) as f: + existing = f.read() + except OSError: + existing = "" + if managed_sentinel not in existing: + # User-authored config — don't clobber it. + if not _config_settings_match(config_file, transport_host, transport_port): + print( + f"Warning: {config_file} was not created by tinyweb; " + "leaving it alone. Edit it manually to change transport/LoRa settings." + ) + return + if _config_settings_match(config_file, transport_host, transport_port): + return + + # Build optional interface blocks + tcp_block = "" + if get_setting("tcp_enabled", "1") == "1": + tcp_block = f""" + [[TCP Transport]] + type = TCPClientInterface + enabled = yes + target_host = {transport_host} + target_port = {transport_port} +""" + + lora_block = "" + if get_setting("lora_enabled", "0") == "1": + lora_port = get_setting("lora_port", "") + if lora_port: + lora_frequency = get_setting("lora_frequency", "867200000") + lora_bandwidth = get_setting("lora_bandwidth", "125000") + lora_txpower = get_setting("lora_txpower", "7") + lora_sf = get_setting("lora_sf", "8") + lora_cr = get_setting("lora_cr", "5") + lora_block = f""" + [[RNode LoRa]] + type = RNodeInterface + enabled = yes + port = {lora_port} + frequency = {lora_frequency} + bandwidth = {lora_bandwidth} + txpower = {lora_txpower} + spreadingfactor = {lora_sf} + codingrate = {lora_cr} +""" + + os.makedirs(config_dir, exist_ok=True) + with open(config_file, "w") as f: + f.write(f"""{managed_sentinel} +[reticulum] + enable_transport = False + share_instance = No + +[logging] + loglevel = 4 + +[interfaces] + [[Default Interface]] + type = AutoInterface + enabled = Yes +{tcp_block}{lora_block}""") + print(f"Created Reticulum config at {config_file}") + + +def _preload_embeddings(): + """Pre-load the embedding model and build the HNSW index in background.""" + if get_setting("semantic_search", "0") != "1": + print("Semantic search disabled.") + return + try: + from embeddings import _get_session, _get_reranker, build_index + _get_session() + build_index() + if get_setting("use_reranker", "0") == "1": + _get_reranker() + print("Semantic search ready (with reranker).") + else: + print("Semantic search ready.") + except Exception as e: + print(f"Semantic search unavailable: {e}") + + +def main(): + parser = argparse.ArgumentParser(prog="tinyweb", description="Personal decentralized search engine") + parser.add_argument("--version", "-v", action="store_true", help="Show version") + parser.add_argument("--port", "-p", type=int, default=None, help="HTTP gateway port (default: 8080)") + parser.add_argument( + "--bind", "-b", default="127.0.0.1", + help="Address to bind the HTTP gateway to (default: 127.0.0.1). " + "Use 0.0.0.0 to expose to the LAN; note that the web UI has no authentication.", + ) + args = parser.parse_args() + + if args.version: + print(f"TinyWeb {get_version()}") + return + + bind_host = args.bind + port = args.port or 8080 + gateway.GATEWAY_PORT = find_available_port(port, host=bind_host) + + init_db() + transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) + transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))) + threading.Thread(target=_preload_embeddings, daemon=True).start() + config_dir = os.environ.get("RNS_CONFIG_DIR") + ensure_rns_config(config_dir, transport_host, transport_port) + reticulum = RNS.Reticulum(configdir=config_dir) + identity = load_or_create_identity() + + destination = RNS.Destination( + identity, + RNS.Destination.IN, + RNS.Destination.SINGLE, + gateway.APP_NAME, + *gateway.ASPECTS, + ) + + destination.register_request_handler( + "/tinyweb", + response_generator=rns_request_handler, + allow=RNS.Destination.ALLOW_ALL, + ) + + # Initialize forum plugin if available + forum = None + try: + from tinyweb_forum import ForumPlugin + from db import get_site_name + forum = ForumPlugin(DATA_DIR, identity, reticulum, site_name=get_site_name()) + if get_setting("forum_enabled", "0") == "1": + forum.enable() + templates_mod.FORUM_ENABLED = True + handlers_mod.forum_plugin = forum + print(f"Forum plugin: {'enabled' if forum.is_enabled() else 'available (enable in settings)'}") + except ImportError: + print("Forum plugin not installed (pip install tinyweb[forum])") + except Exception as e: + print(f"Forum plugin error: {e}") + + # Brief delay to ensure all interfaces (especially TCP) are fully ready + time.sleep(2) + destination.announce() + set_setting("dest_hash", destination.hash.hex()) + start_gateway(reticulum, bind_host=bind_host) + + print(f"TinyWeb running!") + if bind_host in ("0.0.0.0", "::"): + print(f"Open http://localhost:{gateway.GATEWAY_PORT} in your browser") + print(f"WARNING: listening on {bind_host} — the web UI has no authentication. " + "Anyone on your network can control this instance.") + else: + print(f"Open http://{bind_host}:{gateway.GATEWAY_PORT} in your browser") + print(f"Destination hash: {RNS.prettyhexrep(destination.hash)} (share this so friends can subscribe)") + + while True: + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/conftest.py b/conftest.py index 4b5c8be..9a2f26e 100644 --- a/conftest.py +++ b/conftest.py @@ -5,11 +5,15 @@ per-test tempfile, `seeded_db` layers sample rows on top, and `csrf_session` primes the thread-local CSRF token that handlers read. """ import socket +import sys +from pathlib import Path import pytest -import tinyweb.db as db_module -import tinyweb.handlers as handlers_module +sys.path.insert(0, str(Path(__file__).parent)) + +import db as db_module +import handlers as handlers_module @pytest.fixture diff --git a/src/tinyweb/db.py b/db.py similarity index 97% rename from src/tinyweb/db.py rename to db.py index 0e5cea2..97378d9 100644 --- a/src/tinyweb/db.py +++ b/db.py @@ -270,12 +270,6 @@ def init_db(): db.execute("ALTER TABLE pages ADD COLUMN reticulum_dest TEXT DEFAULT ''") db.commit() - # Migrate subscriptions: add forum_enabled column - sub_cols = [row[1] for row in db.execute("PRAGMA table_info(subscriptions)").fetchall()] - if "forum_enabled" not in sub_cols: - db.execute("ALTER TABLE subscriptions ADD COLUMN forum_enabled INTEGER DEFAULT 0") - db.commit() - # Chunks table for semantic search embeddings db.execute( "CREATE TABLE IF NOT EXISTS chunks (" @@ -446,7 +440,7 @@ def index_url(url, note="", reticulum_dest=""): db.commit() if get_setting("semantic_search", "0") == "1": try: - from tinyweb.embeddings import store_embeddings + from embeddings import store_embeddings store_embeddings(page_id, title, body, db) except Exception: pass # embedding generation is best-effort diff --git a/src/tinyweb/embeddings.py b/embeddings.py similarity index 98% rename from src/tinyweb/embeddings.py rename to embeddings.py index 2aecee4..03f6f13 100644 --- a/src/tinyweb/embeddings.py +++ b/embeddings.py @@ -246,7 +246,7 @@ def embed(texts, is_query=False): def _maybe_compress(embeddings): """Compress embeddings to float16 if compression is enabled.""" try: - from tinyweb.db import get_setting + from db import get_setting if get_setting("compress_embeddings", "0") == "1": return embeddings.astype(np.float16) except Exception: @@ -279,7 +279,7 @@ def build_index(db=None): import hnswlib global _hnsw_index, _hnsw_ids - from tinyweb.db import get_db, return_db + from db import get_db, return_db own_db = db is None if own_db: db = get_db() @@ -428,7 +428,7 @@ def semantic_search(query_text, limit=100, db=None): scores = [1.0 - float(d) for d in distances[0]] # Fetch chunk details from DB - from tinyweb.db import get_db, return_db + from db import get_db, return_db own_db = db is None if own_db: db = get_db() @@ -501,7 +501,7 @@ def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=F rerank_ids = all_ids[:20] tail_ids = all_ids[20:30] - from tinyweb.db import get_db, return_db + from db import get_db, return_db own_db = db is None if own_db: db = get_db() @@ -553,7 +553,7 @@ def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=F def reindex_all(db=None, progress_callback=None): """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index.""" - from tinyweb.db import get_db, return_db + from db import get_db, return_db own_db = db is None if own_db: db = get_db() diff --git a/src/tinyweb/gateway.py b/gateway.py similarity index 99% rename from src/tinyweb/gateway.py rename to gateway.py index 5b292ff..fa4b076 100644 --- a/src/tinyweb/gateway.py +++ b/gateway.py @@ -201,7 +201,7 @@ class GatewayHandler(BaseHTTPRequestHandler): def main(): if len(sys.argv) < 2: - print(f"Usage: python -m tinyweb.gateway ") + print(f"Usage: python gateway.py ") print(f" The destination hash is printed by app.py on startup.") sys.exit(1) diff --git a/src/tinyweb/handlers/__init__.py b/handlers/__init__.py similarity index 94% rename from src/tinyweb/handlers/__init__.py rename to handlers/__init__.py index a09b073..228508e 100644 --- a/src/tinyweb/handlers/__init__.py +++ b/handlers/__init__.py @@ -3,10 +3,10 @@ import secrets import threading from urllib.parse import unquote -from tinyweb.db import get_db, return_db, set_setting -import tinyweb.templates as templates_mod -from tinyweb.templates import esc, wrap_page -from tinyweb.rns_client import fetch_remote_sites +from db import get_db, return_db, set_setting +import templates as templates_mod +from templates import esc, wrap_page +from rns_client import fetch_remote_sites from ._helpers import ( _request_local, _get_csrf_token, _csrf_field, _check_csrf, @@ -29,7 +29,6 @@ from .subscriptions import ( handle_subscriptions, handle_subscription_add, handle_subscription_browse, handle_subscription_pick, _sync_subscription, handle_subscription_sync, handle_subscription_autosync, - handle_subscription_forum, handle_subscription_delete, handle_subscription_syncall, _sync_threads, ) @@ -133,7 +132,7 @@ def _dispatch_inner(data): _set_flash("Template reset to default.") return _redirect("/style") elif path == "/style/vacuum": - from tinyweb.db import vacuum_db + from db import vacuum_db vacuum_db() _set_flash("Database vacuumed.") return _redirect("/style") @@ -151,9 +150,6 @@ def _dispatch_inner(data): elif path.startswith("/subscriptions/autosync/"): sid = extract_id("/subscriptions/autosync/") return handle_subscription_autosync(sid) if sid is not None else _error(400) - elif path.startswith("/subscriptions/forum/"): - sid = extract_id("/subscriptions/forum/") - return handle_subscription_forum(sid) if sid is not None else _error(400) elif path.startswith("/subscriptions/delete/"): sid = extract_id("/subscriptions/delete/") return handle_subscription_delete(sid) if sid is not None else _error(400) diff --git a/src/tinyweb/handlers/_helpers.py b/handlers/_helpers.py similarity index 97% rename from src/tinyweb/handlers/_helpers.py rename to handlers/_helpers.py index c7e8d83..2611ce7 100644 --- a/src/tinyweb/handlers/_helpers.py +++ b/handlers/_helpers.py @@ -3,8 +3,8 @@ import re import secrets import threading -from tinyweb.db import get_db, return_db, get_setting, set_setting -from tinyweb.templates import wrap_page +from db import get_db, return_db, get_setting, set_setting +from templates import wrap_page _request_local = threading.local() diff --git a/src/tinyweb/handlers/customize.py b/handlers/customize.py similarity index 98% rename from src/tinyweb/handlers/customize.py rename to handlers/customize.py index 959476d..6b3786d 100644 --- a/src/tinyweb/handlers/customize.py +++ b/handlers/customize.py @@ -1,6 +1,6 @@ -from tinyweb.db import get_db, return_db, get_setting, set_setting, get_site_name -import tinyweb.templates as templates_mod -from tinyweb.templates import esc, DEFAULT_TEMPLATE +from db import get_db, return_db, get_setting, set_setting, get_site_name +import templates as templates_mod +from templates import esc, DEFAULT_TEMPLATE from ._helpers import _respond, _redirect, _json_response, _csrf_field, _get_bookmark_token, _request_local from .subscriptions import _count_shared_pages @@ -48,7 +48,7 @@ def handle_style_form(msg="", gateway_host="", scheme="http"): lora_sf = get_setting("lora_sf", "8") lora_cr = get_setting("lora_cr", "5") csrf = _csrf_field() - from tinyweb.handlers import forum_plugin as _fp + from handlers import forum_plugin as _fp if _fp is not None: forum_body = ( f"
" @@ -223,7 +223,7 @@ def handle_style_submit(body, gateway_host="", scheme="http"): forum_enabled = "1" if body.get("forum_enabled") else "0" current_forum = get_setting("forum_enabled", "0") if forum_enabled != current_forum: - from tinyweb.handlers import forum_plugin + from handlers import forum_plugin if forum_enabled == "1" and forum_plugin is None: _set_flash("Forum plugin not installed. Run: pip install tinyweb-forum") return _redirect("/style") @@ -288,7 +288,7 @@ def handle_field_save(body): if not key: return _json_response({"status": "error", "message": "No key provided."}, 400) if key == "forum_enabled": - from tinyweb.handlers import forum_plugin + from handlers import forum_plugin if value == "1" and forum_plugin is None: return _json_response({"status": "error", "message": "Forum plugin not installed."}, 400) if value == "1": diff --git a/src/tinyweb/handlers/data.py b/handlers/data.py similarity index 95% rename from src/tinyweb/handlers/data.py rename to handlers/data.py index c16a65f..d3a714f 100644 --- a/src/tinyweb/handlers/data.py +++ b/handlers/data.py @@ -1,8 +1,8 @@ import json import threading -from tinyweb.db import get_db, return_db, get_setting, set_setting, index_url -from tinyweb.templates import esc +from db import get_db, return_db, get_setting, set_setting, index_url +from templates import esc from ._helpers import _respond, _json_response, _redirect, _csrf_field MAX_EXPORT = 10000 @@ -111,7 +111,7 @@ def handle_reindex_submit(body): def _run(): try: - from tinyweb.embeddings import reindex_all + from embeddings import reindex_all def progress(current, total): set_setting("reindex_progress", f"{current}/{total}") reindex_all(progress_callback=progress) diff --git a/src/tinyweb/handlers/pages.py b/handlers/pages.py similarity index 98% rename from src/tinyweb/handlers/pages.py rename to handlers/pages.py index 4eb2a12..0799251 100644 --- a/src/tinyweb/handlers/pages.py +++ b/handlers/pages.py @@ -3,8 +3,8 @@ import json import secrets from urllib.parse import unquote -from tinyweb.db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url -from tinyweb.templates import esc +from db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url +from templates import esc from ._helpers import ( _csrf_field, _respond, _redirect, _error, _paginate, _page_nav, _get_page_tags, _set_page_tags, _cleanup_orphaned_tags, @@ -137,7 +137,7 @@ def handle_add_manual_submit(body): if get_setting("semantic_search", "0") == "1": try: - from tinyweb.embeddings import store_embeddings + from embeddings import store_embeddings store_embeddings(page_id, manual_title, manual_desc, db) db.commit() except Exception as e: diff --git a/src/tinyweb/handlers/search.py b/handlers/search.py similarity index 97% rename from src/tinyweb/handlers/search.py rename to handlers/search.py index 951b4d4..d9ee7c1 100644 --- a/src/tinyweb/handlers/search.py +++ b/handlers/search.py @@ -1,5 +1,5 @@ -from tinyweb.db import get_db, return_db, get_setting, get_site_name, clean_url -from tinyweb.templates import esc +from db import get_db, return_db, get_setting, get_site_name, clean_url +from templates import esc from ._helpers import _sanitize_fts_query, _paginate, _get_page_tags, _respond, _page_nav, PER_PAGE @@ -31,7 +31,7 @@ def handle_search(query): chunk_snippets = {} if get_setting("semantic_search", "0") == "1": try: - from tinyweb.embeddings import hybrid_search + from embeddings import hybrid_search use_reranker = get_setting("use_reranker", "1") == "1" fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker) fused_ids = [pid for pid, _ in fused] diff --git a/src/tinyweb/handlers/subscriptions.py b/handlers/subscriptions.py similarity index 95% rename from src/tinyweb/handlers/subscriptions.py rename to handlers/subscriptions.py index 1e0b5b7..97f20b2 100644 --- a/src/tinyweb/handlers/subscriptions.py +++ b/handlers/subscriptions.py @@ -1,9 +1,9 @@ import threading from datetime import datetime -from tinyweb.db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url -from tinyweb.templates import esc -from tinyweb.rns_client import fetch_remote_sites +from db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url +from templates import esc +from rns_client import fetch_remote_sites from ._helpers import ( _get_page_tags, _respond, _redirect, _json_response, _error, _csrf_field, @@ -146,7 +146,6 @@ def handle_subscriptions(msg=""): for s in subs: sub_id = s["id"] auto_label = "on" if s["auto_sync"] else "off" - forum_label = "on" if s.get("forum_enabled") else "off" last = s["last_sync"] or "never" sync_status = get_setting(f"sync_status_{sub_id}", "") is_syncing = sub_id in _sync_threads and _sync_threads[sub_id].is_alive() @@ -178,8 +177,6 @@ def handle_subscriptions(msg=""): f'{sync_btn}' f'
' f'{_csrf_field()}
' - f'
' - f'{_csrf_field()}
' f'
' f'{_csrf_field()}
' f'' @@ -392,7 +389,7 @@ def _sync_subscription(sub_id): ) if get_setting("semantic_search", "0") == "1": try: - from tinyweb.embeddings import store_remote_embeddings + from embeddings import store_remote_embeddings rp_id = db.execute( "SELECT id FROM remote_pages WHERE subscription_id = ? AND url = ?", (sub_id, s["url"]), @@ -433,16 +430,6 @@ def handle_subscription_autosync(sub_id): return _redirect("/subscriptions") -def handle_subscription_forum(sub_id): - db = get_db() - try: - db.execute("UPDATE subscriptions SET forum_enabled = 1 - forum_enabled WHERE id = ?", (sub_id,)) - db.commit() - finally: - return_db(db) - return _redirect("/subscriptions") - - def handle_subscription_delete(sub_id): db = get_db() try: diff --git a/src/tinyweb/handlers/tags.py b/handlers/tags.py similarity index 96% rename from src/tinyweb/handlers/tags.py rename to handlers/tags.py index 46189d6..f0e927d 100644 --- a/src/tinyweb/handlers/tags.py +++ b/handlers/tags.py @@ -1,5 +1,5 @@ -from tinyweb.db import get_db, return_db -from tinyweb.templates import esc +from db import get_db, return_db +from templates import esc from ._helpers import _respond, _paginate, _page_nav, _get_page_tags, BROWSE_PER_PAGE diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 6dfbe36..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,12 +0,0 @@ -[project] -name = "tinyweb" -version = "0.1.0" -description = "Personal decentralized search engine" -requires-python = ">=3.10" - -[tool.setuptools.packages.find] -where = ["src"] - -[build-system] -requires = ["setuptools"] -build-backend = "setuptools.backends._legacy:_Backend" diff --git a/src/tinyweb/rns_client.py b/rns_client.py similarity index 100% rename from src/tinyweb/rns_client.py rename to rns_client.py diff --git a/src/tinyweb/__init__.py b/src/tinyweb/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/src/tinyweb/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/src/tinyweb/app.py b/src/tinyweb/app.py deleted file mode 100644 index ca3b6de..0000000 --- a/src/tinyweb/app.py +++ /dev/null @@ -1,312 +0,0 @@ -import os -import sys -import time -import threading -import argparse -import RNS -from http.server import HTTPServer, ThreadingHTTPServer - -from tinyweb.db import init_db, get_setting, set_setting -from tinyweb.handlers import dispatch_request -import tinyweb.handlers as handlers_mod -import tinyweb.templates as templates_mod -import tinyweb.gateway -from tinyweb.gateway import GatewayState, GatewayHandler - -IDENTITY_FILE = "tinyweb_identity" -DEFAULT_TRANSPORT_HOST = "rnode.bre.land" -DEFAULT_TRANSPORT_PORT = 4242 -DATA_DIR = os.environ.get("TINYWEB_DATA_DIR") or os.path.expanduser("~/.tinyweb") - - -def get_transport_config(): - host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) - port = get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT)) - return host, int(port) - - -def find_available_port(start=8080, max_attempts=20, host="127.0.0.1"): - """Find an available port starting from start.""" - import socket - for port in range(start, start + max_attempts): - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.bind((host, port)) - return port - except OSError: - continue - return start - - -def get_version(): - """Get version from git tag or VERSION file.""" - try: - import subprocess - tag = subprocess.check_output( - ["git", "describe", "--tags", "--abbrev=0"], - stderr=subprocess.DEVNULL, - text=True - ).strip() - if tag.startswith("v"): - return tag[1:] - return tag - except Exception: - version_file = os.path.join(os.path.dirname(__file__), "VERSION") - if os.path.exists(version_file): - with open(version_file) as f: - return f.read().strip() - return "0.0.0" - - -def load_or_create_identity(): - os.makedirs(DATA_DIR, exist_ok=True) - identity_path = os.path.join(DATA_DIR, IDENTITY_FILE) - if os.path.isfile(identity_path): - current = os.stat(identity_path).st_mode & 0o777 - if current != 0o600: - os.chmod(identity_path, 0o600) - return RNS.Identity.from_file(identity_path) - identity = RNS.Identity() - identity.to_file(identity_path) - os.chmod(identity_path, 0o600) - return identity - - -# Remote peers on the Reticulum mesh can only reach a narrow, read-only surface. -# Any other method/path is rejected here — CSRF cannot authenticate mesh callers -# (the attacker controls both the "cookie" and the "form" side of the check), so -# gating by whitelist is the only safe option. -_RNS_ALLOWED = {("GET", "/api/sites")} - - -def rns_request_handler(path, data, request_id, link_id, remote_identity, requested_at): - if data is None: - data = {"method": "GET", "path": "/", "query": {}, "body": {}, "gateway_host": ""} - method = data.get("method", "GET") - req_path = data.get("path", "/") - if (method, req_path) not in _RNS_ALLOWED: - return { - "status": 403, - "content_type": "text/plain; charset=utf-8", - "body": "Forbidden: this endpoint is not available over Reticulum.", - "headers": {}, - } - return dispatch_request(data) - - -def start_gateway(reticulum, bind_host="127.0.0.1"): - GatewayState.reticulum = reticulum - GatewayState.local_dispatch = dispatch_request - HTTPServer.allow_reuse_address = True - server = ThreadingHTTPServer((bind_host, gateway.GATEWAY_PORT), GatewayHandler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - - -def _config_settings_match(config_file, desired_host, desired_port): - """Check if existing config transport and LoRa settings match desired values.""" - import configparser - try: - config = configparser.ConfigParser() - config.read(config_file) - # Check TCP transport - tcp_enabled = get_setting("tcp_enabled", "1") == "1" - has_tcp = config.has_section("TCP Transport") - if tcp_enabled != has_tcp: - return False - if tcp_enabled and has_tcp: - if (config.get("TCP Transport", "target_host") != desired_host or - config.get("TCP Transport", "target_port") != str(desired_port)): - return False - # Check LoRa - lora_enabled = get_setting("lora_enabled", "0") == "1" - has_lora = config.has_section("RNode LoRa") - if lora_enabled != has_lora: - return False - if lora_enabled and has_lora: - if config.get("RNode LoRa", "port", fallback="") != get_setting("lora_port", ""): - return False - if config.get("RNode LoRa", "frequency", fallback="") != get_setting("lora_frequency", "867200000"): - return False - return True - except Exception: - pass - return False - - -def ensure_rns_config(config_dir, transport_host=None, transport_port=None): - """Generate a default Reticulum config with internet transport if none exists.""" - if config_dir is None: - config_dir = os.path.expanduser("~/.reticulum") - config_file = os.path.join(config_dir, "config") - if transport_host is None: - transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) - if transport_port is None: - transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))) - - managed_sentinel = "# managed by tinyweb" - if os.path.exists(config_file): - try: - with open(config_file) as f: - existing = f.read() - except OSError: - existing = "" - if managed_sentinel not in existing: - # User-authored config — don't clobber it. - if not _config_settings_match(config_file, transport_host, transport_port): - print( - f"Warning: {config_file} was not created by tinyweb; " - "leaving it alone. Edit it manually to change transport/LoRa settings." - ) - return - if _config_settings_match(config_file, transport_host, transport_port): - return - - # Build optional interface blocks - tcp_block = "" - if get_setting("tcp_enabled", "1") == "1": - tcp_block = f""" - [[TCP Transport]] - type = TCPClientInterface - enabled = yes - target_host = {transport_host} - target_port = {transport_port} -""" - - lora_block = "" - if get_setting("lora_enabled", "0") == "1": - lora_port = get_setting("lora_port", "") - if lora_port: - lora_frequency = get_setting("lora_frequency", "867200000") - lora_bandwidth = get_setting("lora_bandwidth", "125000") - lora_txpower = get_setting("lora_txpower", "7") - lora_sf = get_setting("lora_sf", "8") - lora_cr = get_setting("lora_cr", "5") - lora_block = f""" - [[RNode LoRa]] - type = RNodeInterface - enabled = yes - port = {lora_port} - frequency = {lora_frequency} - bandwidth = {lora_bandwidth} - txpower = {lora_txpower} - spreadingfactor = {lora_sf} - codingrate = {lora_cr} -""" - - os.makedirs(config_dir, exist_ok=True) - with open(config_file, "w") as f: - f.write(f"""{managed_sentinel} -[reticulum] - enable_transport = False - share_instance = No - -[logging] - loglevel = 4 - -[interfaces] - [[Default Interface]] - type = AutoInterface - enabled = Yes -{tcp_block}{lora_block}""") - print(f"Created Reticulum config at {config_file}") - - -def _preload_embeddings(): - """Pre-load the embedding model and build the HNSW index in background.""" - if get_setting("semantic_search", "0") != "1": - print("Semantic search disabled.") - return - try: - from tinyweb.embeddings import _get_session, _get_reranker, build_index - _get_session() - build_index() - if get_setting("use_reranker", "0") == "1": - _get_reranker() - print("Semantic search ready (with reranker).") - else: - print("Semantic search ready.") - except Exception as e: - print(f"Semantic search unavailable: {e}") - - -def main(): - parser = argparse.ArgumentParser(prog="tinyweb", description="Personal decentralized search engine") - parser.add_argument("--version", "-v", action="store_true", help="Show version") - parser.add_argument("--port", "-p", type=int, default=None, help="HTTP gateway port (default: 8080)") - parser.add_argument( - "--bind", "-b", default="127.0.0.1", - help="Address to bind the HTTP gateway to (default: 127.0.0.1). " - "Use 0.0.0.0 to expose to the LAN; note that the web UI has no authentication.", - ) - args = parser.parse_args() - - if args.version: - print(f"TinyWeb {get_version()}") - return - - bind_host = args.bind - port = args.port or 8080 - gateway.GATEWAY_PORT = find_available_port(port, host=bind_host) - - init_db() - transport_host = get_setting("transport_host", DEFAULT_TRANSPORT_HOST) - transport_port = int(get_setting("transport_port", str(DEFAULT_TRANSPORT_PORT))) - threading.Thread(target=_preload_embeddings, daemon=True).start() - config_dir = os.environ.get("RNS_CONFIG_DIR") - ensure_rns_config(config_dir, transport_host, transport_port) - reticulum = RNS.Reticulum(configdir=config_dir) - identity = load_or_create_identity() - - destination = RNS.Destination( - identity, - RNS.Destination.IN, - RNS.Destination.SINGLE, - gateway.APP_NAME, - *gateway.ASPECTS, - ) - - destination.register_request_handler( - "/tinyweb", - response_generator=rns_request_handler, - allow=RNS.Destination.ALLOW_ALL, - ) - - # Initialize forum plugin if available - forum = None - try: - from tinyweb_forum import ForumPlugin - from tinyweb.db import get_site_name - forum = ForumPlugin(DATA_DIR, identity, reticulum, site_name=get_site_name()) - if get_setting("forum_enabled", "0") == "1": - forum.enable() - templates_mod.FORUM_ENABLED = True - handlers_mod.forum_plugin = forum - print(f"Forum plugin: {'enabled' if forum.is_enabled() else 'available (enable in settings)'}") - except ImportError: - print("Forum plugin not installed (pip install tinyweb[forum])") - except Exception as e: - print(f"Forum plugin error: {e}") - - # Brief delay to ensure all interfaces (especially TCP) are fully ready - time.sleep(2) - destination.announce() - set_setting("dest_hash", destination.hash.hex()) - start_gateway(reticulum, bind_host=bind_host) - - print(f"TinyWeb running!") - if bind_host in ("0.0.0.0", "::"): - print(f"Open http://localhost:{gateway.GATEWAY_PORT} in your browser") - print(f"WARNING: listening on {bind_host} — the web UI has no authentication. " - "Anyone on your network can control this instance.") - else: - print(f"Open http://{bind_host}:{gateway.GATEWAY_PORT} in your browser") - print(f"Destination hash: {RNS.prettyhexrep(destination.hash)} (share this so friends can subscribe)") - - while True: - time.sleep(1) - - -if __name__ == "__main__": - main() diff --git a/src/tinyweb/templates.py b/templates.py similarity index 97% rename from src/tinyweb/templates.py rename to templates.py index 8208211..27a1a6b 100644 --- a/src/tinyweb/templates.py +++ b/templates.py @@ -1,5 +1,5 @@ import html -from tinyweb.db import get_setting +from db import get_setting FORUM_ENABLED = False diff --git a/tests/test_csrf.py b/tests/test_csrf.py index 82f2584..43b4487 100644 --- a/tests/test_csrf.py +++ b/tests/test_csrf.py @@ -4,8 +4,8 @@ Every POST handler calls this to verify the submitted _csrf field matches the token stored in the thread-local (which is seeded from the cookie by `dispatch_request`). Missing or mismatched tokens must fail closed. """ -import tinyweb.handlers as handlers_module -from tinyweb.handlers import _check_csrf, _csrf_field, _get_csrf_token +import handlers as handlers_module +from handlers import _check_csrf, _csrf_field, _get_csrf_token def _set_token(token): diff --git a/tests/test_db_index_url.py b/tests/test_db_index_url.py index aac60a5..50f73ce 100644 --- a/tests/test_db_index_url.py +++ b/tests/test_db_index_url.py @@ -6,8 +6,8 @@ in sync via triggers, and the connection pool returning clean connections. from unittest.mock import patch from conftest import patch_dns_ok -import tinyweb.db as db_module -from tinyweb.db import get_db, return_db, index_url +import db as db_module +from db import get_db, return_db, index_url def _mock_fetch_page(title="Test Page", body="test body text", links=None, meta=""): diff --git a/tests/test_db_schema.py b/tests/test_db_schema.py index 4bc6691..5a4f77c 100644 --- a/tests/test_db_schema.py +++ b/tests/test_db_schema.py @@ -3,7 +3,7 @@ `init_db` is called unconditionally on startup, so it must be idempotent and create every table/trigger the rest of the app expects. """ -from tinyweb.db import get_db, return_db, init_db, get_setting, set_setting, get_site_name +from db import get_db, return_db, init_db, get_setting, set_setting, get_site_name EXPECTED_TABLES = { diff --git a/tests/test_fts_sanitizer.py b/tests/test_fts_sanitizer.py index 08afb6c..ad061da 100644 --- a/tests/test_fts_sanitizer.py +++ b/tests/test_fts_sanitizer.py @@ -6,7 +6,7 @@ could escape the quoting. These tests keep that regression dead. """ import pytest -from tinyweb.handlers import _sanitize_fts_query +from handlers import _sanitize_fts_query def test_empty_query_returns_no_match_token(): diff --git a/tests/test_gateway_limits.py b/tests/test_gateway_limits.py index a772968..6033c3a 100644 --- a/tests/test_gateway_limits.py +++ b/tests/test_gateway_limits.py @@ -8,8 +8,8 @@ import io import pytest -from tinyweb import app as app_module -from tinyweb.gateway import GatewayHandler, MAX_BODY_SIZE +import app as app_module +from gateway import GatewayHandler, MAX_BODY_SIZE class FakeHeaders: @@ -72,7 +72,7 @@ def test_post_at_size_cap_accepted(): rfile=io.BytesIO(b""), ) # Stub out local_dispatch so _forward doesn't try the network path. - from tinyweb.gateway import GatewayState + from gateway import GatewayState original = GatewayState.local_dispatch GatewayState.local_dispatch = lambda data: { "status": 404, "content_type": "text/plain", "body": "nope", diff --git a/tests/test_handlers_pages.py b/tests/test_handlers_pages.py index bc80ad0..ab4704c 100644 --- a/tests/test_handlers_pages.py +++ b/tests/test_handlers_pages.py @@ -4,8 +4,8 @@ The bulk-delete confirmation flow is a data-loss guard added in commit 8dffd8c — a stray POST without `confirmed=1` must render the confirmation page instead of actually deleting. """ -from tinyweb.db import get_db, return_db -from tinyweb.handlers import ( +from db import get_db, return_db +from handlers import ( handle_bulk_action, handle_edit_form, handle_edit_submit, diff --git a/tests/test_handlers_search.py b/tests/test_handlers_search.py index 3f4fb14..f7d2f9e 100644 --- a/tests/test_handlers_search.py +++ b/tests/test_handlers_search.py @@ -1,5 +1,5 @@ """Tests for `handle_search` — the home page + primary user flow.""" -from tinyweb.handlers import handle_search +from handlers import handle_search def test_empty_index_empty_query_shows_welcome(temp_db, csrf_session): diff --git a/tests/test_handlers_subs.py b/tests/test_handlers_subs.py index c24ea7b..93ee97d 100644 --- a/tests/test_handlers_subs.py +++ b/tests/test_handlers_subs.py @@ -6,9 +6,9 @@ available and falls back to a live fetch otherwise. """ from unittest.mock import patch -import tinyweb.handlers as handlers_module -from tinyweb.db import get_db, return_db -from tinyweb.handlers import handle_subscription_add, handle_subscription_browse +import handlers as handlers_module +from db import get_db, return_db +from handlers import handle_subscription_add, handle_subscription_browse VALID_HASH = "a" * 32 diff --git a/tests/test_handlers_tags.py b/tests/test_handlers_tags.py index 3afdad7..7ec8f05 100644 --- a/tests/test_handlers_tags.py +++ b/tests/test_handlers_tags.py @@ -4,8 +4,8 @@ Tags are stored via a join table, so orphaned rows in `tags` can accumulate if `_cleanup_orphaned_tags` isn't called after deletion/retagging. Tag counts shown in the UI rely on this being right. """ -from tinyweb.db import get_db, return_db -from tinyweb.handlers import ( +from db import get_db, return_db +from handlers import ( _cleanup_orphaned_tags, _get_page_tags, _set_page_tags, diff --git a/tests/test_link_extraction.py b/tests/test_link_extraction.py index 0baba34..2d8c741 100644 --- a/tests/test_link_extraction.py +++ b/tests/test_link_extraction.py @@ -7,7 +7,7 @@ skip Wikipedia special pages, resolve relatives via urljoin. from unittest.mock import patch from conftest import patch_dns_ok -import tinyweb.db as db_module +import db as db_module class FakeResponse: diff --git a/tests/test_pagination.py b/tests/test_pagination.py index 6b6f727..05077e0 100644 --- a/tests/test_pagination.py +++ b/tests/test_pagination.py @@ -1,5 +1,5 @@ """Tests for `_paginate` and `_page_nav`.""" -from tinyweb.handlers import _paginate, _page_nav, PER_PAGE +from handlers import _paginate, _page_nav, PER_PAGE def test_paginate_default_is_one(): diff --git a/tests/test_regressions.py b/tests/test_regressions.py index eeab752..f8a5df7 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -14,12 +14,12 @@ from unittest.mock import patch import pytest -from tinyweb import app as app_module -import tinyweb.db as db_module -import tinyweb.handlers as handlers_module +import app as app_module +import db as db_module +import handlers as handlers_module from conftest import patch_dns_fail, patch_dns_ok -from tinyweb.db import clean_url -from tinyweb.handlers import _sanitize_fts_query, handle_bulk_action +from db import clean_url +from handlers import _sanitize_fts_query, handle_bulk_action def test_6ffd38d_clean_url_preserves_www_when_bare_domain_fails(monkeypatch): @@ -47,7 +47,7 @@ def test_1bc695f_fts_sanitizer_drops_operator_words(op): def test_1bc695f_gateway_rejects_oversize_body(): """1bc695f: 16 MiB body-size cap prevents memory-exhaustion DoS.""" from tests.test_gateway_limits import FakeGatewayHandler - from tinyweb.gateway import MAX_BODY_SIZE + from gateway import MAX_BODY_SIZE h = FakeGatewayHandler( path="/add", method="POST", headers={"Content-Length": str(MAX_BODY_SIZE + 1)}, @@ -70,7 +70,7 @@ def test_1bc695f_mesh_rejects_non_whitelisted_paths(): def test_1bc695f_pool_returns_clean_connection(temp_db, monkeypatch): """1bc695f: uncommitted transactions on a pooled connection used to leak into the next consumer.""" - from tinyweb.db import get_db, return_db + from db import get_db, return_db db = get_db() db.execute( "INSERT INTO pages (url, title, body) VALUES (?, ?, ?)", @@ -88,7 +88,7 @@ def test_1bc695f_pool_returns_clean_connection(temp_db, monkeypatch): def test_8dffd8c_bulk_delete_requires_confirmation(seeded_db, csrf_session): """8dffd8c: bulk delete without confirmed=1 must render a confirm page instead of deleting — the JS confirm on /pages is a first-line filter only.""" - from tinyweb.db import get_db, return_db + from db import get_db, return_db db = get_db() try: pid = db.execute("SELECT id FROM pages LIMIT 1").fetchone()["id"] diff --git a/tests/test_sharing_logic.py b/tests/test_sharing_logic.py index 36dca46..c9c06d4 100644 --- a/tests/test_sharing_logic.py +++ b/tests/test_sharing_logic.py @@ -6,7 +6,7 @@ hiding pages the user meant to share — both are worth a regression net. """ import pytest -from tinyweb.handlers import _page_is_shared +from handlers import _page_is_shared @pytest.mark.parametrize("mode", ["exclude_private", "require_public"]) diff --git a/tests/test_ssrf.py b/tests/test_ssrf.py index 31eb132..807f9bd 100644 --- a/tests/test_ssrf.py +++ b/tests/test_ssrf.py @@ -9,7 +9,7 @@ from unittest.mock import patch import pytest -from tinyweb.db import _validate_url_target +from db import _validate_url_target def _mock_getaddrinfo(address): diff --git a/tests/test_url_cleanup.py b/tests/test_url_cleanup.py index 8ade28b..1eef72b 100644 --- a/tests/test_url_cleanup.py +++ b/tests/test_url_cleanup.py @@ -6,7 +6,7 @@ this function can silently cause duplicate rows or mask legitimate saves. import pytest from conftest import patch_dns_ok, patch_dns_fail -from tinyweb.db import clean_url, TRACKING_PARAMS +from db import clean_url, TRACKING_PARAMS def test_strips_fragment(monkeypatch):