5ded9f1339
Implements a three-stage search pipeline: 1. BM25 keyword search via FTS5 with column weights 2. Semantic search via Snowflake arctic-embed-s bi-encoder + HNSW index 3. Optional cross-encoder reranking (on by default, toggleable in settings) Top 20 results are reranked for precision, next 10 appended from RRF for coverage, giving 30 total results across 3 pages. - New embeddings.py with ONNX Runtime inference, text chunking, HNSW index management, RRF fusion, and cross-encoder reranking - Meta description extraction for authentic page snippets with centroid extractive fallback - Stopword filtering in FTS5 queries to avoid overly strict matching - /reindex page for batch embedding of existing pages - Semantic embedding of remote pages during subscription sync - ~125MB dependency footprint (onnxruntime, tokenizers, hnswlib, numpy) - Models: 34MB bi-encoder + 22MB cross-encoder (downloaded on first use)
128 lines
3.8 KiB
Python
128 lines
3.8 KiB
Python
import os
|
|
import time
|
|
import threading
|
|
import RNS
|
|
from http.server import HTTPServer
|
|
|
|
from db import init_db, set_setting
|
|
from handlers import dispatch_request
|
|
from gateway import GatewayState, GatewayHandler, GATEWAY_PORT
|
|
|
|
APP_NAME = "tinyweb"
|
|
ASPECTS = ["server"]
|
|
IDENTITY_FILE = "tinyweb_identity"
|
|
DEFAULT_TRANSPORT_HOST = "reticulum.derickphan.com"
|
|
DEFAULT_TRANSPORT_PORT = 4242
|
|
|
|
|
|
def load_or_create_identity():
|
|
if os.path.isfile(IDENTITY_FILE):
|
|
# Ensure identity file is only readable by owner
|
|
current = os.stat(IDENTITY_FILE).st_mode & 0o777
|
|
if current != 0o600:
|
|
os.chmod(IDENTITY_FILE, 0o600)
|
|
return RNS.Identity.from_file(IDENTITY_FILE)
|
|
identity = RNS.Identity()
|
|
identity.to_file(IDENTITY_FILE)
|
|
os.chmod(IDENTITY_FILE, 0o600)
|
|
return identity
|
|
|
|
|
|
def rns_request_handler(path, data, request_id, link_id, remote_identity, requested_at):
|
|
if data is None:
|
|
data = {"method": "GET", "path": "/", "query": {}, "body": {}, "gateway_host": ""}
|
|
return dispatch_request(data)
|
|
|
|
|
|
def start_gateway(reticulum):
|
|
GatewayState.reticulum = reticulum
|
|
GatewayState.local_dispatch = dispatch_request
|
|
server = HTTPServer(("0.0.0.0", GATEWAY_PORT), GatewayHandler)
|
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
thread.start()
|
|
|
|
|
|
def ensure_rns_config(config_dir):
|
|
"""Generate a default Reticulum config with internet transport if none exists."""
|
|
if config_dir is None:
|
|
config_dir = os.path.expanduser("~/.reticulum")
|
|
config_file = os.path.join(config_dir, "config")
|
|
if os.path.exists(config_file):
|
|
return
|
|
os.makedirs(config_dir, exist_ok=True)
|
|
with open(config_file, "w") as f:
|
|
f.write(f"""[reticulum]
|
|
enable_transport = False
|
|
share_instance = No
|
|
|
|
[logging]
|
|
loglevel = 4
|
|
|
|
[interfaces]
|
|
[[Default Interface]]
|
|
type = AutoInterface
|
|
enabled = Yes
|
|
|
|
[[TCP Transport]]
|
|
type = TCPClientInterface
|
|
enabled = yes
|
|
target_host = {DEFAULT_TRANSPORT_HOST}
|
|
target_port = {DEFAULT_TRANSPORT_PORT}
|
|
""")
|
|
print(f"Created Reticulum config at {config_file}")
|
|
|
|
|
|
def _preload_embeddings():
|
|
"""Pre-load the embedding model and build the HNSW index in background."""
|
|
try:
|
|
from embeddings import _get_session, _get_reranker, build_index
|
|
_get_session() # downloads model on first run, loads ONNX session
|
|
build_index() # builds HNSW index from existing chunks
|
|
# Preload cross-encoder unless user has explicitly disabled it
|
|
if get_setting("use_reranker", "1") == "1":
|
|
_get_reranker()
|
|
print("Semantic search ready (with reranker).")
|
|
else:
|
|
print("Semantic search ready.")
|
|
except Exception as e:
|
|
print(f"Semantic search unavailable: {e}")
|
|
|
|
|
|
def main():
|
|
init_db()
|
|
threading.Thread(target=_preload_embeddings, daemon=True).start()
|
|
config_dir = os.environ.get("RNS_CONFIG_DIR")
|
|
ensure_rns_config(config_dir)
|
|
reticulum = RNS.Reticulum(configdir=config_dir)
|
|
identity = load_or_create_identity()
|
|
|
|
destination = RNS.Destination(
|
|
identity,
|
|
RNS.Destination.IN,
|
|
RNS.Destination.SINGLE,
|
|
APP_NAME,
|
|
*ASPECTS,
|
|
)
|
|
|
|
destination.register_request_handler(
|
|
"/tinyweb",
|
|
response_generator=rns_request_handler,
|
|
allow=RNS.Destination.ALLOW_ALL,
|
|
)
|
|
|
|
# Brief delay to ensure all interfaces (especially TCP) are fully ready
|
|
time.sleep(2)
|
|
destination.announce()
|
|
set_setting("dest_hash", destination.hash.hex())
|
|
start_gateway(reticulum)
|
|
|
|
print(f"TinyWeb running!")
|
|
print(f"Open http://localhost:{GATEWAY_PORT} in your browser")
|
|
print(f"Destination hash: {RNS.prettyhexrep(destination.hash)} (share this so friends can subscribe)")
|
|
|
|
while True:
|
|
time.sleep(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|