tags (actual content, not UI) - sentences = [] - if paragraphs: - raw = [] - for p in paragraphs: - raw.extend(re.split(r'(?<=[.!?])\s+', p)) - sentences = _filter_sentences(raw) - - # Fall back to full body text - if not sentences: - raw = re.split(r'(?<=[.!?])\s+', body) - sentences = _filter_sentences(raw) - - if not sentences: - return title[:200] if title else "" - - # Prefer a sentence that mentions the site name - if title: - title_words = [w.lower() for w in re.split(r'\W+', title) if len(w) >= 3] - for s in sentences: - s_lower = s.lower() - if sum(1 for w in title_words if w in s_lower) >= max(1, len(title_words) // 2): - return s[:200] - - # Otherwise use the first quality sentence - return sentences[0][:200] - def index_url(url, note=""): url = clean_url(url) - title, body, links, meta_desc, paragraphs = fetch_page(url) + title, body, links, meta_desc = fetch_page(url) # Use meta description if available and meaningful, otherwise generate from body - summary = meta_desc if meta_desc and len(meta_desc) > 20 else _generate_summary(title, body, paragraphs) + summary = meta_desc if meta_desc and len(meta_desc) > 20 else "" db = get_db() try: now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") @@ -425,11 +381,12 @@ def index_url(url, note=""): (page_id, href, label), ) db.commit() - try: - from embeddings import store_embeddings - store_embeddings(page_id, title, body, db) - except Exception: - pass # embedding generation is best-effort + if get_setting("semantic_search", "1") == "1": + try: + from embeddings import store_embeddings + store_embeddings(page_id, title, body, db) + except Exception: + pass # embedding generation is best-effort finally: return_db(db) return title diff --git a/embeddings.py b/embeddings.py index 8ad1362..aa6a4ff 100644 --- a/embeddings.py +++ b/embeddings.py @@ -507,7 +507,7 @@ def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=F def reindex_all(db=None, progress_callback=None): """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index.""" - from db import get_db, return_db, _generate_summary + from db import get_db, return_db own_db = db is None if own_db: db = get_db() @@ -523,11 +523,6 @@ def reindex_all(db=None, progress_callback=None): total = len(rows) for i, row in enumerate(rows): store_embeddings(row["id"], row["title"], row["body"], db) - # Only regenerate summary if missing - if not row["summary"]: - summary = _generate_summary(row["title"], row["body"]) - db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"])) - db.commit() if progress_callback: progress_callback(i + 1, total) diff --git a/handlers.py b/handlers.py index 2f6f31f..5903d0b 100644 --- a/handlers.py +++ b/handlers.py @@ -6,7 +6,7 @@ from datetime import datetime from urllib.parse import unquote from db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url -from templates import esc, snippet, wrap_page, DEFAULT_TEMPLATE +from templates import esc, wrap_page, DEFAULT_TEMPLATE from rns_client import fetch_remote_sites _request_local = threading.local() @@ -205,13 +205,16 @@ def handle_search(query): # Hybrid search: merge BM25 + semantic via RRF bm25_ids = [r["id"] for r in bm25_rows] chunk_snippets = {} # page_id -> best chunk text - try: - from embeddings import hybrid_search - use_reranker = get_setting("use_reranker", "1") == "1" - fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker) - fused_ids = [pid for pid, _ in fused] - chunk_snippets = {pid: text for pid, text in fused if text} - except Exception: + if get_setting("semantic_search", "1") == "1": + try: + from embeddings import hybrid_search + use_reranker = get_setting("use_reranker", "1") == "1" + fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker) + fused_ids = [pid for pid, _ in fused] + chunk_snippets = {pid: text for pid, text in fused if text} + except Exception: + fused_ids = bm25_ids + else: fused_ids = bm25_ids total_results = len(fused_ids) @@ -239,13 +242,12 @@ def handle_search(query): if tags: tag_links = " ".join(f'[{esc(t)}]' for t in tags) tags_html = f'
' - # Use page summary as snippet (meta description or centroid sentence) - snip = r["summary"] if r["summary"] else snippet(r["body"], q) + snip_html = f'{esc(url)} blocks automated access. " + f"You can still save it manually:
" + f'" + f'back' + ) + return handle_add_form(f"Error: could not fetch or index that URL. {esc(str(e)[:100])}") + + +def handle_add_manual_submit(body): + url = clean_url(body.get("url", [""])[0].strip()) + note = body.get("note", [""])[0].strip() + tags = body.get("tags", [""])[0].strip() + manual_title = body.get("manual_title", [""])[0].strip() + manual_desc = body.get("manual_description", [""])[0].strip() + + if not url: + return handle_add_form("URL is required.") + if not manual_title or not manual_desc: + return handle_add_form("Title and description are required for manual entry.") + + db = get_db() + try: + now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + + # Insert the page + db.execute( + "INSERT INTO pages (url, title, body, note, last_modified, summary) VALUES (?, ?, ?, ?, ?, ?) " + "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, " + "note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary", + (url, manual_title, manual_desc, note, now, manual_desc[:200]), + ) + + # Get the page ID + page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0] + + # Add tags if provided + if tags: + _set_page_tags(page_id, tags, db) + + db.commit() + + # Generate embeddings for this page (if semantic search is enabled) + if get_setting("semantic_search", "1") == "1": + try: + from embeddings import store_embeddings + # Pass the page_id, title, description, and db connection + store_embeddings(page_id, manual_title, manual_desc, db) + db.commit() + except Exception as e: + # Log error but don't fail the whole operation + print(f"Error generating embeddings: {e}") + + return handle_add_form(f'Added manually: {esc(manual_title)}') + finally: + return_db(db) def handle_pages(query=None): @@ -557,8 +634,12 @@ def handle_style_form(msg=""): name = get_site_name() sharing = get_setting("sharing_enabled", "0") checked = " checked" if sharing == "1" else "" + semantic = get_setting("semantic_search", "1") + semantic_checked = " checked" if semantic == "1" else "" reranker = get_setting("use_reranker", "1") reranker_checked = " checked" if reranker == "1" else "" + disabled = "" if semantic == "1" else " disabled" + dimmed = ' style="opacity:0.4"' if semantic != "1" else "" return _respond( f"Edit the full page template. Use {esc('{{content}}')} "
f"where page content should appear.
Semantic search is disabled. Enable it in settings to use embeddings.
" + f'' + ) db = get_db() try: total_pages = db.execute("SELECT count(*) FROM pages").fetchone()[0] @@ -1184,6 +1284,8 @@ def _dispatch_inner(data): return _respond("Invalid or missing CSRF token.
", status=403) if path == "/add": return handle_add_submit(body) + elif path == "/add/manual": + return handle_add_manual_submit(body) elif path.startswith("/edit/"): pid = extract_id("/edit/") return handle_edit_submit(pid, body) if pid is not None else _error(400) diff --git a/templates.py b/templates.py index 372e736..48beace 100644 --- a/templates.py +++ b/templates.py @@ -6,14 +6,6 @@ def esc(s): return html.escape(str(s)) -def snippet(text, query, ctx=80): - pos = text.lower().find(query.lower()) - if pos == -1: - return text[:200] - start = max(0, pos - ctx) - end = min(len(text), pos + len(query) + ctx) - return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") - DEFAULT_TEMPLATE = "\n\n\n\n{{content}}\n\n"