tags (actual content, not UI) + sentences = [] + if paragraphs: + raw = [] + for p in paragraphs: + raw.extend(re.split(r'(?<=[.!?])\s+', p)) + sentences = _filter_sentences(raw) + + # Fall back to full body text + if not sentences: + raw = re.split(r'(?<=[.!?])\s+', body) + sentences = _filter_sentences(raw) + + if not sentences: + return title[:200] if title else "" + + # Prefer a sentence that mentions the site name + if title: + title_words = [w.lower() for w in re.split(r'\W+', title) if len(w) >= 3] + for s in sentences: + s_lower = s.lower() + if sum(1 for w in title_words if w in s_lower) >= max(1, len(title_words) // 2): + return s[:200] + + # Otherwise use the first quality sentence + return sentences[0][:200] def index_url(url, note=""): url = clean_url(url) - title, body, links, meta_desc = fetch_page(url) + title, body, links, meta_desc, paragraphs = fetch_page(url) # Use meta description if available and meaningful, otherwise generate from body - summary = meta_desc if meta_desc and len(meta_desc) > 20 else "" + summary = meta_desc if meta_desc and len(meta_desc) > 20 else _generate_summary(title, body, paragraphs) db = get_db() try: now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") @@ -381,12 +425,11 @@ def index_url(url, note=""): (page_id, href, label), ) db.commit() - if get_setting("semantic_search", "1") == "1": - try: - from embeddings import store_embeddings - store_embeddings(page_id, title, body, db) - except Exception: - pass # embedding generation is best-effort + try: + from embeddings import store_embeddings + store_embeddings(page_id, title, body, db) + except Exception: + pass # embedding generation is best-effort finally: return_db(db) return title diff --git a/embeddings.py b/embeddings.py index aa6a4ff..8ad1362 100644 --- a/embeddings.py +++ b/embeddings.py @@ -507,7 +507,7 @@ def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=F def reindex_all(db=None, progress_callback=None): """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index.""" - from db import get_db, return_db + from db import get_db, return_db, _generate_summary own_db = db is None if own_db: db = get_db() @@ -523,6 +523,11 @@ def reindex_all(db=None, progress_callback=None): total = len(rows) for i, row in enumerate(rows): store_embeddings(row["id"], row["title"], row["body"], db) + # Only regenerate summary if missing + if not row["summary"]: + summary = _generate_summary(row["title"], row["body"]) + db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"])) + db.commit() if progress_callback: progress_callback(i + 1, total) diff --git a/handlers.py b/handlers.py index 5903d0b..2f6f31f 100644 --- a/handlers.py +++ b/handlers.py @@ -6,7 +6,7 @@ from datetime import datetime from urllib.parse import unquote from db import get_db, return_db, get_setting, set_setting, get_site_name, index_url, clean_url -from templates import esc, wrap_page, DEFAULT_TEMPLATE +from templates import esc, snippet, wrap_page, DEFAULT_TEMPLATE from rns_client import fetch_remote_sites _request_local = threading.local() @@ -205,16 +205,13 @@ def handle_search(query): # Hybrid search: merge BM25 + semantic via RRF bm25_ids = [r["id"] for r in bm25_rows] chunk_snippets = {} # page_id -> best chunk text - if get_setting("semantic_search", "1") == "1": - try: - from embeddings import hybrid_search - use_reranker = get_setting("use_reranker", "1") == "1" - fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker) - fused_ids = [pid for pid, _ in fused] - chunk_snippets = {pid: text for pid, text in fused if text} - except Exception: - fused_ids = bm25_ids - else: + try: + from embeddings import hybrid_search + use_reranker = get_setting("use_reranker", "1") == "1" + fused = hybrid_search(q, bm25_ids, limit=100, db=db, use_reranker=use_reranker) + fused_ids = [pid for pid, _ in fused] + chunk_snippets = {pid: text for pid, text in fused if text} + except Exception: fused_ids = bm25_ids total_results = len(fused_ids) @@ -242,12 +239,13 @@ def handle_search(query): if tags: tag_links = " ".join(f'[{esc(t)}]' for t in tags) tags_html = f'
' - snip_html = f'{esc(url)} blocks automated access. " - f"You can still save it manually:
" - f'" - f'back' - ) - return handle_add_form(f"Error: could not fetch or index that URL. {esc(str(e)[:100])}") - - -def handle_add_manual_submit(body): - url = clean_url(body.get("url", [""])[0].strip()) - note = body.get("note", [""])[0].strip() - tags = body.get("tags", [""])[0].strip() - manual_title = body.get("manual_title", [""])[0].strip() - manual_desc = body.get("manual_description", [""])[0].strip() - - if not url: - return handle_add_form("URL is required.") - if not manual_title or not manual_desc: - return handle_add_form("Title and description are required for manual entry.") - - db = get_db() - try: - now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") - - # Insert the page - db.execute( - "INSERT INTO pages (url, title, body, note, last_modified, summary) VALUES (?, ?, ?, ?, ?, ?) " - "ON CONFLICT(url) DO UPDATE SET title=excluded.title, body=excluded.body, " - "note=excluded.note, last_modified=excluded.last_modified, summary=excluded.summary", - (url, manual_title, manual_desc, note, now, manual_desc[:200]), - ) - - # Get the page ID - page_id = db.execute("SELECT id FROM pages WHERE url = ?", (url,)).fetchone()[0] - - # Add tags if provided - if tags: - _set_page_tags(page_id, tags, db) - - db.commit() - - # Generate embeddings for this page (if semantic search is enabled) - if get_setting("semantic_search", "1") == "1": - try: - from embeddings import store_embeddings - # Pass the page_id, title, description, and db connection - store_embeddings(page_id, manual_title, manual_desc, db) - db.commit() - except Exception as e: - # Log error but don't fail the whole operation - print(f"Error generating embeddings: {e}") - - return handle_add_form(f'Added manually: {esc(manual_title)}') - finally: - return_db(db) + except Exception: + return handle_add_form("Error: could not fetch or index that URL.") def handle_pages(query=None): @@ -634,12 +557,8 @@ def handle_style_form(msg=""): name = get_site_name() sharing = get_setting("sharing_enabled", "0") checked = " checked" if sharing == "1" else "" - semantic = get_setting("semantic_search", "1") - semantic_checked = " checked" if semantic == "1" else "" reranker = get_setting("use_reranker", "1") reranker_checked = " checked" if reranker == "1" else "" - disabled = "" if semantic == "1" else " disabled" - dimmed = ' style="opacity:0.4"' if semantic != "1" else "" return _respond( f"Edit the full page template. Use {esc('{{content}}')} "
f"where page content should appear.
Semantic search is disabled. Enable it in settings to use embeddings.
" - f'' - ) db = get_db() try: total_pages = db.execute("SELECT count(*) FROM pages").fetchone()[0] @@ -1284,8 +1184,6 @@ def _dispatch_inner(data): return _respond("Invalid or missing CSRF token.
", status=403) if path == "/add": return handle_add_submit(body) - elif path == "/add/manual": - return handle_add_manual_submit(body) elif path.startswith("/edit/"): pid = extract_id("/edit/") return handle_edit_submit(pid, body) if pid is not None else _error(400) diff --git a/templates.py b/templates.py index 48beace..372e736 100644 --- a/templates.py +++ b/templates.py @@ -6,6 +6,14 @@ def esc(s): return html.escape(str(s)) +def snippet(text, query, ctx=80): + pos = text.lower().find(query.lower()) + if pos == -1: + return text[:200] + start = max(0, pos - ctx) + end = min(len(text), pos + len(query) + ctx) + return ("..." if start > 0 else "") + text[start:end] + ("..." if end < len(text) else "") + DEFAULT_TEMPLATE = "\n\n\n\n{{content}}\n\n"