app.py

@@ -4,7 +4,7 @@ import threading
 import RNS
 from http.server import HTTPServer
 
-from db import init_db, get_setting, set_setting
+from db import init_db, set_setting
 from handlers import dispatch_request
 from gateway import GatewayState, GatewayHandler, GATEWAY_PORT
 

db.py

@@ -317,97 +317,88 @@ def fetch_page(url):
         label = a.get_text(strip=True) or href
         links.append((href, label[:200]))
 
-    # Extract meta description before stripping tags (case-insensitive)
+    # Extract meta description before stripping tags
     meta_desc = ""
-    for m in soup.find_all("meta"):
+    meta_tag = soup.find("meta", attrs={"name": "description"})
-        name = (m.get("name") or "").lower()
+    if meta_tag and meta_tag.get("content"):
-        prop = (m.get("property") or "").lower()
+        meta_desc = meta_tag["content"].strip()
-        content = (m.get("content") or "").strip()
+    if not meta_desc:
-        if not content:
+        # Try og:description as fallback
-            continue
+        og_tag = soup.find("meta", attrs={"property": "og:description"})
-        if name == "description" and len(content) > len(meta_desc):
+        if og_tag and og_tag.get("content"):
-            meta_desc = content
+            meta_desc = og_tag["content"].strip()
-        elif prop == "og:description" and not meta_desc:
-            meta_desc = content
 
-    for tag in soup(["script", "style", "nav", "footer", "header", "noscript", "aside"]):
+    for tag in soup(["script", "style", "nav", "footer", "header"]):
         tag.decompose()
     title = soup.title.string.strip() if soup.title and soup.title.string else url
-
-    # Extract paragraph text for better summary generation
-    paragraphs = []
-    for p in soup.find_all("p"):
-        text = p.get_text(strip=True)
-        if len(text) >= 40:
-            paragraphs.append(text)
-
     body = soup.get_text(separator=" ", strip=True)
-    return title, body, links, meta_desc, paragraphs
+    return title, body, links, meta_desc
 
 
-def _generate_summary(title, body, paragraphs=None):
+def _generate_summary(title, body):
-    """Generate a summary by extracting the best sentence from the page.
+    """Generate a summary from body text using centroid extractive method.
 
-    Priority: sentence mentioning the site name > first paragraph sentence
+    Filters out UI debris, embeds remaining sentences, finds the one
-    > first body sentence > title.
+    closest to the centroid (most representative of the page).
     """
     import re
+    # Split on sentence boundaries
+    raw = re.split(r'(?<=[.!?])\s+', body)
+    sentences = []
     noise_patterns = re.compile(
         r'arrow-|fedilink|message-square|link-external|'
         r'skip to|cookie|subscribe|sign up|log in|'
         r'privacy policy|terms of|©|\bads?\b',
         re.IGNORECASE
     )
-
+    for s in raw:
-    def _filter_sentences(raw):
+        s = s.strip()
-        result = []
+        if len(s) < 40:
-        for s in raw:
+            continue
-            s = s.strip()
+        words = s.split()
-            if len(s) < 40 or len(s.split()) < 7:
+        if len(words) < 7:
-                continue
+            continue
-            alpha_chars = sum(1 for c in s if c.isalpha() or c == ' ')
+        # Skip if mostly non-alpha (icons, arrows, encoded chars)
-            if alpha_chars < len(s) * 0.6:
+        alpha_chars = sum(1 for c in s if c.isalpha() or c == ' ')
-                continue
+        if alpha_chars < len(s) * 0.6:
-            if s.count('|') > 2 or s.count('·') > 2 or s.count('►') > 0:
+            continue
-                continue
+        # Skip nav/menu patterns
-            if noise_patterns.search(s):
+        if s.count('|') > 2 or s.count('·') > 2 or s.count('►') > 0:
-                continue
+            continue
-            result.append(s)
+        # Skip UI debris
-        return result
+        if noise_patterns.search(s):
-
+            continue
-    # Prefer sentences from <p> tags (actual content, not UI)
+        sentences.append(s)
-    sentences = []
-    if paragraphs:
-        raw = []
-        for p in paragraphs:
-            raw.extend(re.split(r'(?<=[.!?])\s+', p))
-        sentences = _filter_sentences(raw)
-
-    # Fall back to full body text
-    if not sentences:
-        raw = re.split(r'(?<=[.!?])\s+', body)
-        sentences = _filter_sentences(raw)
 
     if not sentences:
-        return title[:200] if title else ""
+        # Last resort: take the first chunk of body that looks like prose
-
+        clean = re.sub(r'\s+', ' ', body).strip()
-    # Prefer a sentence that mentions the site name
+        return clean[:160] + "..." if len(clean) > 160 else clean
-    if title:
+    if len(sentences) == 1:
-        title_words = [w.lower() for w in re.split(r'\W+', title) if len(w) >= 3]
+        s = sentences[0]
-        for s in sentences:
+        return s[:200] if len(s) > 200 else s
-            s_lower = s.lower()
+    try:
-            if sum(1 for w in title_words if w in s_lower) >= max(1, len(title_words) // 2):
+        from embeddings import embed
-                return s[:200]
+        import numpy as np
-
+        embs = embed(sentences[:50])  # cap to avoid embedding too many
-    # Otherwise use the first quality sentence
+        centroid = embs.mean(axis=0, keepdims=True)
-    return sentences[0][:200]
+        centroid = centroid / max(np.linalg.norm(centroid), 1e-12)
+        scores = (embs @ centroid.T).flatten()
+        best_idx = int(np.argmax(scores))
+        result = sentences[best_idx]
+        # Try to add a second sentence if it fits
+        if best_idx + 1 < len(sentences) and len(result) + len(sentences[best_idx + 1]) + 1 <= 200:
+            result += " " + sentences[best_idx + 1]
+        return result[:200] if len(result) > 200 else result
+    except Exception:
+        return sentences[0][:200]
 
 
 def index_url(url, note=""):
     url = clean_url(url)
-    title, body, links, meta_desc, paragraphs = fetch_page(url)
+    title, body, links, meta_desc = fetch_page(url)
-    # Use meta description if available and meaningful, otherwise generate from body
+    # Use meta description if available, otherwise generate from body
-    summary = meta_desc if meta_desc and len(meta_desc) > 20 else _generate_summary(title, body, paragraphs)
+    summary = meta_desc if meta_desc else _generate_summary(title, body)
     db = get_db()
     try:
         now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

embeddings.py

@@ -506,24 +506,21 @@ def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=F
 # ---------------------------------------------------------------------------
 
 def reindex_all(db=None, progress_callback=None):
-    """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index."""
+    """Embed all pages that don't yet have chunks. Also generates missing summaries. Rebuilds HNSW index."""
     from db import get_db, return_db, _generate_summary
     own_db = db is None
     if own_db:
         db = get_db()
     try:
-        # Clear existing chunks so everything is regenerated
-        db.execute("DELETE FROM chunks")
-        db.commit()
-
         rows = db.execute(
-            "SELECT p.id, p.title, p.body, p.summary FROM pages p"
+            "SELECT p.id, p.title, p.body, p.summary FROM pages p "
+            "WHERE p.id NOT IN (SELECT DISTINCT page_id FROM chunks WHERE page_id IS NOT NULL)"
         ).fetchall()
 
         total = len(rows)
         for i, row in enumerate(rows):
             store_embeddings(row["id"], row["title"], row["body"], db)
-            # Only regenerate summary if missing
+            # Generate summary if missing
             if not row["summary"]:
                 summary = _generate_summary(row["title"], row["body"])
                 db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"]))
@@ -531,9 +528,20 @@ def reindex_all(db=None, progress_callback=None):
             if progress_callback:
                 progress_callback(i + 1, total)
 
+        # Generate summaries for pages that already have chunks but no summary
+        no_summary = db.execute(
+            "SELECT id, title, body FROM pages WHERE summary = '' OR summary IS NULL"
+        ).fetchall()
+        for row in no_summary:
+            summary = _generate_summary(row["title"], row["body"])
+            db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"]))
+        if no_summary:
+            db.commit()
+
         # Also handle remote pages
         remote_rows = db.execute(
-            "SELECT rp.id, rp.title, rp.note FROM remote_pages rp"
+            "SELECT rp.id, rp.title, rp.note FROM remote_pages rp "
+            "WHERE rp.id NOT IN (SELECT DISTINCT remote_page_id FROM chunks WHERE remote_page_id IS NOT NULL)"
         ).fetchall()
 
         for rp in remote_rows:

handlers.py

@@ -116,7 +116,6 @@ def _error(status):
 
 
 PER_PAGE = 10
-BROWSE_PER_PAGE = 50
 
 
 def _paginate(query, key="p"):
@@ -127,11 +126,10 @@ def _paginate(query, key="p"):
     return max(1, page)
 
 
-def _page_nav(page, total, base_url, per_page=None):
+def _page_nav(page, total, base_url):
-    per_page = per_page or PER_PAGE
+    if total <= PER_PAGE:
-    if total <= per_page:
         return ""
-    total_pages = (total + per_page - 1) // per_page
+    total_pages = (total + PER_PAGE - 1) // PER_PAGE
     sep = "&" if "?" in base_url else "?"
     parts = []
     if page > 1:
@@ -379,13 +377,13 @@ def handle_add_submit(body):
 
 def handle_pages(query=None):
     page = _paginate(query or {})
-    offset = (page - 1) * BROWSE_PER_PAGE
+    offset = (page - 1) * PER_PAGE
     db = get_db()
     try:
         total = db.execute("SELECT count(*) FROM pages").fetchone()[0]
         rows = db.execute(
             "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?",
-            (BROWSE_PER_PAGE, offset),
+            (PER_PAGE, offset),
         ).fetchall()
         items = ""
         for r in rows:
@@ -406,7 +404,7 @@ def handle_pages(query=None):
     return _respond(
         f"<h1>indexed pages ({total})</h1>"
         f"<ul>{items}</ul>"
-        f'{_page_nav(page, total, "/pages", BROWSE_PER_PAGE)}'
+        f'{_page_nav(page, total, "/pages")}'
         f'<p><a href="/export">export</a> | <a href="/import">import</a></p>'
         f'<a href="/">back</a>'
     )
@@ -680,7 +678,7 @@ def handle_tags():
 
 def handle_tag_browse(tag_name, query=None):
     page = _paginate(query or {})
-    offset = (page - 1) * BROWSE_PER_PAGE
+    offset = (page - 1) * PER_PAGE
     db = get_db()
     try:
         total = db.execute(
@@ -692,7 +690,7 @@ def handle_tag_browse(tag_name, query=None):
             "JOIN page_tags pt ON p.id = pt.page_id "
             "JOIN tags t ON t.id = pt.tag_id "
             "WHERE t.name = ? ORDER BY p.id DESC LIMIT ? OFFSET ?",
-            (tag_name, BROWSE_PER_PAGE, offset),
+            (tag_name, PER_PAGE, offset),
         ).fetchall()
         items = ""
         for r in rows:
@@ -709,7 +707,7 @@ def handle_tag_browse(tag_name, query=None):
         f'<h1>tag: {esc(tag_name)}</h1>'
         f'<p>{total} page(s)</p>'
         f'<ul>{items}</ul>'
-        f'{_page_nav(page, total, f"/tags/{esc(tag_name)}", BROWSE_PER_PAGE)}'
+        f'{_page_nav(page, total, f"/tags/{esc(tag_name)}")}'
         f'<a href="/tags">all tags</a> | <a href="/">back</a>'
     )