Phase 3: merge junimos branch — heuristic snippets, noscript stripping, junimo theme, browse 50

- Case-insensitive meta description extraction (fixes sites like Lemmy
  with capitalized "Description" meta name)
- Strip aside and noscript tags for cleaner body text
- Extract paragraph text separately for better sentence quality
- Prefer sentences mentioning the site name, then first quality
  paragraph, then title as fallback
- Skip meta descriptions under 20 chars (e.g. just "Lemmy")
- Remove embedding/centroid dependency from summary generation

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app.py

@@ -4,7 +4,7 @@ import threading
 import RNS
 from http.server import HTTPServer
 
-from db import init_db, set_setting
+from db import init_db, get_setting, set_setting
 from handlers import dispatch_request
 from gateway import GatewayState, GatewayHandler, GATEWAY_PORT
 

db.py

@@ -317,88 +317,97 @@ def fetch_page(url):
         label = a.get_text(strip=True) or href
         links.append((href, label[:200]))
 
-    # Extract meta description before stripping tags
+    # Extract meta description before stripping tags (case-insensitive)
     meta_desc = ""
-    meta_tag = soup.find("meta", attrs={"name": "description"})
-    if meta_tag and meta_tag.get("content"):
-        meta_desc = meta_tag["content"].strip()
-    if not meta_desc:
-        # Try og:description as fallback
-        og_tag = soup.find("meta", attrs={"property": "og:description"})
-        if og_tag and og_tag.get("content"):
-            meta_desc = og_tag["content"].strip()
+    for m in soup.find_all("meta"):
+        name = (m.get("name") or "").lower()
+        prop = (m.get("property") or "").lower()
+        content = (m.get("content") or "").strip()
+        if not content:
+            continue
+        if name == "description" and len(content) > len(meta_desc):
+            meta_desc = content
+        elif prop == "og:description" and not meta_desc:
+            meta_desc = content
 
-    for tag in soup(["script", "style", "nav", "footer", "header"]):
+    for tag in soup(["script", "style", "nav", "footer", "header", "noscript", "aside"]):
         tag.decompose()
     title = soup.title.string.strip() if soup.title and soup.title.string else url
+
+    # Extract paragraph text for better summary generation
+    paragraphs = []
+    for p in soup.find_all("p"):
+        text = p.get_text(strip=True)
+        if len(text) >= 40:
+            paragraphs.append(text)
+
     body = soup.get_text(separator=" ", strip=True)
-    return title, body, links, meta_desc
+    return title, body, links, meta_desc, paragraphs
 
 
-def _generate_summary(title, body):
-    """Generate a summary from body text using centroid extractive method.
+def _generate_summary(title, body, paragraphs=None):
+    """Generate a summary by extracting the best sentence from the page.
 
-    Filters out UI debris, embeds remaining sentences, finds the one
-    closest to the centroid (most representative of the page).
+    Priority: sentence mentioning the site name > first paragraph sentence
+    > first body sentence > title.
     """
     import re
-    # Split on sentence boundaries
-    raw = re.split(r'(?<=[.!?])\s+', body)
-    sentences = []
     noise_patterns = re.compile(
         r'arrow-|fedilink|message-square|link-external|'
         r'skip to|cookie|subscribe|sign up|log in|'
         r'privacy policy|terms of|©|\bads?\b',
         re.IGNORECASE
     )
-    for s in raw:
-        s = s.strip()
-        if len(s) < 40:
-            continue
-        words = s.split()
-        if len(words) < 7:
-            continue
-        # Skip if mostly non-alpha (icons, arrows, encoded chars)
-        alpha_chars = sum(1 for c in s if c.isalpha() or c == ' ')
-        if alpha_chars < len(s) * 0.6:
-            continue
-        # Skip nav/menu patterns
-        if s.count('|') > 2 or s.count('·') > 2 or s.count('►') > 0:
-            continue
-        # Skip UI debris
-        if noise_patterns.search(s):
-            continue
-        sentences.append(s)
+
+    def _filter_sentences(raw):
+        result = []
+        for s in raw:
+            s = s.strip()
+            if len(s) < 40 or len(s.split()) < 7:
+                continue
+            alpha_chars = sum(1 for c in s if c.isalpha() or c == ' ')
+            if alpha_chars < len(s) * 0.6:
+                continue
+            if s.count('|') > 2 or s.count('·') > 2 or s.count('►') > 0:
+                continue
+            if noise_patterns.search(s):
+                continue
+            result.append(s)
+        return result
+
+    # Prefer sentences from <p> tags (actual content, not UI)
+    sentences = []
+    if paragraphs:
+        raw = []
+        for p in paragraphs:
+            raw.extend(re.split(r'(?<=[.!?])\s+', p))
+        sentences = _filter_sentences(raw)
+
+    # Fall back to full body text
+    if not sentences:
+        raw = re.split(r'(?<=[.!?])\s+', body)
+        sentences = _filter_sentences(raw)
 
     if not sentences:
-        # Last resort: take the first chunk of body that looks like prose
-        clean = re.sub(r'\s+', ' ', body).strip()
-        return clean[:160] + "..." if len(clean) > 160 else clean
-    if len(sentences) == 1:
-        s = sentences[0]
-        return s[:200] if len(s) > 200 else s
-    try:
-        from embeddings import embed
-        import numpy as np
-        embs = embed(sentences[:50])  # cap to avoid embedding too many
-        centroid = embs.mean(axis=0, keepdims=True)
-        centroid = centroid / max(np.linalg.norm(centroid), 1e-12)
-        scores = (embs @ centroid.T).flatten()
-        best_idx = int(np.argmax(scores))
-        result = sentences[best_idx]
-        # Try to add a second sentence if it fits
-        if best_idx + 1 < len(sentences) and len(result) + len(sentences[best_idx + 1]) + 1 <= 200:
-            result += " " + sentences[best_idx + 1]
-        return result[:200] if len(result) > 200 else result
-    except Exception:
-        return sentences[0][:200]
+        return title[:200] if title else ""
+
+    # Prefer a sentence that mentions the site name
+    if title:
+        title_words = [w.lower() for w in re.split(r'\W+', title) if len(w) >= 3]
+        for s in sentences:
+            s_lower = s.lower()
+            if sum(1 for w in title_words if w in s_lower) >= max(1, len(title_words) // 2):
+                return s[:200]
+
+    # Otherwise use the first quality sentence
+    return sentences[0][:200]
 
 
 def index_url(url, note=""):
     url = clean_url(url)
-    title, body, links, meta_desc = fetch_page(url)
-    # Use meta description if available, otherwise generate from body
-    summary = meta_desc if meta_desc else _generate_summary(title, body)
+    title, body, links, meta_desc, paragraphs = fetch_page(url)
+    # Use meta description if available and meaningful, otherwise generate from body
+    summary = meta_desc if meta_desc and len(meta_desc) > 20 else _generate_summary(title, body, paragraphs)
     db = get_db()
     try:
         now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S")

embeddings.py

@@ -506,21 +506,24 @@ def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=F
 # ---------------------------------------------------------------------------
 
 def reindex_all(db=None, progress_callback=None):
-    """Embed all pages that don't yet have chunks. Also generates missing summaries. Rebuilds HNSW index."""
+    """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index."""
     from db import get_db, return_db, _generate_summary
     own_db = db is None
     if own_db:
         db = get_db()
     try:
+        # Clear existing chunks so everything is regenerated
+        db.execute("DELETE FROM chunks")
+        db.commit()
+
         rows = db.execute(
-            "SELECT p.id, p.title, p.body, p.summary FROM pages p "
-            "WHERE p.id NOT IN (SELECT DISTINCT page_id FROM chunks WHERE page_id IS NOT NULL)"
+            "SELECT p.id, p.title, p.body, p.summary FROM pages p"
         ).fetchall()
 
         total = len(rows)
         for i, row in enumerate(rows):
             store_embeddings(row["id"], row["title"], row["body"], db)
-            # Generate summary if missing
+            # Only regenerate summary if missing
             if not row["summary"]:
                 summary = _generate_summary(row["title"], row["body"])
                 db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"]))
@@ -528,20 +531,9 @@ def reindex_all(db=None, progress_callback=None):
             if progress_callback:
                 progress_callback(i + 1, total)
 
-        # Generate summaries for pages that already have chunks but no summary
-        no_summary = db.execute(
-            "SELECT id, title, body FROM pages WHERE summary = '' OR summary IS NULL"
-        ).fetchall()
-        for row in no_summary:
-            summary = _generate_summary(row["title"], row["body"])
-            db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"]))
-        if no_summary:
-            db.commit()
-
         # Also handle remote pages
         remote_rows = db.execute(
-            "SELECT rp.id, rp.title, rp.note FROM remote_pages rp "
-            "WHERE rp.id NOT IN (SELECT DISTINCT remote_page_id FROM chunks WHERE remote_page_id IS NOT NULL)"
+            "SELECT rp.id, rp.title, rp.note FROM remote_pages rp"
         ).fetchall()
 
         for rp in remote_rows:

handlers.py

@@ -116,6 +116,7 @@ def _error(status):
 
 
 PER_PAGE = 10
+BROWSE_PER_PAGE = 50
 
 
 def _paginate(query, key="p"):
@@ -126,10 +127,11 @@ def _paginate(query, key="p"):
     return max(1, page)
 
 
-def _page_nav(page, total, base_url):
-    if total <= PER_PAGE:
+def _page_nav(page, total, base_url, per_page=None):
+    per_page = per_page or PER_PAGE
+    if total <= per_page:
         return ""
-    total_pages = (total + PER_PAGE - 1) // PER_PAGE
+    total_pages = (total + per_page - 1) // per_page
     sep = "&" if "?" in base_url else "?"
     parts = []
     if page > 1:
@@ -377,13 +379,13 @@ def handle_add_submit(body):
 
 def handle_pages(query=None):
     page = _paginate(query or {})
-    offset = (page - 1) * PER_PAGE
+    offset = (page - 1) * BROWSE_PER_PAGE
     db = get_db()
     try:
         total = db.execute("SELECT count(*) FROM pages").fetchone()[0]
         rows = db.execute(
             "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?",
-            (PER_PAGE, offset),
+            (BROWSE_PER_PAGE, offset),
         ).fetchall()
         items = ""
         for r in rows:
@@ -404,7 +406,7 @@ def handle_pages(query=None):
     return _respond(
         f"<h1>indexed pages ({total})</h1>"
         f"<ul>{items}</ul>"
-        f'{_page_nav(page, total, "/pages")}'
+        f'{_page_nav(page, total, "/pages", BROWSE_PER_PAGE)}'
         f'<p><a href="/export">export</a> | <a href="/import">import</a></p>'
         f'<a href="/">back</a>'
     )
@@ -678,7 +680,7 @@ def handle_tags():
 
 def handle_tag_browse(tag_name, query=None):
     page = _paginate(query or {})
-    offset = (page - 1) * PER_PAGE
+    offset = (page - 1) * BROWSE_PER_PAGE
     db = get_db()
     try:
         total = db.execute(
@@ -690,7 +692,7 @@ def handle_tag_browse(tag_name, query=None):
             "JOIN page_tags pt ON p.id = pt.page_id "
             "JOIN tags t ON t.id = pt.tag_id "
             "WHERE t.name = ? ORDER BY p.id DESC LIMIT ? OFFSET ?",
-            (tag_name, PER_PAGE, offset),
+            (tag_name, BROWSE_PER_PAGE, offset),
         ).fetchall()
         items = ""
         for r in rows:
@@ -707,7 +709,7 @@ def handle_tag_browse(tag_name, query=None):
         f'<h1>tag: {esc(tag_name)}</h1>'
         f'<p>{total} page(s)</p>'
         f'<ul>{items}</ul>'
-        f'{_page_nav(page, total, f"/tags/{esc(tag_name)}")}'
+        f'{_page_nav(page, total, f"/tags/{esc(tag_name)}", BROWSE_PER_PAGE)}'
         f'<a href="/tags">all tags</a> | <a href="/">back</a>'
     )