tags (actual content, not UI) - sentences = [] - if paragraphs: - raw = [] - for p in paragraphs: - raw.extend(re.split(r'(?<=[.!?])\s+', p)) - sentences = _filter_sentences(raw) - - # Fall back to full body text - if not sentences: - raw = re.split(r'(?<=[.!?])\s+', body) - sentences = _filter_sentences(raw) + for s in raw: + s = s.strip() + if len(s) < 40: + continue + words = s.split() + if len(words) < 7: + continue + # Skip if mostly non-alpha (icons, arrows, encoded chars) + alpha_chars = sum(1 for c in s if c.isalpha() or c == ' ') + if alpha_chars < len(s) * 0.6: + continue + # Skip nav/menu patterns + if s.count('|') > 2 or s.count('·') > 2 or s.count('►') > 0: + continue + # Skip UI debris + if noise_patterns.search(s): + continue + sentences.append(s) if not sentences: - return title[:200] if title else "" - - # Prefer a sentence that mentions the site name - if title: - title_words = [w.lower() for w in re.split(r'\W+', title) if len(w) >= 3] - for s in sentences: - s_lower = s.lower() - if sum(1 for w in title_words if w in s_lower) >= max(1, len(title_words) // 2): - return s[:200] - - # Otherwise use the first quality sentence - return sentences[0][:200] + # Last resort: take the first chunk of body that looks like prose + clean = re.sub(r'\s+', ' ', body).strip() + return clean[:160] + "..." if len(clean) > 160 else clean + if len(sentences) == 1: + s = sentences[0] + return s[:200] if len(s) > 200 else s + try: + from embeddings import embed + import numpy as np + embs = embed(sentences[:50]) # cap to avoid embedding too many + centroid = embs.mean(axis=0, keepdims=True) + centroid = centroid / max(np.linalg.norm(centroid), 1e-12) + scores = (embs @ centroid.T).flatten() + best_idx = int(np.argmax(scores)) + result = sentences[best_idx] + # Try to add a second sentence if it fits + if best_idx + 1 < len(sentences) and len(result) + len(sentences[best_idx + 1]) + 1 <= 200: + result += " " + sentences[best_idx + 1] + return result[:200] if len(result) > 200 else result + except Exception: + return sentences[0][:200] def index_url(url, note=""): url = clean_url(url) - title, body, links, meta_desc, paragraphs = fetch_page(url) - # Use meta description if available and meaningful, otherwise generate from body - summary = meta_desc if meta_desc and len(meta_desc) > 20 else _generate_summary(title, body, paragraphs) + title, body, links, meta_desc = fetch_page(url) + # Use meta description if available, otherwise generate from body + summary = meta_desc if meta_desc else _generate_summary(title, body) db = get_db() try: now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") diff --git a/embeddings.py b/embeddings.py index 8ad1362..5575c29 100644 --- a/embeddings.py +++ b/embeddings.py @@ -506,24 +506,21 @@ def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=F # --------------------------------------------------------------------------- def reindex_all(db=None, progress_callback=None): - """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index.""" + """Embed all pages that don't yet have chunks. Also generates missing summaries. Rebuilds HNSW index.""" from db import get_db, return_db, _generate_summary own_db = db is None if own_db: db = get_db() try: - # Clear existing chunks so everything is regenerated - db.execute("DELETE FROM chunks") - db.commit() - rows = db.execute( - "SELECT p.id, p.title, p.body, p.summary FROM pages p" + "SELECT p.id, p.title, p.body, p.summary FROM pages p " + "WHERE p.id NOT IN (SELECT DISTINCT page_id FROM chunks WHERE page_id IS NOT NULL)" ).fetchall() total = len(rows) for i, row in enumerate(rows): store_embeddings(row["id"], row["title"], row["body"], db) - # Only regenerate summary if missing + # Generate summary if missing if not row["summary"]: summary = _generate_summary(row["title"], row["body"]) db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"])) @@ -531,9 +528,20 @@ def reindex_all(db=None, progress_callback=None): if progress_callback: progress_callback(i + 1, total) + # Generate summaries for pages that already have chunks but no summary + no_summary = db.execute( + "SELECT id, title, body FROM pages WHERE summary = '' OR summary IS NULL" + ).fetchall() + for row in no_summary: + summary = _generate_summary(row["title"], row["body"]) + db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"])) + if no_summary: + db.commit() + # Also handle remote pages remote_rows = db.execute( - "SELECT rp.id, rp.title, rp.note FROM remote_pages rp" + "SELECT rp.id, rp.title, rp.note FROM remote_pages rp " + "WHERE rp.id NOT IN (SELECT DISTINCT remote_page_id FROM chunks WHERE remote_page_id IS NOT NULL)" ).fetchall() for rp in remote_rows: diff --git a/handlers.py b/handlers.py index 2f6f31f..abd0d13 100644 --- a/handlers.py +++ b/handlers.py @@ -116,7 +116,6 @@ def _error(status): PER_PAGE = 10 -BROWSE_PER_PAGE = 50 def _paginate(query, key="p"): @@ -127,11 +126,10 @@ def _paginate(query, key="p"): return max(1, page) -def _page_nav(page, total, base_url, per_page=None): - per_page = per_page or PER_PAGE - if total <= per_page: +def _page_nav(page, total, base_url): + if total <= PER_PAGE: return "" - total_pages = (total + per_page - 1) // per_page + total_pages = (total + PER_PAGE - 1) // PER_PAGE sep = "&" if "?" in base_url else "?" parts = [] if page > 1: @@ -379,13 +377,13 @@ def handle_add_submit(body): def handle_pages(query=None): page = _paginate(query or {}) - offset = (page - 1) * BROWSE_PER_PAGE + offset = (page - 1) * PER_PAGE db = get_db() try: total = db.execute("SELECT count(*) FROM pages").fetchone()[0] rows = db.execute( "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?", - (BROWSE_PER_PAGE, offset), + (PER_PAGE, offset), ).fetchall() items = "" for r in rows: @@ -406,7 +404,7 @@ def handle_pages(query=None): return _respond( f"
{total} page(s)
' f'