tags (actual content, not UI) + sentences = [] + if paragraphs: + raw = [] + for p in paragraphs: + raw.extend(re.split(r'(?<=[.!?])\s+', p)) + sentences = _filter_sentences(raw) + + # Fall back to full body text + if not sentences: + raw = re.split(r'(?<=[.!?])\s+', body) + sentences = _filter_sentences(raw) if not sentences: - # Last resort: take the first chunk of body that looks like prose - clean = re.sub(r'\s+', ' ', body).strip() - return clean[:160] + "..." if len(clean) > 160 else clean - if len(sentences) == 1: - s = sentences[0] - return s[:200] if len(s) > 200 else s - try: - from embeddings import embed - import numpy as np - embs = embed(sentences[:50]) # cap to avoid embedding too many - centroid = embs.mean(axis=0, keepdims=True) - centroid = centroid / max(np.linalg.norm(centroid), 1e-12) - scores = (embs @ centroid.T).flatten() - best_idx = int(np.argmax(scores)) - result = sentences[best_idx] - # Try to add a second sentence if it fits - if best_idx + 1 < len(sentences) and len(result) + len(sentences[best_idx + 1]) + 1 <= 200: - result += " " + sentences[best_idx + 1] - return result[:200] if len(result) > 200 else result - except Exception: - return sentences[0][:200] + return title[:200] if title else "" + + # Prefer a sentence that mentions the site name + if title: + title_words = [w.lower() for w in re.split(r'\W+', title) if len(w) >= 3] + for s in sentences: + s_lower = s.lower() + if sum(1 for w in title_words if w in s_lower) >= max(1, len(title_words) // 2): + return s[:200] + + # Otherwise use the first quality sentence + return sentences[0][:200] def index_url(url, note=""): url = clean_url(url) - title, body, links, meta_desc = fetch_page(url) - # Use meta description if available, otherwise generate from body - summary = meta_desc if meta_desc else _generate_summary(title, body) + title, body, links, meta_desc, paragraphs = fetch_page(url) + # Use meta description if available and meaningful, otherwise generate from body + summary = meta_desc if meta_desc and len(meta_desc) > 20 else _generate_summary(title, body, paragraphs) db = get_db() try: now = __import__("datetime").datetime.now().strftime("%Y-%m-%dT%H:%M:%S") diff --git a/embeddings.py b/embeddings.py index 5575c29..8ad1362 100644 --- a/embeddings.py +++ b/embeddings.py @@ -506,21 +506,24 @@ def hybrid_search(query_text, bm25_ranked_ids, limit=10, db=None, use_reranker=F # --------------------------------------------------------------------------- def reindex_all(db=None, progress_callback=None): - """Embed all pages that don't yet have chunks. Also generates missing summaries. Rebuilds HNSW index.""" + """Re-embed all pages and regenerate all summaries. Rebuilds HNSW index.""" from db import get_db, return_db, _generate_summary own_db = db is None if own_db: db = get_db() try: + # Clear existing chunks so everything is regenerated + db.execute("DELETE FROM chunks") + db.commit() + rows = db.execute( - "SELECT p.id, p.title, p.body, p.summary FROM pages p " - "WHERE p.id NOT IN (SELECT DISTINCT page_id FROM chunks WHERE page_id IS NOT NULL)" + "SELECT p.id, p.title, p.body, p.summary FROM pages p" ).fetchall() total = len(rows) for i, row in enumerate(rows): store_embeddings(row["id"], row["title"], row["body"], db) - # Generate summary if missing + # Only regenerate summary if missing if not row["summary"]: summary = _generate_summary(row["title"], row["body"]) db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"])) @@ -528,20 +531,9 @@ def reindex_all(db=None, progress_callback=None): if progress_callback: progress_callback(i + 1, total) - # Generate summaries for pages that already have chunks but no summary - no_summary = db.execute( - "SELECT id, title, body FROM pages WHERE summary = '' OR summary IS NULL" - ).fetchall() - for row in no_summary: - summary = _generate_summary(row["title"], row["body"]) - db.execute("UPDATE pages SET summary = ? WHERE id = ?", (summary, row["id"])) - if no_summary: - db.commit() - # Also handle remote pages remote_rows = db.execute( - "SELECT rp.id, rp.title, rp.note FROM remote_pages rp " - "WHERE rp.id NOT IN (SELECT DISTINCT remote_page_id FROM chunks WHERE remote_page_id IS NOT NULL)" + "SELECT rp.id, rp.title, rp.note FROM remote_pages rp" ).fetchall() for rp in remote_rows: diff --git a/handlers.py b/handlers.py index abd0d13..2f6f31f 100644 --- a/handlers.py +++ b/handlers.py @@ -116,6 +116,7 @@ def _error(status): PER_PAGE = 10 +BROWSE_PER_PAGE = 50 def _paginate(query, key="p"): @@ -126,10 +127,11 @@ def _paginate(query, key="p"): return max(1, page) -def _page_nav(page, total, base_url): - if total <= PER_PAGE: +def _page_nav(page, total, base_url, per_page=None): + per_page = per_page or PER_PAGE + if total <= per_page: return "" - total_pages = (total + PER_PAGE - 1) // PER_PAGE + total_pages = (total + per_page - 1) // per_page sep = "&" if "?" in base_url else "?" parts = [] if page > 1: @@ -377,13 +379,13 @@ def handle_add_submit(body): def handle_pages(query=None): page = _paginate(query or {}) - offset = (page - 1) * PER_PAGE + offset = (page - 1) * BROWSE_PER_PAGE db = get_db() try: total = db.execute("SELECT count(*) FROM pages").fetchone()[0] rows = db.execute( "SELECT id, url, title, note FROM pages ORDER BY id DESC LIMIT ? OFFSET ?", - (PER_PAGE, offset), + (BROWSE_PER_PAGE, offset), ).fetchall() items = "" for r in rows: @@ -404,7 +406,7 @@ def handle_pages(query=None): return _respond( f"
{total} page(s)
' f'