diff --git a/src/tinyweb/handlers/pages.py b/src/tinyweb/handlers/pages.py index 9d7f96c..05d885a 100644 --- a/src/tinyweb/handlers/pages.py +++ b/src/tinyweb/handlers/pages.py @@ -90,7 +90,7 @@ def handle_add_submit(body): return handle_add_form(f"Error: {esc(str(e))}") except Exception as e: error_msg = str(e).lower() - if any(x in error_msg for x in ("block", "cloudflare", "403", "ssl", "handshake", "max retries", "timeout", "connection")): + if any(x in error_msg for x in ("block", "cloudflare", "403", "429", "ssl", "handshake", "max retries", "timeout", "connection")): return _respond( f"

add url (manual entry)

" f"

{esc(url)} blocks automated access. " diff --git a/src/tinyweb/handlers/rns.py b/src/tinyweb/handlers/rns.py index 53a5616..086acde 100644 --- a/src/tinyweb/handlers/rns.py +++ b/src/tinyweb/handlers/rns.py @@ -2,6 +2,8 @@ import json import time import threading import traceback +import datetime +from bs4 import BeautifulSoup from tinyweb.db import get_db, return_db from tinyweb.rns_client import fetch_remote_page from tinyweb.templates import esc @@ -69,26 +71,32 @@ def handle_rns_add_hash(dest_hash, name=""): try: resp = fetch_remote_page(dest_hash, "/") if resp.get("status") == 200: - body = resp.get("body", "") + body_raw = resp.get("body", "") + soup = BeautifulSoup(body_raw, 'html.parser') + for tag in soup(["script", "style", "nav", "footer", "header", "noscript", "aside"]): + tag.decompose() + cleaned = soup.get_text(separator=" ", strip=True) + title = name or dest_hash[:16] - import re - m = re.search(r"]*>(.*?)", body, re.IGNORECASE | re.DOTALL) - if m: - title = m.group(1).strip() + if soup.title and soup.title.string: + title = soup.title.string.strip() + desc = "" - m = re.search(r']+>", " ", body) - text = re.sub(r"\s+", " ", text).strip() - desc = text[:200].strip() + m = soup.find("meta", attrs={"property": "og:description"}) + if m and m.get("content"): + desc = m["content"].strip() + if not desc: + desc = cleaned[:200].strip() + url = f"rns:{dest_hash}" - import datetime now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") db.execute( "INSERT OR REPLACE INTO pages (url, title, body, last_modified, summary) VALUES (?, ?, ?, ?, ?)", - (url, title, body, now, desc), + (url, title, cleaned, now, desc), ) else: errors.append(f"Remote returned status {resp.get('status')}")