lichenblankie/tinyweb
2
0
Fork 0
Code Issues Pull requests Projects 1 Releases Packages Wiki Activity Actions

rns: store cleaned text instead of raw HTML body

Browse source
This commit is contained in:
blankie 2026-07-02 23:27:05 +00:00
parent ba1b8a783a
commit 4189bd7f24
2 changed files with 22 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/tinyweb/handlers/pages.py
View file

@ -90,7 +90,7 @@ def handle_add_submit(body):
return handle_add_form(f"Error: {esc(str(e))}")
except Exception as e:
error_msg = str(e).lower()
if any(x in error_msg for x in ("block", "cloudflare", "403", "ssl", "handshake", "max retries", "timeout", "connection")):
if any(x in error_msg for x in ("block", "cloudflare", "403", "429", "ssl", "handshake", "max retries", "timeout", "connection")):
return _respond(
f"<h1>add url (manual entry)</h1>"
f"<p><strong>{esc(url)}</strong> blocks automated access. "

34
src/tinyweb/handlers/rns.py
View file

@ -2,6 +2,8 @@ import json
import time
import threading
import traceback
import datetime
from bs4 import BeautifulSoup
from tinyweb.db import get_db, return_db
from tinyweb.rns_client import fetch_remote_page
from tinyweb.templates import esc
@ -69,26 +71,32 @@ def handle_rns_add_hash(dest_hash, name=""):
try:
resp = fetch_remote_page(dest_hash, "/")
if resp.get("status") == 200:
body = resp.get("body", "")
body_raw = resp.get("body", "")
soup = BeautifulSoup(body_raw, 'html.parser')
for tag in soup(["script", "style", "nav", "footer", "header", "noscript", "aside"]):
tag.decompose()
cleaned = soup.get_text(separator=" ", strip=True)
title = name or dest_hash[:16]
import re
m = re.search(r"<title[^>]*>(.*?)</title>", body, re.IGNORECASE | re.DOTALL)
if m:
title = m.group(1).strip()
if soup.title and soup.title.string:
title = soup.title.string.strip()
desc = ""
m = re.search(r'<meta\s+name="description"\s+content="([^"]*)"', body, re.IGNORECASE)
if m:
desc = m.group(1).strip()
m = soup.find("meta", attrs={"name": "description"})
if m and m.get("content"):
desc = m["content"].strip()
if not desc:
text = re.sub(r"<[^>]+>", " ", body)
text = re.sub(r"\s+", " ", text).strip()
desc = text[:200].strip()
m = soup.find("meta", attrs={"property": "og:description"})
if m and m.get("content"):
desc = m["content"].strip()
if not desc:
desc = cleaned[:200].strip()
url = f"rns:{dest_hash}"
import datetime
now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
db.execute(
"INSERT OR REPLACE INTO pages (url, title, body, last_modified, summary) VALUES (?, ?, ?, ?, ?)",
(url, title, body, now, desc),
(url, title, cleaned, now, desc),
)
else:
errors.append(f"Remote returned status {resp.get('status')}")