Added manual entry

This commit is contained in:
blankie 2026-03-28 21:24:10 -07:00
parent c959ee98ae
commit 5593d802b3
2 changed files with 76 additions and 2 deletions

16
db.py
View file

@ -20,6 +20,18 @@ BLOCKED_NETWORKS = [
]
def _is_blocked_response(html, status_code):
"""Check if response is a CDN challenge/block page."""
if status_code == 403:
return True
html_lower = html.lower()
if "just a moment" in html_lower or "cloudflare" in html_lower:
return True
if "enable javascript and cookies" in html_lower:
return True
return False
def _validate_url_target(url):
"""Resolve hostname and block private/internal IPs to prevent SSRF."""
parsed = urlparse(url)
@ -281,6 +293,10 @@ def get_site_name():
def fetch_page(url):
_validate_url_target(url)
resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False)
if _is_blocked_response(resp.text, resp.status_code):
raise Exception("Site blocks automated access (Cloudflare/CDN protection)")
# Follow redirects manually, re-validating each target
max_redirects = 5
while resp.is_redirect and max_redirects > 0: