Added manual entry
This commit is contained in:
parent
c959ee98ae
commit
5593d802b3
2 changed files with 76 additions and 2 deletions
16
db.py
16
db.py
|
|
@ -20,6 +20,18 @@ BLOCKED_NETWORKS = [
|
|||
]
|
||||
|
||||
|
||||
def _is_blocked_response(html, status_code):
|
||||
"""Check if response is a CDN challenge/block page."""
|
||||
if status_code == 403:
|
||||
return True
|
||||
html_lower = html.lower()
|
||||
if "just a moment" in html_lower or "cloudflare" in html_lower:
|
||||
return True
|
||||
if "enable javascript and cookies" in html_lower:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _validate_url_target(url):
|
||||
"""Resolve hostname and block private/internal IPs to prevent SSRF."""
|
||||
parsed = urlparse(url)
|
||||
|
|
@ -281,6 +293,10 @@ def get_site_name():
|
|||
def fetch_page(url):
|
||||
_validate_url_target(url)
|
||||
resp = requests.get(url, timeout=10, headers={"User-Agent": "TinyWeb/1.0"}, allow_redirects=False)
|
||||
|
||||
if _is_blocked_response(resp.text, resp.status_code):
|
||||
raise Exception("Site blocks automated access (Cloudflare/CDN protection)")
|
||||
|
||||
# Follow redirects manually, re-validating each target
|
||||
max_redirects = 5
|
||||
while resp.is_redirect and max_redirects > 0:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue