104 lines
5.0 KiB
Python
104 lines
5.0 KiB
Python
"""Decode and inspect public KHL HTML without executing scripts or retrying restrictions."""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from urllib.parse import urljoin, urlparse
|
||
|
||
from bs4 import BeautifulSoup, UnicodeDammit
|
||
|
||
BASE = 'https://www.khl.ru'
|
||
KINDS = ('players', 'coaches', 'officials')
|
||
|
||
|
||
def decode_response(response):
|
||
# requests defaults text/html without a charset to ISO-8859-1. KHL's UTF-8
|
||
# meta declaration must be decoded from bytes before parsing Russian names.
|
||
content = getattr(response, 'content', None)
|
||
if isinstance(content, bytes) and content:
|
||
decoded = UnicodeDammit(content, is_html=True)
|
||
if decoded.unicode_markup is not None:
|
||
return decoded.unicode_markup, decoded.original_encoding or ''
|
||
return str(getattr(response, 'text', '') or ''), str(getattr(response, 'encoding', '') or '')
|
||
|
||
|
||
def inspect_html(html):
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
compact = lambda value: re.sub(r'\s+', ' ', value).strip()
|
||
title = compact(soup.title.get_text(' ', strip=True)) if soup.title else ''
|
||
h1 = soup.find('h1')
|
||
heading = compact(h1.get_text(' ', strip=True)) if h1 else ''
|
||
for tag in soup(['script', 'style', 'noscript']):
|
||
tag.decompose()
|
||
body = compact(soup.get_text(' ', strip=True)).casefold().replace('ё', 'е')
|
||
head = f'{title} {heading}'.casefold().replace('ё', 'е')
|
||
status = next((int(m.group(1)) for text in (title, heading)
|
||
if (m := re.match(r'^\s*(?:(?:error|ошибка|http)\s*[:\-]?\s*)?([45]\d{2})(?:\b|$)', text, re.I))), None)
|
||
restricted = status in (401, 403, 429) or any(term in head for term in (
|
||
'access denied', 'forbidden', 'captcha', 'just a moment', 'проверка браузера', 'проверка безопасности', 'доступ ограничен', 'доступ запрещен'))
|
||
restricted = restricted or (len(body) < 4000 and any(term in body for term in (
|
||
'access_deny@khl.ru', 'вредоносная активность', 'доступ временно ограничен',
|
||
'ваш доступ временно ограничили', 'verify you are human', 'подтвердите, что вы не робот',
|
||
'access denied', 'forbidden', 'captcha', 'проверка браузера', 'доступ запрещен')))
|
||
if restricted:
|
||
reason = f'КХЛ ограничил доступ (HTML {status})' if status else 'КХЛ вернул страницу проверки или ограничения доступа'
|
||
elif status:
|
||
reason = f'КХЛ вернул страницу ошибки {status}'
|
||
elif len(html) < 600 or '<html' not in html[:12000].casefold():
|
||
reason = 'КХЛ вернул неполную HTML-страницу'
|
||
else:
|
||
reason = ''
|
||
return {'title': title[:200], 'heading': heading[:200], 'html_characters': len(html),
|
||
'text_characters': len(body), 'html_status': status, 'restricted': bool(restricted), 'reason': reason}
|
||
|
||
|
||
def person_url(kind, value):
|
||
if kind not in KINDS or not isinstance(value, str):
|
||
return None
|
||
try:
|
||
# Relative numeric links resolve against the directory, not site root.
|
||
url = urlparse(urljoin(f'{BASE}/{kind}/', value.strip()))
|
||
if url.scheme not in ('http', 'https') or url.hostname not in ('khl.ru', 'www.khl.ru') or url.username or url.password or url.port not in (None, 80, 443):
|
||
return None
|
||
match = re.fullmatch(rf'/{kind}/([1-9]\d{{0,14}})/?', url.path)
|
||
except ValueError:
|
||
return None
|
||
return (int(match[1]), f'{BASE}/{kind}/{int(match[1])}/') if match else None
|
||
|
||
|
||
def person_link_nodes(soup, kind):
|
||
for node in soup.select('[href], [data-href], [data-url]'):
|
||
for attribute in ('href', 'data-href', 'data-url'):
|
||
found = person_url(kind, node.get(attribute))
|
||
if found:
|
||
yield node, found
|
||
break
|
||
|
||
|
||
def detail_urls(kind, html):
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
found = dict(item for _, item in person_link_nodes(soup, kind))
|
||
# Hydrated/JSON-LD directories may embed canonical profile URLs in JSON.
|
||
# Only actual URLs are accepted; arbitrary numeric ids are never enumerated.
|
||
for script in soup.find_all('script', type=re.compile(r'^(?:application/(?:ld\+)?json)$', re.I)):
|
||
raw = script.string or script.get_text()
|
||
if len(raw) > 4_000_000:
|
||
continue
|
||
try:
|
||
document = json.loads(raw)
|
||
except (ValueError, RecursionError):
|
||
continue
|
||
pending = [document]
|
||
visited = 0
|
||
while pending and visited < 100000:
|
||
item = pending.pop(); visited += 1
|
||
if isinstance(item, dict):
|
||
pending.extend(item.values())
|
||
elif isinstance(item, list):
|
||
pending.extend(item)
|
||
elif isinstance(item, str):
|
||
link = person_url(kind, item)
|
||
if link:
|
||
found[link[0]] = link[1]
|
||
return found
|