update 7
This commit is contained in:
103
khl_site/html_source.py
Normal file
103
khl_site/html_source.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""Decode and inspect public KHL HTML without executing scripts or retrying restrictions."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
from bs4 import BeautifulSoup, UnicodeDammit
|
||||
|
||||
BASE = 'https://www.khl.ru'
|
||||
KINDS = ('players', 'coaches', 'officials')
|
||||
|
||||
|
||||
def decode_response(response):
|
||||
# requests defaults text/html without a charset to ISO-8859-1. KHL's UTF-8
|
||||
# meta declaration must be decoded from bytes before parsing Russian names.
|
||||
content = getattr(response, 'content', None)
|
||||
if isinstance(content, bytes) and content:
|
||||
decoded = UnicodeDammit(content, is_html=True)
|
||||
if decoded.unicode_markup is not None:
|
||||
return decoded.unicode_markup, decoded.original_encoding or ''
|
||||
return str(getattr(response, 'text', '') or ''), str(getattr(response, 'encoding', '') or '')
|
||||
|
||||
|
||||
def inspect_html(html):
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
compact = lambda value: re.sub(r'\s+', ' ', value).strip()
|
||||
title = compact(soup.title.get_text(' ', strip=True)) if soup.title else ''
|
||||
h1 = soup.find('h1')
|
||||
heading = compact(h1.get_text(' ', strip=True)) if h1 else ''
|
||||
for tag in soup(['script', 'style', 'noscript']):
|
||||
tag.decompose()
|
||||
body = compact(soup.get_text(' ', strip=True)).casefold().replace('ё', 'е')
|
||||
head = f'{title} {heading}'.casefold().replace('ё', 'е')
|
||||
status = next((int(m.group(1)) for text in (title, heading)
|
||||
if (m := re.match(r'^\s*(?:(?:error|ошибка|http)\s*[:\-]?\s*)?([45]\d{2})(?:\b|$)', text, re.I))), None)
|
||||
restricted = status in (401, 403, 429) or any(term in head for term in (
|
||||
'access denied', 'forbidden', 'captcha', 'just a moment', 'проверка браузера', 'проверка безопасности', 'доступ ограничен', 'доступ запрещен'))
|
||||
restricted = restricted or (len(body) < 4000 and any(term in body for term in (
|
||||
'access_deny@khl.ru', 'вредоносная активность', 'доступ временно ограничен',
|
||||
'ваш доступ временно ограничили', 'verify you are human', 'подтвердите, что вы не робот',
|
||||
'access denied', 'forbidden', 'captcha', 'проверка браузера', 'доступ запрещен')))
|
||||
if restricted:
|
||||
reason = f'КХЛ ограничил доступ (HTML {status})' if status else 'КХЛ вернул страницу проверки или ограничения доступа'
|
||||
elif status:
|
||||
reason = f'КХЛ вернул страницу ошибки {status}'
|
||||
elif len(html) < 600 or '<html' not in html[:12000].casefold():
|
||||
reason = 'КХЛ вернул неполную HTML-страницу'
|
||||
else:
|
||||
reason = ''
|
||||
return {'title': title[:200], 'heading': heading[:200], 'html_characters': len(html),
|
||||
'text_characters': len(body), 'html_status': status, 'restricted': bool(restricted), 'reason': reason}
|
||||
|
||||
|
||||
def person_url(kind, value):
|
||||
if kind not in KINDS or not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
# Relative numeric links resolve against the directory, not site root.
|
||||
url = urlparse(urljoin(f'{BASE}/{kind}/', value.strip()))
|
||||
if url.scheme not in ('http', 'https') or url.hostname not in ('khl.ru', 'www.khl.ru') or url.username or url.password or url.port not in (None, 80, 443):
|
||||
return None
|
||||
match = re.fullmatch(rf'/{kind}/([1-9]\d{{0,14}})/?', url.path)
|
||||
except ValueError:
|
||||
return None
|
||||
return (int(match[1]), f'{BASE}/{kind}/{int(match[1])}/') if match else None
|
||||
|
||||
|
||||
def person_link_nodes(soup, kind):
|
||||
for node in soup.select('[href], [data-href], [data-url]'):
|
||||
for attribute in ('href', 'data-href', 'data-url'):
|
||||
found = person_url(kind, node.get(attribute))
|
||||
if found:
|
||||
yield node, found
|
||||
break
|
||||
|
||||
|
||||
def detail_urls(kind, html):
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
found = dict(item for _, item in person_link_nodes(soup, kind))
|
||||
# Hydrated/JSON-LD directories may embed canonical profile URLs in JSON.
|
||||
# Only actual URLs are accepted; arbitrary numeric ids are never enumerated.
|
||||
for script in soup.find_all('script', type=re.compile(r'^(?:application/(?:ld\+)?json)$', re.I)):
|
||||
raw = script.string or script.get_text()
|
||||
if len(raw) > 4_000_000:
|
||||
continue
|
||||
try:
|
||||
document = json.loads(raw)
|
||||
except (ValueError, RecursionError):
|
||||
continue
|
||||
pending = [document]
|
||||
visited = 0
|
||||
while pending and visited < 100000:
|
||||
item = pending.pop(); visited += 1
|
||||
if isinstance(item, dict):
|
||||
pending.extend(item.values())
|
||||
elif isinstance(item, list):
|
||||
pending.extend(item)
|
||||
elif isinstance(item, str):
|
||||
link = person_url(kind, item)
|
||||
if link:
|
||||
found[link[0]] = link[1]
|
||||
return found
|
||||
@@ -330,8 +330,10 @@ def parse_khl_player_profile_html(html: str) -> dict[str, Any]:
|
||||
def _download_khl_player_page(khl_id: int, matches_page: int = 1) -> str:
|
||||
try:
|
||||
from .khl_site_collector import HttpClient, extract_names
|
||||
from .html_source import decode_response, inspect_html
|
||||
except ImportError:
|
||||
from khl_site_collector import HttpClient, extract_names
|
||||
from html_source import decode_response, inspect_html
|
||||
url = f"https://www.khl.ru/players/{khl_id}/"
|
||||
params = {"PAGEN_1": matches_page, "idplayer": khl_id} if matches_page > 1 else None
|
||||
headers = HttpClient.headers()
|
||||
@@ -342,8 +344,11 @@ def _download_khl_player_page(khl_id: int, matches_page: int = 1) -> str:
|
||||
if response.status_code in (403, 429):
|
||||
raise RuntimeError(f"КХЛ ограничил доступ (HTTP {response.status_code}). Повторите запрос позже")
|
||||
response.raise_for_status()
|
||||
html = response.text
|
||||
if not HttpClient.valid_html(html) or not valid_name(extract_names(BeautifulSoup(html, "html.parser"), "players")[0]):
|
||||
html, _ = decode_response(response)
|
||||
problem = inspect_html(html)
|
||||
if problem['reason']:
|
||||
raise RuntimeError(problem['reason'] + '; прежний кэш сохранён')
|
||||
if not valid_name(extract_names(BeautifulSoup(html, "html.parser"), "players")[0]):
|
||||
raise RuntimeError("Не удалось распознать страницу игрока КХЛ; прежний кэш сохранён")
|
||||
return html
|
||||
|
||||
@@ -843,6 +848,25 @@ def sync_storage() -> dict[str, Any]:
|
||||
|
||||
|
||||
|
||||
def read_update_report() -> dict[str, Any]:
|
||||
path = DATA_DIR / 'khl_update_report.json'
|
||||
try:
|
||||
if path.stat().st_size > 128_000:
|
||||
return {}
|
||||
value = json.loads(path.read_text(encoding='utf-8'))
|
||||
return value if isinstance(value, dict) and isinstance(value.get('items'), list) else {}
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def report_signature():
|
||||
try:
|
||||
stat = (DATA_DIR / 'khl_update_report.json').stat()
|
||||
return stat.st_mtime_ns, stat.st_size
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
class JobManager:
|
||||
def __init__(self) -> None:
|
||||
self.lock = threading.Lock()
|
||||
@@ -866,6 +890,7 @@ class JobManager:
|
||||
|
||||
def _run(self, commands: list[list[str]]) -> None:
|
||||
code = 0
|
||||
previous_report = report_signature()
|
||||
try:
|
||||
for command in commands:
|
||||
with self.lock:
|
||||
@@ -908,6 +933,24 @@ class JobManager:
|
||||
self.message = f"Обновлено карточек: {updated}." + (f" Проблем при загрузке: {issues}; недоступные данные сохранены из прежней базы." if issues else " Обновление завершено.")
|
||||
else:
|
||||
self.message = f"Обновление прервано: {self.message}. Ранее сохранённые карточки доступны."
|
||||
report = read_update_report() if report_signature() != previous_report else {}
|
||||
if report:
|
||||
labels = {'players': 'Игроки', 'coaches': 'Тренеры', 'officials': 'Судьи'}
|
||||
parts = []
|
||||
for item in report.get('items', []):
|
||||
label = labels.get(item.get('kind'), item.get('kind', 'КХЛ'))
|
||||
if item.get('status') in ('ok', 'partial'):
|
||||
message = f"{label}: обновлено {item.get('updated', 0)}"
|
||||
if item.get('discovery_fallback'):
|
||||
message += '; по ранее сохранённым карточкам, новые участники из списка не добавлены'
|
||||
elif item.get('issues'):
|
||||
message += '; часть данных недоступна'
|
||||
else:
|
||||
message = f"{label}: {item.get('message') or 'не обновлены'}"
|
||||
parts.append(message)
|
||||
self.message = '. '.join(parts)
|
||||
if report.get('result') != 'ok':
|
||||
self.message += '. Недоступные данные сохранены. Диагностику можно скачать кнопкой «Диагностика КХЛ».'
|
||||
if not storage.get("ok"):
|
||||
self.message += " " + storage["message"]
|
||||
|
||||
@@ -1009,6 +1052,17 @@ def api_job() -> dict[str, Any]:
|
||||
return JOBS.state()
|
||||
|
||||
|
||||
@APP.get('/api/diagnostics')
|
||||
def api_diagnostics():
|
||||
# This report has response status/title/encoding, never cookies or raw HTML.
|
||||
from fastapi.responses import Response
|
||||
report = read_update_report()
|
||||
if not report:
|
||||
raise HTTPException(404, 'Диагностика появится после следующего обновления КХЛ')
|
||||
return Response(json.dumps(report, ensure_ascii=False, indent=2), media_type='application/json',
|
||||
headers={'Content-Disposition': 'attachment; filename="khl-diagnostics.json"', 'Cache-Control': 'no-store'})
|
||||
|
||||
|
||||
@APP.post("/api/collect/{kind}")
|
||||
def api_collect(kind: str) -> dict[str, Any]:
|
||||
python = sys.executable
|
||||
|
||||
@@ -11,8 +11,8 @@
|
||||
python khl_site_collector.py --kind all
|
||||
|
||||
Парсер не использует khl.api.webcaster.pro и не требует Selenium.
|
||||
Для устойчивости к антибот-защите по возможности применяется curl_cffi
|
||||
с impersonate="chrome", с requests как резервным HTTP-транспортом.
|
||||
Для HTTP применяется curl_cffi или requests. При ограничении доступа
|
||||
сайтом запросы прекращаются, сохранённый архив остаётся доступным.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -45,8 +45,10 @@ except ImportError: # pragma: no cover
|
||||
|
||||
try:
|
||||
from .person_data import season_name, staff_view, valid_name
|
||||
from .html_source import decode_response, inspect_html, detail_urls, person_link_nodes, person_url
|
||||
except ImportError: # Direct CLI execution, including Windows subprocess.
|
||||
from person_data import season_name, staff_view, valid_name
|
||||
from html_source import decode_response, inspect_html, detail_urls, person_link_nodes, person_url
|
||||
|
||||
LOG = logging.getLogger("khl_site_collector")
|
||||
BASE = "https://www.khl.ru"
|
||||
@@ -894,6 +896,8 @@ class HttpClient:
|
||||
self.failure_lock = threading.Lock()
|
||||
self.consecutive_failures = 0
|
||||
self.blocked_error = ""
|
||||
self.last_problem = {}
|
||||
self.last_collection_meta = {}
|
||||
self.config.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@staticmethod
|
||||
@@ -917,7 +921,7 @@ class HttpClient:
|
||||
session = std_requests.Session()
|
||||
retry = Retry(total=self.config.retries, connect=self.config.retries, read=self.config.retries,
|
||||
status=self.config.retries, backoff_factor=0.8,
|
||||
status_forcelist=(408, 425, 429, 500, 502, 503, 504),
|
||||
status_forcelist=(408, 425, 500, 502, 503, 504), respect_retry_after_header=False,
|
||||
allowed_methods=frozenset({"GET"}), raise_on_status=False)
|
||||
adapter = HTTPAdapter(max_retries=retry, pool_connections=max(8, self.config.workers * 2), pool_maxsize=max(8, self.config.workers * 2))
|
||||
session.mount("https://", adapter); session.mount("http://", adapter)
|
||||
@@ -930,18 +934,11 @@ class HttpClient:
|
||||
|
||||
@staticmethod
|
||||
def valid_html(text: str, kind: str | None = None) -> bool:
|
||||
low = text[:12000].casefold()
|
||||
if len(text) < 600 or "<html" not in low:
|
||||
return False
|
||||
# A captcha script in the normal layout is not a challenge page.
|
||||
soup = BeautifulSoup(text, "html.parser")
|
||||
heading = clean(soup.title.get_text(" ", strip=True) if soup.title else "").casefold()
|
||||
if any(marker in heading for marker in (*BAD_MARKERS, "just a moment", "проверка безопасности")):
|
||||
return False
|
||||
for node in soup(["script", "style", "noscript"]):
|
||||
node.decompose()
|
||||
body = clean(soup.get_text(" ", strip=True)).casefold()
|
||||
return not (len(body) < 2000 and any(marker in body for marker in BAD_MARKERS))
|
||||
return not inspect_html(text)['reason']
|
||||
|
||||
def note_problem(self, url: str, reason: str, **details) -> None:
|
||||
with self.failure_lock:
|
||||
self.last_problem = {'url': url, 'reason': reason, 'at': utc_now(), **details}
|
||||
|
||||
def fetch(self, url: str, *, use_cache: bool = True) -> str:
|
||||
cache = self.cache_path(url)
|
||||
@@ -955,6 +952,9 @@ class HttpClient:
|
||||
raise RuntimeError(self.blocked_error)
|
||||
if self.config.delay:
|
||||
time.sleep(self.config.delay)
|
||||
with self.failure_lock:
|
||||
if self.blocked_error:
|
||||
raise RuntimeError(self.blocked_error)
|
||||
try:
|
||||
response = self.session().get(url, timeout=self.config.timeout, allow_redirects=True)
|
||||
except Exception as exc:
|
||||
@@ -962,20 +962,24 @@ class HttpClient:
|
||||
self.consecutive_failures += 1
|
||||
if self.consecutive_failures >= 8:
|
||||
self.blocked_error = "КХЛ не отвечает: запросы остановлены после 8 сетевых ошибок. Повторите обновление позже"
|
||||
raise RuntimeError(f"Нет ответа КХЛ: {type(exc).__name__}") from exc
|
||||
message = f"Нет ответа КХЛ: {type(exc).__name__}"
|
||||
self.note_problem(url, message)
|
||||
raise RuntimeError(message) from exc
|
||||
status = int(getattr(response, "status_code", 0))
|
||||
if status >= 400:
|
||||
reason = {403: "КХЛ ограничил доступ", 429: "КХЛ ограничил частоту запросов"}.get(status, "Ошибка сайта КХЛ")
|
||||
text, encoding = decode_response(response)
|
||||
report = inspect_html(text)
|
||||
if status >= 400 or report['reason']:
|
||||
reason = {401: "КХЛ требует авторизацию", 403: "КХЛ ограничил доступ", 429: "КХЛ ограничил частоту запросов"}.get(status)
|
||||
reason = reason or report['reason'] or "Ошибка сайта КХЛ"
|
||||
message = f"{reason} (HTTP {status}): {url}"
|
||||
if status in (403, 429):
|
||||
self.note_problem(url, message, http_status=status, encoding=encoding,
|
||||
**{key: value for key, value in report.items() if key != 'reason'})
|
||||
if status in (401, 403, 429) or report['restricted']:
|
||||
with self.failure_lock:
|
||||
self.blocked_error = message
|
||||
raise RuntimeError(message)
|
||||
with self.failure_lock:
|
||||
self.consecutive_failures = 0
|
||||
text = response.text
|
||||
if not self.valid_html(text):
|
||||
raise RuntimeError(f"KHL.ru вернул невалидную HTML-страницу: {url}")
|
||||
cache.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.cache_lock:
|
||||
cache.write_text(text, encoding="utf-8")
|
||||
@@ -983,17 +987,7 @@ class HttpClient:
|
||||
|
||||
|
||||
def detail_links(kind: str, html_text: str) -> dict[int, str]:
|
||||
soup = BeautifulSoup(html_text, "html.parser")
|
||||
found: dict[int, str] = {}
|
||||
pattern = DETAIL_RE[kind]
|
||||
for a in soup.find_all("a", href=True):
|
||||
url = urljoin(BASE, str(a.get("href")))
|
||||
parsed = urlparse(url)
|
||||
canonical = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
|
||||
match = pattern.match(canonical)
|
||||
if match:
|
||||
found[int(match.group(1))] = canonical if canonical.endswith("/") else canonical + "/"
|
||||
return found
|
||||
return detail_urls(kind, html_text)
|
||||
|
||||
|
||||
def _strip_list_label(value: str, *labels: str) -> str:
|
||||
@@ -1138,16 +1132,9 @@ def player_list_hints(html_text: str) -> dict[int, dict[str, Any]]:
|
||||
if data.get("name_raw") or data.get("role"):
|
||||
hints[pid] = {**hints.get(pid, {}), **data}
|
||||
|
||||
# Резерв: иногда ФИО есть прямо в ссылке, даже если таблица распарсилась иначе.
|
||||
for a in soup.find_all("a", href=True):
|
||||
url = urljoin(BASE, str(a.get("href")))
|
||||
parsed = urlparse(url)
|
||||
canonical = urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", "", ""))
|
||||
match = DETAIL_RE["players"].match(canonical)
|
||||
if not match:
|
||||
continue
|
||||
pid = int(match.group(1))
|
||||
text = clean(a.get_text(" ", strip=True))
|
||||
# Names in ordinary links and linked responsive cards share the same URL rules.
|
||||
for node, (pid, _) in person_link_nodes(soup, "players"):
|
||||
text = clean(node.get_text(" ", strip=True))
|
||||
if _looks_like_ru_person_name(text):
|
||||
hints.setdefault(pid, {})["name_raw"] = text
|
||||
return hints
|
||||
@@ -1179,6 +1166,8 @@ def discover_by_lists(kind: str, client: HttpClient, config: Config) -> tuple[di
|
||||
except Exception as exc:
|
||||
pages_meta.append({"url": base_url, "error": str(exc)})
|
||||
LOG.warning("%s список %s недоступен: %s", kind, base_url, exc)
|
||||
if getattr(client, 'blocked_error', ''):
|
||||
break
|
||||
continue
|
||||
first_links = detail_links(kind, first_html)
|
||||
found.update(first_links)
|
||||
@@ -1186,6 +1175,12 @@ def discover_by_lists(kind: str, client: HttpClient, config: Config) -> tuple[di
|
||||
_merge_player_hints(player_hints, player_list_hints(first_html))
|
||||
pages_meta.append({"url": base_url, "entry_count": len(first_links), "new_count": len(first_links)})
|
||||
if not first_links:
|
||||
info = inspect_html(first_html)
|
||||
reason = info['reason'] or f"Не распознаны ссылки на карточки в списке; заголовок: {info['title'] or 'не указан'}"
|
||||
pages_meta[-1].update({'warning': reason, 'title': info['title'], 'usable_html': not bool(info['reason'])})
|
||||
if hasattr(client, 'note_problem'):
|
||||
client.note_problem(base_url, reason, **{key: value for key, value in info.items() if key != 'reason'})
|
||||
LOG.warning("%s: %s", kind, reason)
|
||||
continue
|
||||
|
||||
empty_streak = 0
|
||||
@@ -1198,7 +1193,7 @@ def discover_by_lists(kind: str, client: HttpClient, config: Config) -> tuple[di
|
||||
pages_meta.append({"url": url, "error": str(exc)})
|
||||
LOG.warning("%s %s page %d: %s", kind, base_url, page, exc)
|
||||
empty_streak += 1
|
||||
if empty_streak >= 2:
|
||||
if empty_streak >= 2 or getattr(client, 'blocked_error', ''):
|
||||
break
|
||||
continue
|
||||
links = detail_links(kind, html)
|
||||
@@ -1220,7 +1215,7 @@ def discover_by_lists(kind: str, client: HttpClient, config: Config) -> tuple[di
|
||||
break
|
||||
if page % 10 == 0:
|
||||
LOG.info("%s: %s, страница %d, всего %d карточек", kind, base_url, page, len(found))
|
||||
if config.max_people and len(found) >= config.max_people:
|
||||
if (config.max_people and len(found) >= config.max_people) or getattr(client, 'blocked_error', ''):
|
||||
break
|
||||
return found, pages_meta, player_hints
|
||||
|
||||
@@ -1274,24 +1269,45 @@ def scan_official_ids(client: HttpClient, config: Config, existing: dict[int, st
|
||||
return out
|
||||
|
||||
|
||||
def collect_kind(kind: str, client: HttpClient, config: Config) -> Path:
|
||||
urls, pages, player_hints = discover_by_lists(kind, client, config)
|
||||
if not urls:
|
||||
detail = next((p["error"] for p in pages if p.get("error")), "В списке нет ссылок на персональные карточки; возможно, изменилась разметка КХЛ")
|
||||
raise RuntimeError(f"Не удалось прочитать {kind}: {detail}. Сохранённые данные не изменены")
|
||||
if config.max_people:
|
||||
urls = dict(list(sorted(urls.items()))[:config.max_people])
|
||||
LOG.info("%s: найдено %d карточек", kind, len(urls))
|
||||
def saved_profile_urls(kind: str, people: dict[int, dict[str, Any]]) -> dict[int, str]:
|
||||
result = {}
|
||||
for pid, person in people.items():
|
||||
profile = person.get('profile') or {}
|
||||
link = person_url(kind, profile.get('source_url') or person.get('url_ru') or person.get('url') or profile.get('url'))
|
||||
explicit = str((person.get('identity') or {}).get('khl_id') or profile.get('khl_id') or '')
|
||||
if link and link[0] == pid or explicit == str(pid):
|
||||
result[pid] = f'{BASE}/{kind}/{pid}/'
|
||||
return result
|
||||
|
||||
|
||||
def collect_kind(kind: str, client: HttpClient, config: Config) -> Path:
|
||||
out = config.output_dir / f"khl_{kind}_all.json"
|
||||
previous: dict[str, Any] = {}
|
||||
if out.exists():
|
||||
# A corrupt local file is never silently replaced by a partial collection.
|
||||
# Read the archive first: known profiles need not depend on list markup.
|
||||
previous = json.loads(out.read_text(encoding="utf-8"))
|
||||
item_key = "players" if kind == "players" else "people"
|
||||
def person_id(item):
|
||||
return int((item.get("identity") or {}).get("id") or (item.get("profile") or {}).get("id") or item.get("id"))
|
||||
old_people = {person_id(item): item for item in previous.get(item_key, [])}
|
||||
urls, pages, player_hints = discover_by_lists(kind, client, config)
|
||||
if getattr(client, 'blocked_error', ''):
|
||||
raise RuntimeError(f"{client.blocked_error}. Запросы остановлены; сохранённые данные не изменены")
|
||||
discovery_fallback = False
|
||||
if not urls and not config.names_only and not config.current_only and any(p.get('usable_html') for p in pages):
|
||||
urls = saved_profile_urls(kind, old_people)
|
||||
discovery_fallback = bool(urls)
|
||||
if urls:
|
||||
LOG.warning("%s: список не распознан. Обновляю %d ранее известных карточек КХЛ; новые участники из списка пока не добавляются", kind, len(urls))
|
||||
if not urls:
|
||||
detail = next((p["error"] for p in pages if p.get("error")), next((p['warning'] for p in pages if p.get('warning')), "В списке нет ссылок на персональные карточки"))
|
||||
raise RuntimeError(f"Не удалось прочитать {kind}: {detail}. Сохранённые данные не изменены")
|
||||
if config.max_people:
|
||||
urls = dict(list(sorted(urls.items()))[:config.max_people])
|
||||
LOG.info("%s: найдено %d карточек", kind, len(urls))
|
||||
previous_updated_at = previous.get('meta', {}).get('generated_at', '')
|
||||
def previous_date(pid):
|
||||
return old_people.get(pid, {}).get('source_updated_at', previous_updated_at)
|
||||
errors: list[dict[str, str]] = []
|
||||
people: list[Any] = []
|
||||
stage_map: dict[int, dict[str, Any]] = {}
|
||||
@@ -1366,6 +1382,7 @@ def collect_kind(kind: str, client: HttpClient, config: Config) -> Path:
|
||||
player["stages"] = copy.deepcopy(old_people[pid]["stages"])
|
||||
player["site_aggregates"] = copy.deepcopy(old_people[pid].get("site_aggregates") or {})
|
||||
player["profile"]["refresh_warning"] = "Статистика на странице не прочитана; сохранена предыдущая"
|
||||
player['source_updated_at'] = previous_date(pid) if config.names_only or profile.get('refresh_warning') else utc_now()
|
||||
return pid, (player, stages)
|
||||
staff = staff_from_profile(kind, pid, url, html)
|
||||
if not valid_name(staff.get("name_ru")):
|
||||
@@ -1377,6 +1394,7 @@ def collect_kind(kind: str, client: HttpClient, config: Config) -> Path:
|
||||
preserved[key] = staff[key]
|
||||
preserved["refresh_warning"] = "Статистика на странице не прочитана; сохранена предыдущая"
|
||||
staff = preserved
|
||||
staff['source_updated_at'] = previous_date(pid) if staff.get('refresh_warning') else utc_now()
|
||||
return pid, staff
|
||||
|
||||
items = list(sorted(urls.items()))
|
||||
@@ -1405,11 +1423,14 @@ def collect_kind(kind: str, client: HttpClient, config: Config) -> Path:
|
||||
updated_count = len(people)
|
||||
updated_ids = {person_id(item) for item in people}
|
||||
# Partial/current updates are upserts. Historical records and failed profiles survive.
|
||||
people.extend(item for pid, item in old_people.items() if pid not in updated_ids)
|
||||
for pid, item in old_people.items():
|
||||
if pid not in updated_ids:
|
||||
item.setdefault('source_updated_at', previous_updated_at)
|
||||
people.append(item)
|
||||
for stage in previous.get("stages", []):
|
||||
stage_map.setdefault(int(stage["id"]), stage)
|
||||
warning_count = sum(bool((p.get("profile") or {}).get("refresh_warning") or p.get("refresh_warning")) for p in people if person_id(p) in updated_ids)
|
||||
discovery_errors = [p for p in pages if p.get("error")]
|
||||
discovery_errors = [p for p in pages if p.get("error") or p.get('warning')]
|
||||
|
||||
if kind == "players":
|
||||
people.sort(key=lambda x: clean((x.get("identity") or {}).get("full_name")).casefold())
|
||||
@@ -1433,9 +1454,11 @@ def collect_kind(kind: str, client: HttpClient, config: Config) -> Path:
|
||||
if not people:
|
||||
raise RuntimeError(f"KHL.ru не отдал ни одной корректной карточки {kind}; старый JSON не перезаписан")
|
||||
payload["meta"].update({"updated_count": updated_count, "retained_count": len(people) - updated_count,
|
||||
"warning_count": warning_count, "discovery_error_count": len(discovery_errors)})
|
||||
"warning_count": warning_count, "discovery_error_count": len(discovery_errors),
|
||||
"discovery_fallback": discovery_fallback, "discovery_complete": not bool(discovery_errors)})
|
||||
payload["discovery_errors"] = discovery_errors
|
||||
atomic_json(out, payload)
|
||||
client.last_collection_meta = dict(payload['meta'])
|
||||
LOG.info("%s: обновлено %d, сохранено прежних %d, ошибок %d, предупреждений %d", kind, updated_count, len(people) - updated_count, len(errors) + len(discovery_errors), warning_count)
|
||||
LOG.info("Записано: %s", out.resolve())
|
||||
return out
|
||||
@@ -1478,16 +1501,40 @@ def parse_args() -> tuple[Config, list[str]]:
|
||||
def main() -> int:
|
||||
config, kinds = parse_args()
|
||||
client = HttpClient(config)
|
||||
report = {'started_at': utc_now(), 'finished_at': '', 'result': 'running', 'items': [], 'last_problem': {}}
|
||||
code = 0
|
||||
try:
|
||||
for kind in kinds:
|
||||
collect_kind(kind, client, config)
|
||||
try:
|
||||
collect_kind(kind, client, config)
|
||||
meta = dict(client.last_collection_meta)
|
||||
issues = sum(meta.get(k, 0) for k in ('error_count', 'warning_count', 'discovery_error_count'))
|
||||
report['items'].append({'kind': kind, 'status': 'partial' if issues else 'ok', 'updated': meta.get('updated_count', 0),
|
||||
'discovery_fallback': meta.get('discovery_fallback', False), 'issues': issues})
|
||||
except Exception as exc:
|
||||
code = 1
|
||||
report['items'].append({'kind': kind, 'status': 'error', 'updated': 0, 'message': str(exc)})
|
||||
LOG.error("%s: %s", kind, exc)
|
||||
if client.blocked_error:
|
||||
for skipped in kinds[kinds.index(kind) + 1:]:
|
||||
report['items'].append({'kind': skipped, 'status': 'skipped', 'updated': 0, 'message': 'Запросы остановлены из-за недоступности КХЛ'})
|
||||
break
|
||||
# A parsing error in one directory does not block the other directories.
|
||||
except KeyboardInterrupt:
|
||||
code = 130
|
||||
LOG.warning("Остановлено пользователем")
|
||||
return 130
|
||||
except Exception as exc:
|
||||
LOG.exception("Ошибка: %s", exc)
|
||||
return 1
|
||||
return 0
|
||||
finally:
|
||||
report['finished_at'] = utc_now()
|
||||
report['last_problem'] = client.last_problem
|
||||
report['result'] = 'ok' if code == 0 and all(x['status'] == 'ok' for x in report['items']) else 'partial' if any(x.get('updated') for x in report['items']) else 'error'
|
||||
try:
|
||||
atomic_json(config.output_dir / 'khl_update_report.json', report)
|
||||
except OSError as exc:
|
||||
LOG.warning("Не удалось записать диагностику: %s", exc)
|
||||
if code:
|
||||
messages = [x.get('message', '') for x in report['items'] if x['status'] == 'error']
|
||||
LOG.error("Обновление: %s", '; '.join(messages))
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -86,7 +86,7 @@ main{padding:18px;min-width:0;background:linear-gradient(180deg,#0b1018 0%,#090d
|
||||
<button data-collect onclick="collectData('coaches')">Тренеры</button>
|
||||
<button data-collect onclick="collectData('officials-current')">Судьи: текущие</button>
|
||||
<button data-collect onclick="collectData('officials-history')">Судьи: все</button>
|
||||
<button data-collect onclick="collectData('player-names')" title="Восстановить имена из списка КХЛ, сохранив статистику">Восстановить имена</button><button onclick="reloadData()">Перечитать данные</button>
|
||||
<button data-collect onclick="collectData('player-names')" title="Восстановить имена из списка КХЛ, сохранив статистику">Восстановить имена</button><button onclick="reloadData()">Перечитать данные</button><button onclick="downloadDiagnostics()">Диагностика КХЛ</button>
|
||||
</div>
|
||||
</header>
|
||||
<div class="layout">
|
||||
@@ -352,6 +352,13 @@ async function openItem(id,{updateSelection=true}={}){
|
||||
}catch(e){console.error(e);if(seq!==S.openSeq)return;$('detail').className='empty';$('detail').textContent=`Не удалось открыть карточку: ${e.message||e}`}
|
||||
}
|
||||
async function refreshViews(){captureListState();invalidateData();await loadStatus();await loadList(false,{restoreScroll:true});const id=selectedId();if(id)await openItem(id,{updateSelection:false})}
|
||||
async function downloadDiagnostics(){
|
||||
try {
|
||||
const report=await getJSON('/api/diagnostics');
|
||||
const url=URL.createObjectURL(new Blob([JSON.stringify(report,null,2)],{type:'application/json;charset=utf-8'}));
|
||||
const link=document.createElement('a');link.href=url;link.download='khl-diagnostics.json';link.click();setTimeout(()=>URL.revokeObjectURL(url),1000);
|
||||
} catch(error){showMessage('Диагностика КХЛ',error.message)}
|
||||
}
|
||||
async function collectData(kind){
|
||||
setCollectBusy(true);
|
||||
try{await getJSON(`/api/collect/${kind}`,{method:'POST'});await pollJob()}
|
||||
|
||||
Reference in New Issue
Block a user