223 lines
8.7 KiB
Python
223 lines
8.7 KiB
Python
"""Scoped readers for KHL's personality/entity cards and grouped statistics tables.
|
||
|
||
No JavaScript execution or speculative API endpoints. Old layouts are handled by
|
||
the collector's legacy readers. Browser-saved asset paths are never published.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from urllib.parse import urljoin, urlparse
|
||
|
||
from bs4 import BeautifulSoup, Tag
|
||
|
||
BASE = "https://www.khl.ru"
|
||
PARSER_VERSION = 3
|
||
|
||
|
||
def text(value):
|
||
return re.sub(r"\s+", " ", str(value or "")).strip()
|
||
|
||
|
||
def image_url(node):
|
||
if node is None:
|
||
return ""
|
||
src = text(node.get("data-src") or node.get("data-original") or node.get("src"))
|
||
if not src or "_files/" in src or src.startswith(("data:", "file:")):
|
||
return ""
|
||
url = urljoin(BASE, src)
|
||
return url if urlparse(url).scheme in ("https", "http") else ""
|
||
|
||
|
||
def cell_text(node, *, images=False):
|
||
# Tooltip descriptions must not become part of the abbreviated column name.
|
||
value = text(" ".join(str(s) for s in node.find_all(string=True)
|
||
if not any(p.get("role") == "tooltip" or "popup" in p.get("class", [])
|
||
for p in s.parents if isinstance(p, Tag) and p is not node)))
|
||
if not value and images:
|
||
value = text(" ".join(text(i.get("alt")) for i in node.select("img[alt]")))
|
||
return value
|
||
|
||
|
||
def profile_fields(soup, labels):
|
||
root = soup.select_one(".personality-resume__main")
|
||
if root is None:
|
||
return None
|
||
fields, extra = {}, {}
|
||
for row in root.select(".personality-info__list-item"):
|
||
cells = row.find_all(["div", "span"], recursive=False)
|
||
if len(cells) < 2:
|
||
continue
|
||
label, value = cell_text(cells[0]).rstrip(" :"), cell_text(cells[1], images=True)
|
||
if not value:
|
||
continue
|
||
if label.casefold() == "клуб":
|
||
team = {"name": value}
|
||
logo = image_url(cells[1].select_one("img"))
|
||
if logo:
|
||
team["image"] = logo
|
||
fields["team"] = team
|
||
elif label.casefold() in labels:
|
||
fields[labels[label.casefold()]] = value
|
||
else:
|
||
extra[label] = value
|
||
for node in root.select(".personality-attrs__item"):
|
||
value = cell_text(node)
|
||
for key, pattern in (("age", r"(\d+)\s*(?:лет|год|года)"),
|
||
("height", r"(\d+)\s*см"), ("weight", r"(\d+)\s*кг")):
|
||
match = re.fullmatch(pattern, value, re.I)
|
||
if match:
|
||
fields[key] = int(match[1])
|
||
break
|
||
else:
|
||
if re.fullmatch(r"нападающий|защитник|вратарь|(?:главный |линейный )?(?:судья|тренер)", value, re.I):
|
||
fields["role"] = value
|
||
number = root.select_one(".personality-name__number")
|
||
match = re.search(r"\d+", cell_text(number)) if number else None
|
||
if match:
|
||
fields["shirt_number"] = int(match[0])
|
||
if extra:
|
||
fields["extra_fields"] = extra
|
||
return fields
|
||
|
||
|
||
def profile_name(soup):
|
||
root = soup.select_one(".personality-resume__main")
|
||
if root is None:
|
||
return ""
|
||
first, family = root.select_one(".personality-name__first"), root.select_one(".personality-name__full")
|
||
if first is None or family is None:
|
||
return ""
|
||
last = re.sub(r"\s*№.*$", "", cell_text(family)).strip()
|
||
return text(f"{cell_text(first)} {last}")
|
||
|
||
|
||
def _span(cell, key):
|
||
try:
|
||
return min(100, max(1, int(cell.get(key, 1))))
|
||
except (TypeError, ValueError):
|
||
return 1
|
||
|
||
|
||
def headers_grid(table):
|
||
rows = table.select("thead tr")
|
||
if not rows:
|
||
return [], []
|
||
grid = {}
|
||
for ri, row in enumerate(rows):
|
||
ci = 0
|
||
for cell in row.find_all(["th", "td"], recursive=False):
|
||
while (ri, ci) in grid:
|
||
ci += 1
|
||
label = cell_text(cell)
|
||
tip = cell.select_one('[role="tooltip"]')
|
||
description = text(tip.get_text(" ", strip=True)) if tip else label
|
||
for r in range(ri, ri + _span(cell, "rowspan")):
|
||
for c in range(ci, ci + _span(cell, "colspan")):
|
||
grid[r, c] = (label, description)
|
||
ci += _span(cell, "colspan")
|
||
width = max((c for _, c in grid), default=-1) + 1
|
||
headers, descriptions = [], []
|
||
for c in range(width):
|
||
parts, desc = [], ""
|
||
for r in range(len(rows)):
|
||
label, description = grid.get((r, c), ("", ""))
|
||
if label and label not in parts:
|
||
parts.append(label)
|
||
if description:
|
||
desc = description
|
||
if len(parts) > 1 and parts[-1] == "Всего":
|
||
parts.pop()
|
||
headers.append(" · ".join(parts))
|
||
descriptions.append(desc)
|
||
return headers, descriptions
|
||
|
||
|
||
def parse_table(table):
|
||
if not any(str(c).startswith("statistics-") for c in table.get("class", [])):
|
||
return None
|
||
headers, descriptions = headers_grid(table)
|
||
if not headers:
|
||
return None
|
||
rows, section, year = [], "", ""
|
||
for tr in table.find_all("tr"):
|
||
if tr.find_parent("thead"):
|
||
continue
|
||
cells = tr.find_all(["td", "th"], recursive=False)
|
||
if not cells:
|
||
continue
|
||
values = [cell_text(c, images=True) for c in cells]
|
||
if len(cells) <= 2 and cells[0].get("colspan"):
|
||
label = values[0]
|
||
if re.fullmatch(r"(?:19|20)\d{2}", label):
|
||
year = label
|
||
elif label:
|
||
section, year = label, ""
|
||
continue
|
||
if not any(values):
|
||
continue
|
||
aligned = []
|
||
for cell, value in zip(cells, values):
|
||
aligned.extend([value] + [""] * (_span(cell, "colspan") - 1))
|
||
links = [urljoin(BASE, a["href"]) for a in tr.select("a[href]")]
|
||
images = [url for i in tr.select("img") if (url := image_url(i))]
|
||
link_items = [{"title": cell_text(a, images=True), "url": urljoin(BASE, a["href"]),
|
||
"image": image_url(a.select_one("img"))} for a in tr.select("a[href]")]
|
||
rows.append({"values": aligned, "section": section, "year": year,
|
||
"links": list(dict.fromkeys(links)), "images": list(dict.fromkeys(images)),
|
||
"link_items": link_items})
|
||
return {"headers": headers, "descriptions": descriptions, "rows": rows,
|
||
"caption": "", "layout": "khl2026"}
|
||
|
||
|
||
MONTHS = {"янв": 1, "фев": 2, "мар": 3, "апр": 4, "мая": 5, "май": 5,
|
||
"июн": 6, "июл": 7, "авг": 8, "сен": 9, "окт": 10, "ноя": 11, "дек": 12}
|
||
|
||
|
||
def match_date(value, year=""):
|
||
value = text(value)
|
||
if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
|
||
return value
|
||
match = re.fullmatch(r"(\d{1,2})\s+([а-яё]+)\.?\s*(\d{4})?", value, re.I)
|
||
if match and (match[3] or year) and match[2].casefold()[:3] in MONTHS:
|
||
from datetime import date
|
||
try:
|
||
return date(int(match[3] or year), MONTHS[match[2].casefold()[:3]], int(match[1])).isoformat()
|
||
except ValueError:
|
||
pass
|
||
return ""
|
||
|
||
|
||
def matches_payload(tables, page=1):
|
||
table = next((t for t in tables if t.get("layout") == "khl2026" and t["headers"][:2] == ["Дата", "Команды"]), None)
|
||
if table is None:
|
||
return None
|
||
rows = []
|
||
for row in table["rows"]:
|
||
values = list(row["values"])
|
||
day = match_date(values[0], row.get("year", ""))
|
||
if not day or len(values) != len(table["headers"]):
|
||
continue
|
||
values[0] = day
|
||
url = next((u for u in row.get("links", []) if re.search(r"/(?:game|calendar|protocol|text|match)/", u)), "")
|
||
rows.append({"section": row["section"], "values": [row["section"], *values], "url": url})
|
||
return {"headers": ["Турнир", *table["headers"]], "rows": rows, "page": page,
|
||
"page_size": len(rows) or 20, "pages": 1, "total": len(rows),
|
||
"range_start": 1 if rows else 0, "range_end": len(rows),
|
||
"complete": False, "saved_only": True,
|
||
"warning": "Показаны матчи, сохранённые при обновлении КХЛ. Полнота истории матчей не подтверждена."}
|
||
|
||
|
||
def pagination(soup):
|
||
pages = {int(n["data-page"]) for n in soup.select('.table-pagination__pagination [data-page]')
|
||
if str(n.get("data-page", "")).isdigit()}
|
||
return max(pages, default=1)
|
||
|
||
|
||
def stats_link(soup, kind, pid):
|
||
for node in soup.select('a[href]'):
|
||
url = urlparse(urljoin(BASE, node["href"]))
|
||
if (url.hostname in ("khl.ru", "www.khl.ru") and not url.username and not url.password
|
||
and url.scheme in ("http", "https") and url.path.rstrip("/") == f"/{kind}/{pid}/stats"):
|
||
return f"{BASE}/{kind}/{pid}/stats/"
|
||
return ""
|