"""Scoped readers for KHL's personality/entity cards and grouped statistics tables. No JavaScript execution or speculative API endpoints. Old layouts are handled by the collector's legacy readers. Browser-saved asset paths are never published. """ from __future__ import annotations import re from urllib.parse import urljoin, urlparse from bs4 import BeautifulSoup, Tag BASE = "https://www.khl.ru" PARSER_VERSION = 3 def text(value): return re.sub(r"\s+", " ", str(value or "")).strip() def image_url(node): if node is None: return "" src = text(node.get("data-src") or node.get("data-original") or node.get("src")) if not src or "_files/" in src or src.startswith(("data:", "file:")): return "" url = urljoin(BASE, src) return url if urlparse(url).scheme in ("https", "http") else "" def cell_text(node, *, images=False): # Tooltip descriptions must not become part of the abbreviated column name. value = text(" ".join(str(s) for s in node.find_all(string=True) if not any(p.get("role") == "tooltip" or "popup" in p.get("class", []) for p in s.parents if isinstance(p, Tag) and p is not node))) if not value and images: value = text(" ".join(text(i.get("alt")) for i in node.select("img[alt]"))) return value def profile_fields(soup, labels): root = soup.select_one(".personality-resume__main") if root is None: return None fields, extra = {}, {} for row in root.select(".personality-info__list-item"): cells = row.find_all(["div", "span"], recursive=False) if len(cells) < 2: continue label, value = cell_text(cells[0]).rstrip(" :"), cell_text(cells[1], images=True) if not value: continue if label.casefold() == "клуб": team = {"name": value} logo = image_url(cells[1].select_one("img")) if logo: team["image"] = logo fields["team"] = team elif label.casefold() in labels: fields[labels[label.casefold()]] = value else: extra[label] = value for node in root.select(".personality-attrs__item"): value = cell_text(node) for key, pattern in (("age", r"(\d+)\s*(?:лет|год|года)"), ("height", r"(\d+)\s*см"), ("weight", r"(\d+)\s*кг")): match = re.fullmatch(pattern, value, re.I) if match: fields[key] = int(match[1]) break else: if re.fullmatch(r"нападающий|защитник|вратарь|(?:главный |линейный )?(?:судья|тренер)", value, re.I): fields["role"] = value number = root.select_one(".personality-name__number") match = re.search(r"\d+", cell_text(number)) if number else None if match: fields["shirt_number"] = int(match[0]) if extra: fields["extra_fields"] = extra return fields def profile_name(soup): root = soup.select_one(".personality-resume__main") if root is None: return "" first, family = root.select_one(".personality-name__first"), root.select_one(".personality-name__full") if first is None or family is None: return "" last = re.sub(r"\s*№.*$", "", cell_text(family)).strip() return text(f"{cell_text(first)} {last}") def _span(cell, key): try: return min(100, max(1, int(cell.get(key, 1)))) except (TypeError, ValueError): return 1 def headers_grid(table): rows = table.select("thead tr") if not rows: return [], [] grid = {} for ri, row in enumerate(rows): ci = 0 for cell in row.find_all(["th", "td"], recursive=False): while (ri, ci) in grid: ci += 1 label = cell_text(cell) tip = cell.select_one('[role="tooltip"]') description = text(tip.get_text(" ", strip=True)) if tip else label for r in range(ri, ri + _span(cell, "rowspan")): for c in range(ci, ci + _span(cell, "colspan")): grid[r, c] = (label, description) ci += _span(cell, "colspan") width = max((c for _, c in grid), default=-1) + 1 headers, descriptions = [], [] for c in range(width): parts, desc = [], "" for r in range(len(rows)): label, description = grid.get((r, c), ("", "")) if label and label not in parts: parts.append(label) if description: desc = description if len(parts) > 1 and parts[-1] == "Всего": parts.pop() headers.append(" · ".join(parts)) descriptions.append(desc) return headers, descriptions def parse_table(table): if not any(str(c).startswith("statistics-") for c in table.get("class", [])): return None headers, descriptions = headers_grid(table) if not headers: return None rows, section, year = [], "", "" for tr in table.find_all("tr"): if tr.find_parent("thead"): continue cells = tr.find_all(["td", "th"], recursive=False) if not cells: continue values = [cell_text(c, images=True) for c in cells] if len(cells) <= 2 and cells[0].get("colspan"): label = values[0] if re.fullmatch(r"(?:19|20)\d{2}", label): year = label elif label: section, year = label, "" continue if not any(values): continue aligned = [] for cell, value in zip(cells, values): aligned.extend([value] + [""] * (_span(cell, "colspan") - 1)) links = [urljoin(BASE, a["href"]) for a in tr.select("a[href]")] images = [url for i in tr.select("img") if (url := image_url(i))] link_items = [{"title": cell_text(a, images=True), "url": urljoin(BASE, a["href"]), "image": image_url(a.select_one("img"))} for a in tr.select("a[href]")] rows.append({"values": aligned, "section": section, "year": year, "links": list(dict.fromkeys(links)), "images": list(dict.fromkeys(images)), "link_items": link_items}) return {"headers": headers, "descriptions": descriptions, "rows": rows, "caption": "", "layout": "khl2026"} MONTHS = {"янв": 1, "фев": 2, "мар": 3, "апр": 4, "мая": 5, "май": 5, "июн": 6, "июл": 7, "авг": 8, "сен": 9, "окт": 10, "ноя": 11, "дек": 12} def match_date(value, year=""): value = text(value) if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): return value match = re.fullmatch(r"(\d{1,2})\s+([а-яё]+)\.?\s*(\d{4})?", value, re.I) if match and (match[3] or year) and match[2].casefold()[:3] in MONTHS: from datetime import date try: return date(int(match[3] or year), MONTHS[match[2].casefold()[:3]], int(match[1])).isoformat() except ValueError: pass return "" def matches_payload(tables, page=1): table = next((t for t in tables if t.get("layout") == "khl2026" and t["headers"][:2] == ["Дата", "Команды"]), None) if table is None: return None rows = [] for row in table["rows"]: values = list(row["values"]) day = match_date(values[0], row.get("year", "")) if not day or len(values) != len(table["headers"]): continue values[0] = day url = next((u for u in row.get("links", []) if re.search(r"/(?:game|calendar|protocol|text|match)/", u)), "") rows.append({"section": row["section"], "values": [row["section"], *values], "url": url}) return {"headers": ["Турнир", *table["headers"]], "rows": rows, "page": page, "page_size": len(rows) or 20, "pages": 1, "total": len(rows), "range_start": 1 if rows else 0, "range_end": len(rows), "complete": False, "saved_only": True, "warning": "Показаны матчи, сохранённые при обновлении КХЛ. Полнота истории матчей не подтверждена."} def pagination(soup): pages = {int(n["data-page"]) for n in soup.select('.table-pagination__pagination [data-page]') if str(n.get("data-page", "")).isdigit()} return max(pages, default=1) def stats_link(soup, kind, pid): for node in soup.select('a[href]'): url = urlparse(urljoin(BASE, node["href"])) if (url.hostname in ("khl.ru", "www.khl.ru") and not url.username and not url.password and url.scheme in ("http", "https") and url.path.rstrip("/") == f"/{kind}/{pid}/stats"): return f"{BASE}/{kind}/{pid}/stats/" return ""