import requests from bs4 import BeautifulSoup from services.game_service import sync_match_page from source_config import DATA_MATCH_BASE_URL, match_url BASE_MATCH_URL = DATA_MATCH_BASE_URL def fetch_html(url: str) -> str: headers = {"User-Agent": "Mozilla/5.0", "Accept": "*/*"} r = requests.get(url, headers=headers, timeout=20) r.raise_for_status() return r.text def extract_player_id_from_href(href: str) -> str: if not href: return "" return href.rstrip("/").split("/")[-1].strip() def detect_captain(item) -> bool: """Определяет капитана по текстовой метке или отдельному HTML-маркеру сайта. На странице протокола капитан может быть обозначен как ``(К)`` / ``(C)``, а в некоторых версиях вёрстки — отдельным элементом/иконкой с captain в class, title, aria-label или data-атрибуте. Если сайт вообще не указал капитана, функция корректно возвращает False. """ if not item: return False text = item.get_text(" ", strip=True).lower().replace("ё", "е") if any(marker in text for marker in ("(к)", "(c)", "капитан", "captain")): return True # Поддержка отдельной иконки/элемента капитана, если буква не входит # в видимый текст строки игрока. Не привязываемся к одной версии вёрстки. for node in [item, *item.find_all(True)]: classes = " ".join(node.get("class", [])).lower() attrs_text = " ".join( str(node.get(attr) or "") for attr in ("title", "aria-label", "data-title", "data-role", "data-captain") ).lower().replace("ё", "е") if "captain" in classes or "капитан" in classes: return True if "captain" in attrs_text or "капитан" in attrs_text: return True if str(node.get("data-captain") or "").strip().lower() in {"1", "true", "yes"}: return True return False def parse_starting_teams(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]: home_starting = [] away_starting = [] start_teams = soup.select("div.protocol__block--main div.protocol__unit") if len(start_teams) < 2: return home_starting, away_starting def parse_protocol_unit(unit): bench = [] items = unit.select("ul.protocol__list li.protocol__item") for item in items: link = item.select_one("a.protocol__link") if not link: continue href = link.get("href", "") player_id = extract_player_id_from_href(href) name_el = link.select_one("div.protocol__name") role_el = link.select_one("div.protocol__role") num_el = item.select_one("span.protocol__number-text") # cap_el = link.select_one("div.protocol__captain") bench.append( { "player_external_id": player_id, "player_name": name_el.get_text(strip=True) if name_el else "", "number": num_el.get_text(strip=True) if num_el else "", "position": role_el.get_text(strip=True) if role_el else "", "is_captain": detect_captain(item), } ) return bench home_starting = parse_protocol_unit(start_teams[0]) away_starting = parse_protocol_unit(start_teams[1]) return home_starting, away_starting def parse_bench(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]: home_bench = [] away_bench = [] units = soup.select("div.protocol__block--additional div.protocol__unit") if len(units) < 2: return home_bench, away_bench def parse_protocol_unit(unit): bench = [] items = unit.select("ul.protocol__list li.protocol__item") for item in items: link = item.select_one("a.protocol__link") if not link: continue href = link.get("href", "") player_id = extract_player_id_from_href(href) name_el = link.select_one("div.protocol__name") role_el = link.select_one("div.protocol__role") num_el = item.select_one("span.protocol__number-text") # cap_el = link.select_one("div.protocol__captain") bench.append( { "player_external_id": player_id, "player_name": name_el.get_text(strip=True) if name_el else "", "number": num_el.get_text(strip=True) if num_el else "", "position": role_el.get_text(strip=True) if role_el else "", "is_captain": detect_captain(item), } ) return bench home_bench = parse_protocol_unit(units[0]) away_bench = parse_protocol_unit(units[1]) return home_bench, away_bench def parse_coaches(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]: home_coaches = [] away_coaches = [] units = soup.select("div.protocol__block--staff div.protocol__unit") if len(units) < 2: return home_coaches, away_coaches def parse_staff_unit(unit): coaches = [] items = unit.select("ul.protocol__list li.protocol__item a.protocol__link") for link in items: href = link.get("href", "") coach_id = extract_player_id_from_href(href) name_el = link.select_one("div.protocol__name") role_el = link.select_one("div.protocol__staff-position") coaches.append( { "coach_external_id": coach_id, "coach_name": name_el.get_text(strip=True) if name_el else "", "role": role_el.get_text(strip=True) if role_el else "", } ) return coaches home_coaches = parse_staff_unit(units[0]) away_coaches = parse_staff_unit(units[1]) return home_coaches, away_coaches def parse_referees(soup: BeautifulSoup) -> list[dict]: referees = [] nodes = soup.select("div.protocol__block--referees div.referee") for node in nodes: role_el = node.select_one("p.referee__position") first_el = node.select_one("span.referee__name") last_el = node.select_one("span.referee__last-name") first = first_el.get_text(strip=True) if first_el else "" last = last_el.get_text(strip=True).split("(")[0] if last_el else "" full_name = f"{first} {last}".strip() if not full_name: continue referees.append( { "referee_name": full_name, "role": ( role_el.get_text(strip=True).replace(":", "") if role_el else "" ), } ) return referees def parse_game_page(html: str) -> dict: soup = BeautifulSoup(html, "html.parser") home_starting, away_starting = parse_starting_teams(soup) home_bench, away_bench = parse_bench(soup) home_coaches, away_coaches = parse_coaches(soup) referees = parse_referees(soup) return { "home_starting": home_starting, "away_starting": away_starting, "home_bench": home_bench, "away_bench": away_bench, "home_coaches": home_coaches, "away_coaches": away_coaches, "referees": referees, } def run_parser_game(match_external_id: str) -> None: url = match_url(match_external_id) html = fetch_html(url) data = parse_game_page(html) if data: sync_match_page( match_external_id=str(match_external_id).strip(), home_starting=data["home_starting"], away_starting=data["away_starting"], home_bench=data["home_bench"], away_bench=data["away_bench"], home_coaches=data["home_coaches"], away_coaches=data["away_coaches"], referees=data["referees"], ) print( f"[parser_game] match={match_external_id} " f"home_starting={len(data['home_starting'])} " f"away_starting={len(data['away_starting'])} " f"home_bench={len(data['home_bench'])} " f"away_bench={len(data['away_bench'])} " f"home_coaches={len(data['home_coaches'])} " f"away_coaches={len(data['away_coaches'])} " f"referees={len(data['referees'])}" )