254 lines
8.5 KiB
Python
254 lines
8.5 KiB
Python
import requests
|
||
from bs4 import BeautifulSoup
|
||
|
||
from services.game_service import sync_match_page
|
||
from source_config import DATA_MATCH_BASE_URL, match_url
|
||
|
||
BASE_MATCH_URL = DATA_MATCH_BASE_URL
|
||
|
||
|
||
def fetch_html(url: str) -> str:
|
||
headers = {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
|
||
r = requests.get(url, headers=headers, timeout=20)
|
||
r.raise_for_status()
|
||
return r.text
|
||
|
||
|
||
def extract_player_id_from_href(href: str) -> str:
|
||
if not href:
|
||
return ""
|
||
return href.rstrip("/").split("/")[-1].strip()
|
||
|
||
|
||
def detect_captain(item) -> bool:
|
||
"""Определяет капитана по текстовой метке или отдельному HTML-маркеру сайта.
|
||
|
||
На странице протокола капитан может быть обозначен как ``(К)`` / ``(C)``,
|
||
а в некоторых версиях вёрстки — отдельным элементом/иконкой с captain в
|
||
class, title, aria-label или data-атрибуте. Если сайт вообще не указал
|
||
капитана, функция корректно возвращает False.
|
||
"""
|
||
if not item:
|
||
return False
|
||
|
||
text = item.get_text(" ", strip=True).lower().replace("ё", "е")
|
||
if any(marker in text for marker in ("(к)", "(c)", "капитан", "captain")):
|
||
return True
|
||
|
||
# Поддержка отдельной иконки/элемента капитана, если буква не входит
|
||
# в видимый текст строки игрока. Не привязываемся к одной версии вёрстки.
|
||
for node in [item, *item.find_all(True)]:
|
||
classes = " ".join(node.get("class", [])).lower()
|
||
attrs_text = " ".join(
|
||
str(node.get(attr) or "")
|
||
for attr in ("title", "aria-label", "data-title", "data-role", "data-captain")
|
||
).lower().replace("ё", "е")
|
||
|
||
if "captain" in classes or "капитан" in classes:
|
||
return True
|
||
if "captain" in attrs_text or "капитан" in attrs_text:
|
||
return True
|
||
if str(node.get("data-captain") or "").strip().lower() in {"1", "true", "yes"}:
|
||
return True
|
||
|
||
return False
|
||
|
||
def parse_starting_teams(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
|
||
home_starting = []
|
||
away_starting = []
|
||
|
||
start_teams = soup.select("div.protocol__block--main div.protocol__unit")
|
||
if len(start_teams) < 2:
|
||
return home_starting, away_starting
|
||
|
||
def parse_protocol_unit(unit):
|
||
bench = []
|
||
|
||
items = unit.select("ul.protocol__list li.protocol__item")
|
||
for item in items:
|
||
link = item.select_one("a.protocol__link")
|
||
if not link:
|
||
continue
|
||
|
||
href = link.get("href", "")
|
||
player_id = extract_player_id_from_href(href)
|
||
|
||
name_el = link.select_one("div.protocol__name")
|
||
role_el = link.select_one("div.protocol__role")
|
||
num_el = item.select_one("span.protocol__number-text")
|
||
# cap_el = link.select_one("div.protocol__captain")
|
||
|
||
bench.append(
|
||
{
|
||
"player_external_id": player_id,
|
||
"player_name": name_el.get_text(strip=True) if name_el else "",
|
||
"number": num_el.get_text(strip=True) if num_el else "",
|
||
"position": role_el.get_text(strip=True) if role_el else "",
|
||
"is_captain": detect_captain(item),
|
||
}
|
||
)
|
||
|
||
return bench
|
||
|
||
home_starting = parse_protocol_unit(start_teams[0])
|
||
away_starting = parse_protocol_unit(start_teams[1])
|
||
|
||
return home_starting, away_starting
|
||
|
||
|
||
def parse_bench(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
|
||
home_bench = []
|
||
away_bench = []
|
||
|
||
units = soup.select("div.protocol__block--additional div.protocol__unit")
|
||
if len(units) < 2:
|
||
return home_bench, away_bench
|
||
|
||
def parse_protocol_unit(unit):
|
||
bench = []
|
||
|
||
items = unit.select("ul.protocol__list li.protocol__item")
|
||
for item in items:
|
||
link = item.select_one("a.protocol__link")
|
||
if not link:
|
||
continue
|
||
|
||
href = link.get("href", "")
|
||
player_id = extract_player_id_from_href(href)
|
||
|
||
name_el = link.select_one("div.protocol__name")
|
||
role_el = link.select_one("div.protocol__role")
|
||
num_el = item.select_one("span.protocol__number-text")
|
||
# cap_el = link.select_one("div.protocol__captain")
|
||
|
||
|
||
bench.append(
|
||
{
|
||
"player_external_id": player_id,
|
||
"player_name": name_el.get_text(strip=True) if name_el else "",
|
||
"number": num_el.get_text(strip=True) if num_el else "",
|
||
"position": role_el.get_text(strip=True) if role_el else "",
|
||
"is_captain": detect_captain(item),
|
||
}
|
||
)
|
||
|
||
return bench
|
||
|
||
home_bench = parse_protocol_unit(units[0])
|
||
away_bench = parse_protocol_unit(units[1])
|
||
|
||
return home_bench, away_bench
|
||
|
||
|
||
def parse_coaches(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
|
||
home_coaches = []
|
||
away_coaches = []
|
||
|
||
units = soup.select("div.protocol__block--staff div.protocol__unit")
|
||
if len(units) < 2:
|
||
return home_coaches, away_coaches
|
||
|
||
def parse_staff_unit(unit):
|
||
coaches = []
|
||
|
||
items = unit.select("ul.protocol__list li.protocol__item a.protocol__link")
|
||
for link in items:
|
||
href = link.get("href", "")
|
||
coach_id = extract_player_id_from_href(href)
|
||
|
||
name_el = link.select_one("div.protocol__name")
|
||
role_el = link.select_one("div.protocol__staff-position")
|
||
|
||
coaches.append(
|
||
{
|
||
"coach_external_id": coach_id,
|
||
"coach_name": name_el.get_text(strip=True) if name_el else "",
|
||
"role": role_el.get_text(strip=True) if role_el else "",
|
||
}
|
||
)
|
||
|
||
return coaches
|
||
|
||
home_coaches = parse_staff_unit(units[0])
|
||
away_coaches = parse_staff_unit(units[1])
|
||
|
||
return home_coaches, away_coaches
|
||
|
||
|
||
def parse_referees(soup: BeautifulSoup) -> list[dict]:
|
||
referees = []
|
||
|
||
nodes = soup.select("div.protocol__block--referees div.referee")
|
||
for node in nodes:
|
||
role_el = node.select_one("p.referee__position")
|
||
first_el = node.select_one("span.referee__name")
|
||
last_el = node.select_one("span.referee__last-name")
|
||
|
||
first = first_el.get_text(strip=True) if first_el else ""
|
||
last = last_el.get_text(strip=True).split("(")[0] if last_el else ""
|
||
full_name = f"{first} {last}".strip()
|
||
|
||
if not full_name:
|
||
continue
|
||
|
||
referees.append(
|
||
{
|
||
"referee_name": full_name,
|
||
"role": (
|
||
role_el.get_text(strip=True).replace(":", "") if role_el else ""
|
||
),
|
||
}
|
||
)
|
||
|
||
return referees
|
||
|
||
|
||
def parse_game_page(html: str) -> dict:
|
||
soup = BeautifulSoup(html, "html.parser")
|
||
|
||
home_starting, away_starting = parse_starting_teams(soup)
|
||
home_bench, away_bench = parse_bench(soup)
|
||
home_coaches, away_coaches = parse_coaches(soup)
|
||
referees = parse_referees(soup)
|
||
|
||
return {
|
||
"home_starting": home_starting,
|
||
"away_starting": away_starting,
|
||
"home_bench": home_bench,
|
||
"away_bench": away_bench,
|
||
"home_coaches": home_coaches,
|
||
"away_coaches": away_coaches,
|
||
"referees": referees,
|
||
}
|
||
|
||
|
||
def run_parser_game(match_external_id: str) -> None:
|
||
url = match_url(match_external_id)
|
||
html = fetch_html(url)
|
||
data = parse_game_page(html)
|
||
|
||
if data:
|
||
sync_match_page(
|
||
match_external_id=str(match_external_id).strip(),
|
||
home_starting=data["home_starting"],
|
||
away_starting=data["away_starting"],
|
||
home_bench=data["home_bench"],
|
||
away_bench=data["away_bench"],
|
||
home_coaches=data["home_coaches"],
|
||
away_coaches=data["away_coaches"],
|
||
referees=data["referees"],
|
||
)
|
||
|
||
print(
|
||
f"[parser_game] match={match_external_id} "
|
||
f"home_starting={len(data['home_starting'])} "
|
||
f"away_starting={len(data['away_starting'])} "
|
||
f"home_bench={len(data['home_bench'])} "
|
||
f"away_bench={len(data['away_bench'])} "
|
||
f"home_coaches={len(data['home_coaches'])} "
|
||
f"away_coaches={len(data['away_coaches'])} "
|
||
f"referees={len(data['referees'])}"
|
||
)
|
||
|
||
|