229 lines
7.0 KiB
Python
229 lines
7.0 KiB
Python
import requests
|
|
from bs4 import BeautifulSoup
|
|
|
|
from services.game_service import sync_match_page
|
|
from source_config import DATA_MATCH_BASE_URL, match_url
|
|
|
|
BASE_MATCH_URL = DATA_MATCH_BASE_URL
|
|
|
|
|
|
def fetch_html(url: str) -> str:
|
|
headers = {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
|
|
r = requests.get(url, headers=headers, timeout=20)
|
|
r.raise_for_status()
|
|
return r.text
|
|
|
|
|
|
def extract_player_id_from_href(href: str) -> str:
|
|
if not href:
|
|
return ""
|
|
return href.rstrip("/").split("/")[-1].strip()
|
|
|
|
|
|
def detect_captain(item) -> bool:
|
|
if not item:
|
|
return False
|
|
|
|
text = item.get_text(" ", strip=True).lower()
|
|
|
|
return any(x in text for x in ["(к)", "(c)"])
|
|
|
|
def parse_starting_teams(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
|
|
home_starting = []
|
|
away_starting = []
|
|
|
|
start_teams = soup.select("div.protocol__block--main div.protocol__unit")
|
|
if len(start_teams) < 2:
|
|
return home_starting, away_starting
|
|
|
|
def parse_protocol_unit(unit):
|
|
bench = []
|
|
|
|
items = unit.select("ul.protocol__list li.protocol__item")
|
|
for item in items:
|
|
link = item.select_one("a.protocol__link")
|
|
if not link:
|
|
continue
|
|
|
|
href = link.get("href", "")
|
|
player_id = extract_player_id_from_href(href)
|
|
|
|
name_el = link.select_one("div.protocol__name")
|
|
role_el = link.select_one("div.protocol__role")
|
|
num_el = item.select_one("span.protocol__number-text")
|
|
# cap_el = link.select_one("div.protocol__captain")
|
|
|
|
bench.append(
|
|
{
|
|
"player_external_id": player_id,
|
|
"player_name": name_el.get_text(strip=True) if name_el else "",
|
|
"number": num_el.get_text(strip=True) if num_el else "",
|
|
"position": role_el.get_text(strip=True) if role_el else "",
|
|
"is_captain": detect_captain(item),
|
|
}
|
|
)
|
|
|
|
return bench
|
|
|
|
home_starting = parse_protocol_unit(start_teams[0])
|
|
away_starting = parse_protocol_unit(start_teams[1])
|
|
|
|
return home_starting, away_starting
|
|
|
|
|
|
def parse_bench(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
|
|
home_bench = []
|
|
away_bench = []
|
|
|
|
units = soup.select("div.protocol__block--additional div.protocol__unit")
|
|
if len(units) < 2:
|
|
return home_bench, away_bench
|
|
|
|
def parse_protocol_unit(unit):
|
|
bench = []
|
|
|
|
items = unit.select("ul.protocol__list li.protocol__item")
|
|
for item in items:
|
|
link = item.select_one("a.protocol__link")
|
|
if not link:
|
|
continue
|
|
|
|
href = link.get("href", "")
|
|
player_id = extract_player_id_from_href(href)
|
|
|
|
name_el = link.select_one("div.protocol__name")
|
|
role_el = link.select_one("div.protocol__role")
|
|
num_el = item.select_one("span.protocol__number-text")
|
|
# cap_el = link.select_one("div.protocol__captain")
|
|
|
|
|
|
bench.append(
|
|
{
|
|
"player_external_id": player_id,
|
|
"player_name": name_el.get_text(strip=True) if name_el else "",
|
|
"number": num_el.get_text(strip=True) if num_el else "",
|
|
"position": role_el.get_text(strip=True) if role_el else "",
|
|
"is_captain": detect_captain(item),
|
|
}
|
|
)
|
|
|
|
return bench
|
|
|
|
home_bench = parse_protocol_unit(units[0])
|
|
away_bench = parse_protocol_unit(units[1])
|
|
|
|
return home_bench, away_bench
|
|
|
|
|
|
def parse_coaches(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
|
|
home_coaches = []
|
|
away_coaches = []
|
|
|
|
units = soup.select("div.protocol__block--staff div.protocol__unit")
|
|
if len(units) < 2:
|
|
return home_coaches, away_coaches
|
|
|
|
def parse_staff_unit(unit):
|
|
coaches = []
|
|
|
|
items = unit.select("ul.protocol__list li.protocol__item a.protocol__link")
|
|
for link in items:
|
|
href = link.get("href", "")
|
|
coach_id = extract_player_id_from_href(href)
|
|
|
|
name_el = link.select_one("div.protocol__name")
|
|
role_el = link.select_one("div.protocol__staff-position")
|
|
|
|
coaches.append(
|
|
{
|
|
"coach_external_id": coach_id,
|
|
"coach_name": name_el.get_text(strip=True) if name_el else "",
|
|
"role": role_el.get_text(strip=True) if role_el else "",
|
|
}
|
|
)
|
|
|
|
return coaches
|
|
|
|
home_coaches = parse_staff_unit(units[0])
|
|
away_coaches = parse_staff_unit(units[1])
|
|
|
|
return home_coaches, away_coaches
|
|
|
|
|
|
def parse_referees(soup: BeautifulSoup) -> list[dict]:
|
|
referees = []
|
|
|
|
nodes = soup.select("div.protocol__block--referees div.referee")
|
|
for node in nodes:
|
|
role_el = node.select_one("p.referee__position")
|
|
first_el = node.select_one("span.referee__name")
|
|
last_el = node.select_one("span.referee__last-name")
|
|
|
|
first = first_el.get_text(strip=True) if first_el else ""
|
|
last = last_el.get_text(strip=True).split("(")[0] if last_el else ""
|
|
full_name = f"{first} {last}".strip()
|
|
|
|
if not full_name:
|
|
continue
|
|
|
|
referees.append(
|
|
{
|
|
"referee_name": full_name,
|
|
"role": (
|
|
role_el.get_text(strip=True).replace(":", "") if role_el else ""
|
|
),
|
|
}
|
|
)
|
|
|
|
return referees
|
|
|
|
|
|
def parse_game_page(html: str) -> dict:
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
|
|
home_starting, away_starting = parse_starting_teams(soup)
|
|
home_bench, away_bench = parse_bench(soup)
|
|
home_coaches, away_coaches = parse_coaches(soup)
|
|
referees = parse_referees(soup)
|
|
|
|
return {
|
|
"home_starting": home_starting,
|
|
"away_starting": away_starting,
|
|
"home_bench": home_bench,
|
|
"away_bench": away_bench,
|
|
"home_coaches": home_coaches,
|
|
"away_coaches": away_coaches,
|
|
"referees": referees,
|
|
}
|
|
|
|
|
|
def run_parser_game(match_external_id: str) -> None:
|
|
url = match_url(match_external_id)
|
|
html = fetch_html(url)
|
|
data = parse_game_page(html)
|
|
|
|
if data:
|
|
sync_match_page(
|
|
match_external_id=str(match_external_id).strip(),
|
|
home_starting=data["home_starting"],
|
|
away_starting=data["away_starting"],
|
|
home_bench=data["home_bench"],
|
|
away_bench=data["away_bench"],
|
|
home_coaches=data["home_coaches"],
|
|
away_coaches=data["away_coaches"],
|
|
referees=data["referees"],
|
|
)
|
|
|
|
print(
|
|
f"[parser_game] match={match_external_id} "
|
|
f"home_starting={len(data['home_starting'])} "
|
|
f"away_starting={len(data['away_starting'])} "
|
|
f"home_bench={len(data['home_bench'])} "
|
|
f"away_bench={len(data['away_bench'])} "
|
|
f"home_coaches={len(data['home_coaches'])} "
|
|
f"away_coaches={len(data['away_coaches'])} "
|
|
f"referees={len(data['referees'])}"
|
|
)
|
|
|
|
|