first commit

parsers/parser_game.py

@@ -0,0 +1,226 @@
+import requests
+from bs4 import BeautifulSoup
+
+from services.game_service import sync_match_page
+
+BASE_MATCH_URL = "https://wfl.rfs.ru/match/"
+
+
+def fetch_html(url: str) -> str:
+    headers = {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
+    r = requests.get(url, headers=headers, timeout=20)
+    r.raise_for_status()
+    return r.text
+
+
+def extract_player_id_from_href(href: str) -> str:
+    if not href:
+        return ""
+    return href.rstrip("/").split("/")[-1].strip()
+
+
+def detect_captain(item) -> bool:
+    if not item:
+        return False
+
+    text = item.get_text(" ", strip=True).lower()
+
+    return any(x in text for x in ["(к)", "(c)"])
+
+def parse_starting_teams(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
+    home_starting = []
+    away_starting = []
+
+    start_teams = soup.select("div.protocol__block--main div.protocol__unit")
+    if len(start_teams) < 2:
+        return home_starting, away_starting
+
+    def parse_protocol_unit(unit):
+        bench = []
+
+        items = unit.select("ul.protocol__list li.protocol__item")
+        for item in items:
+            link = item.select_one("a.protocol__link")
+            if not link:
+                continue
+
+            href = link.get("href", "")
+            player_id = extract_player_id_from_href(href)
+
+            name_el = link.select_one("div.protocol__name")
+            role_el = link.select_one("div.protocol__role")
+            num_el = item.select_one("span.protocol__number-text")
+            # cap_el = link.select_one("div.protocol__captain")
+
+            bench.append(
+                {
+                    "player_external_id": player_id,
+                    "player_name": name_el.get_text(strip=True) if name_el else "",
+                    "number": num_el.get_text(strip=True) if num_el else "",
+                    "position": role_el.get_text(strip=True) if role_el else "",
+                    "is_captain": detect_captain(item),
+                }
+            )
+
+        return bench
+
+    home_starting = parse_protocol_unit(start_teams[0])
+    away_starting = parse_protocol_unit(start_teams[1])
+
+    return home_starting, away_starting
+
+
+def parse_bench(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
+    home_bench = []
+    away_bench = []
+
+    units = soup.select("div.protocol__block--additional div.protocol__unit")
+    if len(units) < 2:
+        return home_bench, away_bench
+
+    def parse_protocol_unit(unit):
+        bench = []
+
+        items = unit.select("ul.protocol__list li.protocol__item")
+        for item in items:
+            link = item.select_one("a.protocol__link")
+            if not link:
+                continue
+
+            href = link.get("href", "")
+            player_id = extract_player_id_from_href(href)
+
+            name_el = link.select_one("div.protocol__name")
+            role_el = link.select_one("div.protocol__role")
+            num_el = item.select_one("span.protocol__number-text")
+            # cap_el = link.select_one("div.protocol__captain")
+
+
+            bench.append(
+                {
+                    "player_external_id": player_id,
+                    "player_name": name_el.get_text(strip=True) if name_el else "",
+                    "number": num_el.get_text(strip=True) if num_el else "",
+                    "position": role_el.get_text(strip=True) if role_el else "",
+                    "is_captain": detect_captain(item),
+                }
+            )
+
+        return bench
+
+    home_bench = parse_protocol_unit(units[0])
+    away_bench = parse_protocol_unit(units[1])
+
+    return home_bench, away_bench
+
+
+def parse_coaches(soup: BeautifulSoup) -> tuple[list[dict], list[dict]]:
+    home_coaches = []
+    away_coaches = []
+
+    units = soup.select("div.protocol__block--staff div.protocol__unit")
+    if len(units) < 2:
+        return home_coaches, away_coaches
+
+    def parse_staff_unit(unit):
+        coaches = []
+
+        items = unit.select("ul.protocol__list li.protocol__item a.protocol__link")
+        for link in items:
+            href = link.get("href", "")
+            coach_id = extract_player_id_from_href(href)
+
+            name_el = link.select_one("div.protocol__name")
+            role_el = link.select_one("div.protocol__staff-position")
+
+            coaches.append(
+                {
+                    "coach_external_id": coach_id,
+                    "coach_name": name_el.get_text(strip=True) if name_el else "",
+                    "role": role_el.get_text(strip=True) if role_el else "",
+                }
+            )
+
+        return coaches
+
+    home_coaches = parse_staff_unit(units[0])
+    away_coaches = parse_staff_unit(units[1])
+
+    return home_coaches, away_coaches
+
+
+def parse_referees(soup: BeautifulSoup) -> list[dict]:
+    referees = []
+
+    nodes = soup.select("div.protocol__block--referees div.referee")
+    for node in nodes:
+        role_el = node.select_one("p.referee__position")
+        first_el = node.select_one("span.referee__name")
+        last_el = node.select_one("span.referee__last-name")
+
+        first = first_el.get_text(strip=True) if first_el else ""
+        last = last_el.get_text(strip=True).split("(")[0] if last_el else ""
+        full_name = f"{first} {last}".strip()
+
+        if not full_name:
+            continue
+
+        referees.append(
+            {
+                "referee_name": full_name,
+                "role": (
+                    role_el.get_text(strip=True).replace(":", "") if role_el else ""
+                ),
+            }
+        )
+
+    return referees
+
+
+def parse_game_page(html: str) -> dict:
+    soup = BeautifulSoup(html, "html.parser")
+
+    home_starting, away_starting = parse_starting_teams(soup)
+    home_bench, away_bench = parse_bench(soup)
+    home_coaches, away_coaches = parse_coaches(soup)
+    referees = parse_referees(soup)
+
+    return {
+        "home_starting": home_starting,
+        "away_starting": away_starting,
+        "home_bench": home_bench,
+        "away_bench": away_bench,
+        "home_coaches": home_coaches,
+        "away_coaches": away_coaches,
+        "referees": referees,
+    }
+
+
+def run_parser_game(match_external_id: str) -> None:
+    url = f"{BASE_MATCH_URL}{str(match_external_id).strip()}"
+    html = fetch_html(url)
+    data = parse_game_page(html)
+
+    sync_match_page(
+        match_external_id=str(match_external_id).strip(),
+        home_starting=data["home_starting"],
+        away_starting=data["away_starting"],
+        home_bench=data["home_bench"],
+        away_bench=data["away_bench"],
+        home_coaches=data["home_coaches"],
+        away_coaches=data["away_coaches"],
+        referees=data["referees"],
+    )
+
+    print(
+        f"[parser_game] match={match_external_id} "
+        f"home_starting={len(data['home_starting'])} "
+        f"away_starting={len(data['away_starting'])} "
+        f"home_bench={len(data['home_bench'])} "
+        f"away_bench={len(data['away_bench'])} "
+        f"home_coaches={len(data['home_coaches'])} "
+        f"away_coaches={len(data['away_coaches'])} "
+        f"referees={len(data['referees'])}"
+    )
+
+

parsers/parser_players.py

@@ -0,0 +1,174 @@
+import requests
+from bs4 import BeautifulSoup
+from concurrent.futures import ThreadPoolExecutor
+
+from services.players_service import sync_team_roster
+
+
+URL_TEAMS = "https://wfl.rfs.ru/tournament/1061879/teams"
+
+AMPLUA_FULL = {
+    "Пз.": "Полузащитник",
+    "Вр.": "Вратарь",
+    "Зщ.": "Защитник",
+    "Нп.": "Нападающий",
+    "": "",
+}
+
+
+def fetch_html(url: str) -> str:
+    headers = {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
+    r = requests.get(url, headers=headers, timeout=20)
+    r.raise_for_status()
+    return r.text
+
+
+def get_links(html: str) -> list[dict]:
+    soup = BeautifulSoup(html, "html.parser")
+    links: list[dict] = []
+
+    items = soup.find("ul", class_="teams__list").find_all("li")
+    for i in items:
+        href = i.find("a", class_="teams__link").get("href")
+        team_external_id = href.split("team_id=")[-1].strip()
+
+        links.append(
+            {
+                "team_external_id": team_external_id,
+                "url": "https://wfl.rfs.ru" + href,
+            }
+        )
+
+    return links
+
+
+def parse_team(html: str) -> dict:
+    soup = BeautifulSoup(html, "html.parser")
+
+    team_el = soup.find("a", class_="team-promo__team-name")
+    team_name = team_el.get_text(strip=True) if team_el else None
+
+    tabs = soup.find("div", class_="tabs__content")
+    tbody = tabs.find("tbody") if tabs else None
+    row_players = tbody.find_all("tr", class_="table__row") if tbody else []
+
+    coaches_root = soup.find("ul", class_="composition-list")
+    row_coaches = (
+        coaches_root.find_all("div", class_="composition-list__item-back")
+        if coaches_root
+        else []
+    )
+
+    players = []
+    for row in row_players:
+        player_td = row.find("td", class_="table__cell table__cell--player")
+        player_a = player_td.find("a") if player_td else None
+        href = player_a.get("href") if player_a else None
+        player_id = href.split("/")[-1] if href else None
+
+        vars_td = row.find_all("td", class_="table__cell table__cell--variable")
+
+        games = vars_td[0].get_text(strip=True) if len(vars_td) > 0 else "0"
+
+        goals_raw = vars_td[1].get_text(strip=True) if len(vars_td) > 1 else ""
+        goals_parts = goals_raw.split()
+        goals = goals_parts[0] if len(goals_parts) > 0 else "0"
+        penaltys = (
+            goals_parts[1].replace("(", "").replace(")", "")
+            if len(goals_parts) > 1
+            else "0"
+        )
+
+        assists = vars_td[2].get_text(strip=True) if len(vars_td) > 2 else "0"
+        yellows = vars_td[3].get_text(strip=True) if len(vars_td) > 3 else "0"
+        reds = vars_td[4].get_text(strip=True) if len(vars_td) > 4 else "0"
+
+        number_td = row.find("td", class_="table__cell table__cell--number")
+        pos_td = row.find(
+            "td", class_="table__cell table__cell--amplua table__cell--amplua"
+        )
+        name_p = row.find("p", class_="table__player-name")
+        born_td = row.find("td", class_="table__cell table__cell--middle table__cell--birth mobile-hide")
+
+        full_player = name_p.get_text(strip=True) if name_p else ""
+        parts = full_player.split()
+
+        players.append(
+            {
+                "player_id": player_id or "",
+                "number": number_td.get_text(strip=True) if number_td else "",
+                "pos": pos_td.get_text(strip=True) if pos_td else "",
+                "amplua": AMPLUA_FULL[pos_td.get_text(strip=True) if pos_td else ""],
+                "player": full_player,
+                "lastname": parts[0] if len(parts) >= 1 else "",
+                "name": parts[-1] if len(parts) >= 2 else "",
+                "born": born_td.get_text(strip=True).split(",")[0] if born_td else "",
+                "games": games,
+                "goals": goals,
+                "penaltys": penaltys,
+                "assists": assists,
+                "yellows": yellows,
+                "reds": reds,
+            }
+        )
+
+    coaches = []
+    for coach in row_coaches:
+        coach_link = coach.find("a", class_="composition-list__player")
+        href = coach_link.get("href") if coach_link else ""
+        coach_id = href.split("/")[-1] if href else ""
+
+        name = coach.find("span", class_="composition-list__player-first-name")
+        lastname = coach.find("span", class_="composition-list__player-last-name")
+        born = coach.find("span", class_="composition-list__player-birth-date")
+        amplua = coach.find("span", class_="composition-list__player-games-text")
+
+        first_name = name.get_text(strip=True) if name else ""
+        last_name = lastname.get_text(strip=True) if lastname else ""
+
+        coaches.append(
+            {
+                "coach_id": coach_id,
+                "name": first_name,
+                "lastname": last_name,
+                "player": f"{last_name} {first_name}".strip(),
+                "born": born.get_text(strip=True).replace(",", "") if born else "",
+                "amplua": amplua.get_text(strip=True) if amplua else "",
+            }
+        )
+
+    return {
+        "team_name": team_name,
+        "players": players,
+        "coaches": coaches,
+    }
+
+
+def run_parser_players() -> None:
+    html = fetch_html(URL_TEAMS)
+    links = get_links(html)
+
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        futures = {pool.submit(fetch_html, item["url"]): item for item in links}
+
+        for future, item in futures.items():
+            try:
+                html = future.result()
+                team_data = parse_team(html)
+
+                sync_team_roster(
+                    team_external_id=item["team_external_id"],
+                    team_data=team_data,
+                )
+
+                print(
+                    f"[parser_players] team={item['team_external_id']} "
+                    f"players={len(team_data.get('players') or [])} "
+                    f"coaches={len(team_data.get('coaches') or [])}"
+                )
+            except Exception as e:
+                print(f"[parser_players] error team={item['team_external_id']}: {e}")
+
+
+if __name__ == "__main__":
+    run_parser_players()

parsers/parser_schedule.py

@@ -0,0 +1,173 @@
+import requests
+from bs4 import BeautifulSoup
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+from services.schedule_service import sync_matches
+from repositories.team_repository import get_team_external_id_by_name, get_team_id_by_external_id
+
+
+MONTHS_RU = {
+    "янв.": 1,
+    "февр.": 2,
+    "мар.": 3,
+    "апр.": 4,
+    "мая": 5,
+    "июн.": 6,
+    "июл.": 7,
+    "авг.": 8,
+    "сент.": 9,
+    "окт.": 10,
+    "нояб.": 11,
+    "дек.": 12,
+}
+
+TZ = ZoneInfo("Europe/Moscow")
+
+URL_SCHEDULE = (
+    "https://wfl.rfs.ru/tournament/1061879/calendar?round_id=1117550&type=tours"
+)
+
+SEASON = "2025/2026"
+
+
+def parse_russian_date(date_str: str, year: int | None = None) -> datetime:
+    # пример: "07 мар., пт, 13:00"
+    if year is None:
+        year = datetime.now(TZ).year
+
+    parts = [p.strip() for p in date_str.split(",")]
+    if len(parts) < 3:
+        raise ValueError(f"Некорректный формат даты: {date_str}")
+
+    day_month = parts[0]
+    time_part = parts[2]
+
+    day, month_str = day_month.split()
+    month = MONTHS_RU[month_str]
+
+    hh, mm = time_part.split(":")
+    return datetime(year, month, int(day), int(hh), int(mm), tzinfo=TZ)
+
+
+def fetch_html(url: str) -> str:
+    headers = {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
+    response = requests.get(url, headers=headers, timeout=20)
+    response.raise_for_status()
+    return response.text
+
+
+def normalize_status(score1: str | None, score2: str | None, dt: datetime) -> str:
+    now = datetime.now(TZ)
+
+    if score1 is not None and score2 is not None:
+        return "finished"
+
+    if dt <= now:
+        return "live"
+
+    return "scheduled"
+
+
+def safe_int(value: str | None) -> int | None:
+    if value is None:
+        return None
+    value = value.strip()
+    return int(value) if value.isdigit() else None
+
+
+def parse_schedule(html: str) -> list[dict]:
+    soup = BeautifulSoup(html, "html.parser")
+    matches_data: list[dict] = []
+
+    table_div = soup.find("div", class_="timetable__main")
+    if not table_div:
+        return matches_data
+
+    rows = table_div.find_all("div", class_="timetable__unit js-schedule-games-cont")
+
+    for row in rows:
+        tour_el = row.find("span", class_="timetable__head-text")
+        tour = tour_el.get_text(strip=True) if tour_el else None
+
+        matches = row.find_all("li", class_="timetable__item")
+        for match in matches:
+            score_link = match.find("a", class_="timetable__score")
+            href = score_link.get("href") if score_link else ""
+            match_id = href.split("/")[-1] if href else None
+
+            time_el = match.find("span", class_="timetable__time")
+            time_site = time_el.get_text(strip=True) if time_el else None
+            if not time_site or not match_id:
+                continue
+
+            try:
+                dt = parse_russian_date(time_site)
+            except Exception as exc:
+                print(f"[parser_schedule] Ошибка даты для матча {match_id}: {exc}")
+                continue
+
+            teams = match.find_all("div", class_="timetable__team-name")
+            team1_name = teams[0].get_text(strip=True) if len(teams) > 0 else None
+            team2_name = teams[1].get_text(strip=True) if len(teams) > 1 else None
+
+            if not team1_name or not team2_name:
+                print(f"[parser_schedule] Пропуск матча {match_id}: нет названий команд")
+                continue
+
+            home_team_external_id = get_team_external_id_by_name(team1_name)
+            away_team_external_id = get_team_external_id_by_name(team2_name)
+
+            if not home_team_external_id or not away_team_external_id:
+                print(
+                    f"[parser_schedule] Пропуск матча {match_id}: "
+                    f"не найдены команды в БД ({team1_name} vs {team2_name})"
+                )
+                continue
+
+            score1 = score2 = None
+            score_main_el = match.find("div", class_="timetable__score-main")
+            if score_main_el:
+                score_main = score_main_el.get_text(strip=True)
+                if "-" in score_main and "- : -" not in score_main:
+                    parts = [s.strip() for s in score_main.split("-")]
+                    if len(parts) == 2:
+                        score1, score2 = parts[0], parts[1]
+
+            home_score = safe_int(score1)
+            away_score = safe_int(score2)
+            status = normalize_status(home_score, away_score, dt)
+            place_el = match.find("span", class_="timetable__place-name")
+            place = place_el.get_text(strip=True) if place_el else None
+            score_add_el = match.find("div", class_="timetable__score-additional")
+            score_add = score_add_el.get_text(strip=True) if score_add_el else None
+            matches_data.append(
+                {
+                    "external_id": str(match_id),
+                    "home_team_external_id": str(home_team_external_id),
+                    "away_team_external_id": str(away_team_external_id),
+                    "match_date": dt.strftime("%Y-%m-%d %H:%M:%S"),
+                    "status": status,
+                    "home_score": home_score,
+                    "away_score": away_score,
+                    "tour": tour,
+                    "season": SEASON,
+                    "place": place,
+                    "date_raw": time_site,
+                    "score_add": score_add,
+                }
+            )
+            
+    matches_data.sort(key=lambda x: x["match_date"] or "")
+    return matches_data
+
+
+def run_parser_schedule() -> None:
+    html = fetch_html(URL_SCHEDULE)
+    matches_data = parse_schedule(html)
+    sync_matches(matches_data)
+    print(f"[parser_schedule] Matches synced: {len(matches_data)}")
+
+
+if __name__ == "__main__":
+    run_parser_schedule()

parsers/parser_standings.py

@@ -0,0 +1,81 @@
+import requests
+from bs4 import BeautifulSoup
+
+from services.standings_service import sync_standings
+
+URL_STANDINGS = "https://wfl.rfs.ru/tournament/1061879/tables"
+SEASON = "2025/2026"
+
+
+def fetch_html(url: str) -> str:
+    headers = {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
+    r = requests.get(url, headers=headers, timeout=20)
+    r.raise_for_status()
+    return r.text
+
+
+def parse_standings(html: str) -> list[dict]:
+    soup = BeautifulSoup(html, "html.parser")
+    standings: list[dict] = []
+
+    table_div = soup.find("ul", class_="custom-table__body")
+    if not table_div:
+        return standings
+
+    rows = table_div.find_all("li", class_="custom-table__line")
+
+    for row in rows:
+        rank = row.find("div", class_="custom-table__number-wrapper").get_text(strip=True)
+
+        team_name = row.find("div", class_="custom-table__team-name").get_text(strip=True)
+
+        team_external_id = (
+            row.find("a", class_="custom-table__team custom-table__cell")
+            .get("href")
+            .split("?team_id=")[-1]
+        )
+
+        vars_td = row.find_all("div", class_="custom-table__content")[1:]
+
+        games = int(vars_td[0].get_text(strip=True))
+        wins = int(vars_td[1].get_text(strip=True))
+        draws = int(vars_td[2].get_text(strip=True))
+        losses = int(vars_td[3].get_text(strip=True))
+
+        plus_minus = vars_td[4].get_text(strip=True)
+        gf = int(plus_minus.split("-")[0].strip())
+        ga = int(plus_minus.split("-")[1].strip())
+
+        points = int(vars_td[5].get_text(strip=True))
+
+        standings.append(
+            {
+                "team_external_id": str(team_external_id),
+                "played": games,
+                "wins": wins,
+                "losses": losses,
+                "draws": draws,
+                "points_for": gf,
+                "points_against": ga,
+                "points": points,
+                "position": int(rank),
+            }
+        )
+
+    return standings
+
+
+def run_parser_standings() -> None:
+    html = fetch_html(URL_STANDINGS)
+    standings_rows = parse_standings(html)
+
+    sync_standings(
+        season=SEASON,
+        standings_rows=standings_rows,
+    )
+
+    print(f"[parser_standings] Synced rows: {len(standings_rows)}")
+
+
+if __name__ == "__main__":
+    run_parser_standings()

parsers/parser_teams.py

@@ -0,0 +1,88 @@
+import requests
+from bs4 import BeautifulSoup
+from concurrent.futures import ThreadPoolExecutor
+
+from services.teams_service import sync_teams
+
+
+TEAMS_URL = "https://wfl.rfs.ru/tournament/1061879/teams"
+
+
+def fetch_html(url: str) -> str:
+    headers = {"User-Agent": "Mozilla/5.0", "Accept": "*/*"}
+    response = requests.get(url, timeout=30, headers=headers)
+    response.raise_for_status()
+    return response.text
+
+
+def get_links(html) -> list:
+    soup = BeautifulSoup(html, "html.parser")
+    links: list[dict] = []
+    items = soup.find("ul", class_="teams__list").find_all("li")
+    for i in items:
+        links.append(
+            "https://wfl.rfs.ru/team/"
+            + i.find("a", class_="teams__link").get("href").split("team_id=")[-1]
+        )
+    return links
+
+
+def get_url_teams() -> list[dict]:
+    html = fetch_html(TEAMS_URL)
+    links = get_links(html)
+    teams_data: list[dict] = []
+
+    with ThreadPoolExecutor() as pool:
+        responses = [
+            pool.submit(
+                fetch_html,
+                link,
+            )
+            for link in links
+        ]
+        for result in responses:
+            try:
+                html = result.result()
+                team_data = parse_teams_html(html)
+                teams_data.append(team_data)
+            except Exception as e:
+                print(f"Error fetching team data: {e}")
+
+    return teams_data
+
+
+def parse_teams_html(html: str) -> dict:
+    soup = BeautifulSoup(html, "html.parser")
+    teams_data: dict = {}
+    name = soup.find("a", class_="team-promo__team-name").text.strip()
+    external_id = soup.find("a", class_="team-promo__logo").get("href").split("/")[-1]
+    stat_info = soup.find("ul", class_="stats-info").find_all(
+        "div", class_="stats-info__number"
+    )
+    logo_url = soup.find("img", class_="team-promo__img").get("src")
+    games = stat_info[0].text.strip()
+    wins = stat_info[1].text.strip()
+    goals = stat_info[2].text.strip()
+    tournaments = stat_info[3].text.strip()
+
+    teams_data = {
+        "external_id": str(external_id),
+        "name": name,
+        "logo_url": logo_url,
+        "games": games,
+        "wins": wins,
+        "goals": goals,
+        "tournaments": tournaments,
+    }
+
+    return teams_data
+
+
+def run_parser_teams() -> None:
+    teams_data = get_url_teams()
+    sync_teams(teams_data)
+    print(f"Teams synced: {len(teams_data)}")
+
+
+if __name__ == "__main__":
+    run_parser_teams()