"""Incremental KHL archive import and read-only roster career analytics.""" from __future__ import annotations import csv import hashlib import io import json import re from datetime import datetime from threading import RLock from types import SimpleNamespace from urllib.parse import urlparse from sqlalchemy import and_, or_, select, text as sql_text from khl_site.person_data import player_name, staff_name, staff_view, valid_name from khl_site.career_stats import career_modes from .models import Game, GameRoster, Player, PlayerTournament, KhlCareerRecord, KhlImportState _IMPORT_LOCK=RLock() STAT_FIELDS={'gp':'games','g':'goals','a':'assists','pts':'points','pm':'plus_minus','pim':'penalty_minutes','shots':'shots','shot_pct':'shot_pct','toi_avg':'average_time_on_ice','g_pp':'power_play_goals','g_sh':'short_handed_goals','gwg':'game_winning_goals','hits':'hits','blocks':'blocked_shots','faceoff_pct':'faceoff_pct','w':'wins','l':'losses','ga':'goals_against','sv':'saves','sv_pct':'save_pct','gaa':'goals_against_average','so':'shutouts','shots_against':'shots_against','toi':'time_on_ice'} def name_key(name): return ' '.join(sorted(re.findall(r'[a-zа-я]+',str(name or '').casefold().replace('ё','е')))) if valid_name(name) or valid_name(name,'en') else '' def birthday(value): raw=str(value or '').strip() for fmt in ('%Y-%m-%d','%d.%m.%Y'): try:return datetime.strptime(raw,fmt).date().isoformat() except ValueError:pass months='января февраля марта апреля мая июня июля августа сентября октября ноября декабря'.split() parts=raw.casefold().split() if len(parts)==3 and parts[1] in months: try:return datetime(int(parts[2]),months.index(parts[1])+1,int(parts[0])).date().isoformat() except ValueError:pass return '' def raw_object(value): try: result=json.loads(value or '{}');return result if isinstance(result,dict) else {} except (ValueError,TypeError):return {} def explicit_khl_id(player): raw=raw_object(player.raw_payload) for key in ('khl_id','khlId','id_khl','idKhl'): value=str(raw.get(key) or '') if value.isdigit():return value url=urlparse(player.photo_url or '') if url.hostname and (url.hostname=='khl.ru' or url.hostname.endswith('.khl.ru')): m=re.search(r'/players/(\d+)(?:/|$)',url.path) if m:return m.group(1) return '' class KhlCareerService: def __init__(self,database): self.database=database def import_store(self,store): # Reload swaps immutable source objects. Keep references, not a second 50 MB copy. with store.lock: groups={'players':store.players_data,'coaches':store.coaches_data,'officials':store.officials_data} stages=store.player_stages source_versions=getattr(store,'source_versions',{}) imported=0 with _IMPORT_LOCK,self.database.session() as session: if session.bind.dialect.name=='postgresql': session.execute(sql_text('SELECT pg_advisory_xact_lock(20260927, 51)')) states={x.kind:x for x in session.scalars(select(KhlImportState))} for kind,source in groups.items(): people=source.get('players' if kind=='players' else 'people') or [] if not people:continue # Missing source never deletes the database archive. signature=hashlib.sha256(json.dumps({'schema': 2, 'source': source_versions.get(kind) or source},ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()).hexdigest() state=states.get(kind) if state and state.source_signature==signature:continue existing={row.khl_id:row for row in session.scalars(select(KhlCareerRecord).where(KhlCareerRecord.kind==kind))} for person in people: profile=person.get('profile') or {};identity=person.get('identity') or {} pid=str(identity.get('khl_id') or profile.get('khl_id') or identity.get('id') or person.get('id') or '') if not pid.isdigit():continue name=player_name(person) if kind=='players' else staff_name(person) data={'name':name,'name_en':player_name(person,'en') if kind=='players' else staff_name(person,'en'), 'profile':profile if kind=='players' else {k:v for k,v in profile.items() if k in ('fields','photos')}, 'statistics':career_modes(person,stages) if kind=='players' else staff_view(person,kind), 'source_updated_at':person.get('source_updated_at', source.get('meta',{}).get('generated_at','')), 'source_url':f'https://www.khl.ru/{kind}/{pid}/'} # Keep the complete parsed record, not just career totals. # No database migration needed: the existing JSON payload # stores all season rows, grouped columns and saved matches. data['archive'] = {k: v for k, v in person.items() if k != 'languages'} if kind == 'players': data['stage_metadata'] = {str(sid): stages[str(sid)] for sid in (person.get('stages') or {}) if str(sid) in stages} payload=json.dumps(data,ensure_ascii=False,separators=(',',':'),sort_keys=True) fingerprint=hashlib.sha256(payload.encode()).hexdigest() row=existing.get(pid) if row and row.fingerprint==fingerprint:continue if row is None: row=KhlCareerRecord(kind=kind,khl_id=pid);session.add(row) row.name_key=name_key(name);row.birth_date=birthday(profile.get('birthday') or (profile.get('fields') or {}).get('Дата рождения')) row.payload=payload;row.fingerprint=fingerprint;row.updated_at=datetime.utcnow();imported+=1 if state is None: state=KhlImportState(kind=kind);session.add(state) state.source_signature=signature;state.record_count=len(people);state.updated_at=datetime.utcnow() session.flush() counts={row.kind:row.record_count for row in session.scalars(select(KhlImportState))} return {'mode':'database','ok':True,'updated':imported,'counts':counts,'message':'Данные КХЛ сохранены в общей БД'} def game_career(self,external_id,*,all_members=False,mode='all',language='ru'): with self.database.session() as session: game=session.scalar(select(Game).where(Game.external_id==str(external_id))) if game is None:return None roster=list(session.scalars(select(GameRoster).where(GameRoster.game_external_id==str(external_id)))) roster_ids={r.player_external_id for r in roster} players={p.external_id:p for p in session.scalars(select(Player).where(Player.external_id.in_(roster_ids)))} if roster_ids else {} sides={side:{} for side in ('home','away')} for r in roster: if r.side in sides: person=players.get(r.player_external_id) if person is None: person=SimpleNamespace(external_id=r.player_external_id, first_name_ru='',last_name_ru='',full_name_ru='Имя не загружено', first_name_en='',last_name_en='',full_name_en='',birth_date=None,photo_url='',raw_payload='',country_code='') sides[r.side][r.player_external_id]=(person,r.number,r.role,True) if all_members: for side in sides: club=getattr(game,f'{side}_team_external_id');entry=getattr(game,f'{side}_entry_external_id') if not club and not entry:continue team_match=(or_(PlayerTournament.team_entry_external_id==entry,and_(PlayerTournament.team_entry_external_id=='',PlayerTournament.club_external_id==club)) if entry and club else PlayerTournament.team_entry_external_id==entry if entry else PlayerTournament.club_external_id==club) members=session.execute(select(Player,PlayerTournament).join(PlayerTournament,PlayerTournament.player_id==Player.id).where(PlayerTournament.tournament_external_id==game.tournament_external_id,PlayerTournament.active.is_(True),Player.active.is_(True),team_match)).all() for p,m in members:sides[side].setdefault(p.external_id,(p,m.jersey_number,p.position_en or p.position_ru,False)) candidates=[p for side in sides.values() for p,_,_,_ in side.values()] ids={str(p.external_id) for p in candidates}|{explicit_khl_id(p) for p in candidates} keys={name_key(p.full_name_ru or f'{p.first_name_ru} {p.last_name_ru}') for p in candidates};keys.discard('') records=list(session.scalars(select(KhlCareerRecord).where(KhlCareerRecord.kind=='players',or_(KhlCareerRecord.khl_id.in_(ids),KhlCareerRecord.name_key.in_(keys))))) if candidates else [] by_id={r.khl_id:r for r in records};by_name={} for r in records:by_name.setdefault(r.name_key,[]).append(r) result={'available':True,'scope':'team' if all_members else 'match','mode':mode,'source':'KHL.ru','home':{'skaters':[],'goalkeepers':[]},'away':{'skaters':[],'goalkeepers':[]},'missing':0,'matched':0} for side,side_players in sides.items(): result[side]['team_name']=getattr(game,f'{side}_team_name_{language}') or getattr(game,f'{side}_team_name_ru') for p,num,role,in_match in side_players.values(): ru=' '.join(filter(None,(p.first_name_ru,p.last_name_ru))) or p.full_name_ru en=' '.join(filter(None,(p.first_name_en,p.last_name_en))) or p.full_name_en key=name_key(ru);dob=birthday(p.birth_date);explicit=explicit_khl_id(p) record=by_id.get(explicit) if explicit else None if record and dob and record.birth_date and record.birth_date!=dob:record=None if record is None: candidate=by_id.get(str(p.external_id)) if candidate and dob and candidate.birth_date==dob and (not candidate.name_key or candidate.name_key==key):record=candidate if record is None and key and dob: matches=[r for r in by_name.get(key,[]) if r.birth_date==dob] if len(matches)==1:record=matches[0] data=raw_object(record.payload) if record else {};stats={} for metric in ((data.get('statistics') or {}).get(mode) or {}).get('stats') or []: if metric.get('id') in STAT_FIELDS and metric.get('val') not in (None,'','-','—'):stats[STAT_FIELDS[metric['id']]]=metric['val'] goalie=any(x in str(role).casefold() for x in ('врат','goal','gk')) or str(role).casefold()=='g' if not goalie:goalie=any(x in str((data.get('profile') or {}).get('role')).casefold() for x in ('врат','goal')) stats['available']=bool(stats);stats['kind']='goalkeeper' if goalie else 'skater' item={'id':p.external_id,'external_id':p.external_id,'side':side,'number':num,'name':(en if language=='en' else ru) or ru or en, 'names':{'ru':ru or data.get('name',''),'en':en or data.get('name_en','')}, 'role':'goalkeeper' if goalie else 'skater','position':role,'in_match':in_match,'country_code':p.country_code,'birth_date':dob or (record.birth_date if record else ''),'photo_url':p.photo_url, 'birth_date_source':'player_directory' if dob else 'KHL.ru' if record and record.birth_date else '', 'khl_id':record.khl_id if record else '', 'source_updated_at':data.get('source_updated_at',''),'source_url':data.get('source_url',''), 'career_status':'available' if stats['available'] else 'no_statistics' if record else 'unmatched','season_statistics':stats} result[side]['goalkeepers' if goalie else 'skaters'].append(item) result['matched' if stats['available'] else 'missing']+=1 for role_key in ('skaters','goalkeepers'):result[side][role_key].sort(key=lambda p:(int(p['number']) if str(p['number']).isdigit() else 999,p['name'])) return result def career_csv(payload,side="all"): output=io.StringIO(newline='');writer=csv.writer(output,delimiter=';') fields=list(dict.fromkeys(STAT_FIELDS.values())) labels=['Игры','Голы','Передачи','Очки','Плюс/минус','Штрафные минуты','Броски','Реализация, %','Среднее время на льду','Голы в большинстве','Голы в меньшинстве','Победные голы','Силовые приёмы','Блокированные броски','Вбрасывания, %','Победы','Поражения','Пропущено шайб','Отражено бросков','Отражено, %','Коэффициент надёжности','Сухие матчи','Броски по воротам','Время на льду'] writer.writerow(['Команда','Игрок','Роль','В заявке','KHL ID','Статус','Охват','Этап','Дата данных','Источник',*labels]) def safe(value): text=str(value if value is not None else '') return "'"+text if text.lstrip().startswith(('=','+','-','@','\t','\r')) and not isinstance(value,(int,float)) else text for side in (('home','away') if side=='all' else (side,)): for role in ('skaters','goalkeepers'): for p in payload[side][role]: stats=p['season_statistics'];writer.writerow([safe(v) for v in [payload[side]['team_name'],p['name'],p['role'],int(p['in_match']),p['khl_id'],{'available':'Есть данные','unmatched':'Не сопоставлен','no_statistics':'Нет статистики'}[p['career_status']],payload['scope'],payload['mode'],p['source_updated_at'],p['source_url'],*[stats.get(k) for k in fields]]]) return '\ufeff'+output.getvalue()