178 lines
14 KiB
Python
178 lines
14 KiB
Python
"""Incremental KHL archive import and read-only roster career analytics."""
|
||
from __future__ import annotations
|
||
import csv
|
||
import hashlib
|
||
import io
|
||
import json
|
||
import re
|
||
from datetime import datetime
|
||
from threading import RLock
|
||
from types import SimpleNamespace
|
||
from urllib.parse import urlparse
|
||
|
||
from sqlalchemy import and_, or_, select, text as sql_text
|
||
from khl_site.person_data import player_name, staff_name, staff_view, valid_name
|
||
from khl_site.career_stats import career_modes
|
||
from .models import Game, GameRoster, Player, PlayerTournament, KhlCareerRecord, KhlImportState
|
||
|
||
_IMPORT_LOCK=RLock()
|
||
STAT_FIELDS={'gp':'games','g':'goals','a':'assists','pts':'points','pm':'plus_minus','pim':'penalty_minutes','shots':'shots','shot_pct':'shot_pct','toi_avg':'average_time_on_ice','g_pp':'power_play_goals','g_sh':'short_handed_goals','gwg':'game_winning_goals','hits':'hits','blocks':'blocked_shots','faceoff_pct':'faceoff_pct','w':'wins','l':'losses','ga':'goals_against','sv':'saves','sv_pct':'save_pct','gaa':'goals_against_average','so':'shutouts','shots_against':'shots_against','toi':'time_on_ice'}
|
||
|
||
def name_key(name):
|
||
return ' '.join(sorted(re.findall(r'[a-zа-я]+',str(name or '').casefold().replace('ё','е')))) if valid_name(name) or valid_name(name,'en') else ''
|
||
|
||
def birthday(value):
|
||
raw=str(value or '').strip()
|
||
for fmt in ('%Y-%m-%d','%d.%m.%Y'):
|
||
try:return datetime.strptime(raw,fmt).date().isoformat()
|
||
except ValueError:pass
|
||
months='января февраля марта апреля мая июня июля августа сентября октября ноября декабря'.split()
|
||
parts=raw.casefold().split()
|
||
if len(parts)==3 and parts[1] in months:
|
||
try:return datetime(int(parts[2]),months.index(parts[1])+1,int(parts[0])).date().isoformat()
|
||
except ValueError:pass
|
||
return ''
|
||
|
||
def raw_object(value):
|
||
try:
|
||
result=json.loads(value or '{}');return result if isinstance(result,dict) else {}
|
||
except (ValueError,TypeError):return {}
|
||
|
||
def explicit_khl_id(player):
|
||
raw=raw_object(player.raw_payload)
|
||
for key in ('khl_id','khlId','id_khl','idKhl'):
|
||
value=str(raw.get(key) or '')
|
||
if value.isdigit():return value
|
||
url=urlparse(player.photo_url or '')
|
||
if url.hostname and (url.hostname=='khl.ru' or url.hostname.endswith('.khl.ru')):
|
||
m=re.search(r'/players/(\d+)(?:/|$)',url.path)
|
||
if m:return m.group(1)
|
||
return ''
|
||
|
||
class KhlCareerService:
|
||
def __init__(self,database):
|
||
self.database=database
|
||
|
||
def import_store(self,store):
|
||
# Reload swaps immutable source objects. Keep references, not a second 50 MB copy.
|
||
with store.lock:
|
||
groups={'players':store.players_data,'coaches':store.coaches_data,'officials':store.officials_data}
|
||
stages=store.player_stages
|
||
source_versions=getattr(store,'source_versions',{})
|
||
imported=0
|
||
with _IMPORT_LOCK,self.database.session() as session:
|
||
if session.bind.dialect.name=='postgresql':
|
||
session.execute(sql_text('SELECT pg_advisory_xact_lock(20260927, 51)'))
|
||
states={x.kind:x for x in session.scalars(select(KhlImportState))}
|
||
for kind,source in groups.items():
|
||
people=source.get('players' if kind=='players' else 'people') or []
|
||
if not people:continue # Missing source never deletes the database archive.
|
||
signature=hashlib.sha256(json.dumps({'schema': 2, 'source': source_versions.get(kind) or source},ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()).hexdigest()
|
||
state=states.get(kind)
|
||
if state and state.source_signature==signature:continue
|
||
existing={row.khl_id:row for row in session.scalars(select(KhlCareerRecord).where(KhlCareerRecord.kind==kind))}
|
||
for person in people:
|
||
profile=person.get('profile') or {};identity=person.get('identity') or {}
|
||
pid=str(identity.get('khl_id') or profile.get('khl_id') or identity.get('id') or person.get('id') or '')
|
||
if not pid.isdigit():continue
|
||
name=player_name(person) if kind=='players' else staff_name(person)
|
||
data={'name':name,'name_en':player_name(person,'en') if kind=='players' else staff_name(person,'en'),
|
||
'profile':profile if kind=='players' else {k:v for k,v in profile.items() if k in ('fields','photos')},
|
||
'statistics':career_modes(person,stages) if kind=='players' else staff_view(person,kind),
|
||
'source_updated_at':person.get('source_updated_at', source.get('meta',{}).get('generated_at','')),
|
||
'source_url':f'https://www.khl.ru/{kind}/{pid}/'}
|
||
# Keep the complete parsed record, not just career totals.
|
||
# No database migration needed: the existing JSON payload
|
||
# stores all season rows, grouped columns and saved matches.
|
||
data['archive'] = {k: v for k, v in person.items() if k != 'languages'}
|
||
if kind == 'players':
|
||
data['stage_metadata'] = {str(sid): stages[str(sid)] for sid in (person.get('stages') or {}) if str(sid) in stages}
|
||
payload=json.dumps(data,ensure_ascii=False,separators=(',',':'),sort_keys=True)
|
||
fingerprint=hashlib.sha256(payload.encode()).hexdigest()
|
||
row=existing.get(pid)
|
||
if row and row.fingerprint==fingerprint:continue
|
||
if row is None:
|
||
row=KhlCareerRecord(kind=kind,khl_id=pid);session.add(row)
|
||
row.name_key=name_key(name);row.birth_date=birthday(profile.get('birthday') or (profile.get('fields') or {}).get('Дата рождения'))
|
||
row.payload=payload;row.fingerprint=fingerprint;row.updated_at=datetime.utcnow();imported+=1
|
||
if state is None:
|
||
state=KhlImportState(kind=kind);session.add(state)
|
||
state.source_signature=signature;state.record_count=len(people);state.updated_at=datetime.utcnow()
|
||
session.flush()
|
||
counts={row.kind:row.record_count for row in session.scalars(select(KhlImportState))}
|
||
return {'mode':'database','ok':True,'updated':imported,'counts':counts,'message':'Данные КХЛ сохранены в общей БД'}
|
||
|
||
def game_career(self,external_id,*,all_members=False,mode='all',language='ru'):
|
||
with self.database.session() as session:
|
||
game=session.scalar(select(Game).where(Game.external_id==str(external_id)))
|
||
if game is None:return None
|
||
roster=list(session.scalars(select(GameRoster).where(GameRoster.game_external_id==str(external_id))))
|
||
roster_ids={r.player_external_id for r in roster}
|
||
players={p.external_id:p for p in session.scalars(select(Player).where(Player.external_id.in_(roster_ids)))} if roster_ids else {}
|
||
sides={side:{} for side in ('home','away')}
|
||
for r in roster:
|
||
if r.side in sides:
|
||
person=players.get(r.player_external_id)
|
||
if person is None:
|
||
person=SimpleNamespace(external_id=r.player_external_id, first_name_ru='',last_name_ru='',full_name_ru='Имя не загружено',
|
||
first_name_en='',last_name_en='',full_name_en='',birth_date=None,photo_url='',raw_payload='',country_code='')
|
||
sides[r.side][r.player_external_id]=(person,r.number,r.role,True)
|
||
if all_members:
|
||
for side in sides:
|
||
club=getattr(game,f'{side}_team_external_id');entry=getattr(game,f'{side}_entry_external_id')
|
||
if not club and not entry:continue
|
||
team_match=(or_(PlayerTournament.team_entry_external_id==entry,and_(PlayerTournament.team_entry_external_id=='',PlayerTournament.club_external_id==club)) if entry and club else PlayerTournament.team_entry_external_id==entry if entry else PlayerTournament.club_external_id==club)
|
||
members=session.execute(select(Player,PlayerTournament).join(PlayerTournament,PlayerTournament.player_id==Player.id).where(PlayerTournament.tournament_external_id==game.tournament_external_id,PlayerTournament.active.is_(True),Player.active.is_(True),team_match)).all()
|
||
for p,m in members:sides[side].setdefault(p.external_id,(p,m.jersey_number,p.position_en or p.position_ru,False))
|
||
candidates=[p for side in sides.values() for p,_,_,_ in side.values()]
|
||
ids={str(p.external_id) for p in candidates}|{explicit_khl_id(p) for p in candidates}
|
||
keys={name_key(p.full_name_ru or f'{p.first_name_ru} {p.last_name_ru}') for p in candidates};keys.discard('')
|
||
records=list(session.scalars(select(KhlCareerRecord).where(KhlCareerRecord.kind=='players',or_(KhlCareerRecord.khl_id.in_(ids),KhlCareerRecord.name_key.in_(keys))))) if candidates else []
|
||
by_id={r.khl_id:r for r in records};by_name={}
|
||
for r in records:by_name.setdefault(r.name_key,[]).append(r)
|
||
result={'available':True,'scope':'team' if all_members else 'match','mode':mode,'source':'KHL.ru','home':{'skaters':[],'goalkeepers':[]},'away':{'skaters':[],'goalkeepers':[]},'missing':0,'matched':0}
|
||
for side,side_players in sides.items():
|
||
result[side]['team_name']=getattr(game,f'{side}_team_name_{language}') or getattr(game,f'{side}_team_name_ru')
|
||
for p,num,role,in_match in side_players.values():
|
||
ru=' '.join(filter(None,(p.first_name_ru,p.last_name_ru))) or p.full_name_ru
|
||
en=' '.join(filter(None,(p.first_name_en,p.last_name_en))) or p.full_name_en
|
||
key=name_key(ru);dob=birthday(p.birth_date);explicit=explicit_khl_id(p)
|
||
record=by_id.get(explicit) if explicit else None
|
||
if record and dob and record.birth_date and record.birth_date!=dob:record=None
|
||
if record is None:
|
||
candidate=by_id.get(str(p.external_id))
|
||
if candidate and dob and candidate.birth_date==dob and (not candidate.name_key or candidate.name_key==key):record=candidate
|
||
if record is None and key and dob:
|
||
matches=[r for r in by_name.get(key,[]) if r.birth_date==dob]
|
||
if len(matches)==1:record=matches[0]
|
||
data=raw_object(record.payload) if record else {};stats={}
|
||
for metric in ((data.get('statistics') or {}).get(mode) or {}).get('stats') or []:
|
||
if metric.get('id') in STAT_FIELDS and metric.get('val') not in (None,'','-','—'):stats[STAT_FIELDS[metric['id']]]=metric['val']
|
||
goalie=any(x in str(role).casefold() for x in ('врат','goal','gk')) or str(role).casefold()=='g'
|
||
if not goalie:goalie=any(x in str((data.get('profile') or {}).get('role')).casefold() for x in ('врат','goal'))
|
||
stats['available']=bool(stats);stats['kind']='goalkeeper' if goalie else 'skater'
|
||
item={'id':p.external_id,'external_id':p.external_id,'side':side,'number':num,'name':(en if language=='en' else ru) or ru or en,
|
||
'names':{'ru':ru or data.get('name',''),'en':en or data.get('name_en','')},
|
||
'role':'goalkeeper' if goalie else 'skater','position':role,'in_match':in_match,'country_code':p.country_code,'birth_date':dob or (record.birth_date if record else ''),'photo_url':p.photo_url,
|
||
'birth_date_source':'player_directory' if dob else 'KHL.ru' if record and record.birth_date else '',
|
||
'khl_id':record.khl_id if record else '', 'source_updated_at':data.get('source_updated_at',''),'source_url':data.get('source_url',''),
|
||
'career_status':'available' if stats['available'] else 'no_statistics' if record else 'unmatched','season_statistics':stats}
|
||
result[side]['goalkeepers' if goalie else 'skaters'].append(item)
|
||
result['matched' if stats['available'] else 'missing']+=1
|
||
for role_key in ('skaters','goalkeepers'):result[side][role_key].sort(key=lambda p:(int(p['number']) if str(p['number']).isdigit() else 999,p['name']))
|
||
return result
|
||
|
||
def career_csv(payload,side="all"):
|
||
output=io.StringIO(newline='');writer=csv.writer(output,delimiter=';')
|
||
fields=list(dict.fromkeys(STAT_FIELDS.values()))
|
||
labels=['Игры','Голы','Передачи','Очки','Плюс/минус','Штрафные минуты','Броски','Реализация, %','Среднее время на льду','Голы в большинстве','Голы в меньшинстве','Победные голы','Силовые приёмы','Блокированные броски','Вбрасывания, %','Победы','Поражения','Пропущено шайб','Отражено бросков','Отражено, %','Коэффициент надёжности','Сухие матчи','Броски по воротам','Время на льду']
|
||
writer.writerow(['Команда','Игрок','Роль','В заявке','KHL ID','Статус','Охват','Этап','Дата данных','Источник',*labels])
|
||
def safe(value):
|
||
text=str(value if value is not None else '')
|
||
return "'"+text if text.lstrip().startswith(('=','+','-','@','\t','\r')) and not isinstance(value,(int,float)) else text
|
||
for side in (('home','away') if side=='all' else (side,)):
|
||
for role in ('skaters','goalkeepers'):
|
||
for p in payload[side][role]:
|
||
stats=p['season_statistics'];writer.writerow([safe(v) for v in [payload[side]['team_name'],p['name'],p['role'],int(p['in_match']),p['khl_id'],{'available':'Есть данные','unmatched':'Не сопоставлен','no_statistics':'Нет статистики'}[p['career_status']],payload['scope'],payload['mode'],p['source_updated_at'],p['source_url'],*[stats.get(k) for k in fields]]])
|
||
return '\ufeff'+output.getvalue()
|