242 lines
15 KiB
Python
242 lines
15 KiB
Python
from __future__ import annotations
|
||
import json
|
||
from datetime import date
|
||
from types import SimpleNamespace
|
||
from threading import RLock
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from bs4 import BeautifulSoup
|
||
from sqlalchemy import select
|
||
from fastapi import FastAPI, HTTPException
|
||
from fastapi.testclient import TestClient
|
||
|
||
from khl_site import khl_site_collector as collector
|
||
from khl_site import khl_data_center as center
|
||
from khl_site.person_data import staff_view, player_name, season_name, valid_name
|
||
from khl_site.career_stats import aggregate
|
||
from hockey_data.khl_career import KhlCareerService, career_csv
|
||
from hockey_data.models import KhlCareerRecord, KhlImportState, Game, Player, PlayerTournament, GameRoster, Tournament, MatchControlLease
|
||
from tests.support import LocalTestDatabase
|
||
|
||
|
||
def player(pid=10,name='Иван Иванов',games=10,goals=3,shots=20):
|
||
return {'identity':{'id':pid,'khl_id':pid,'full_name':name,'first_name':name.split()[0],'last_name':name.split()[-1]},'profile':{'id':pid,'khl_id':pid,'name':name,'birthday':'1 января 2000','role':'нападающий'},'stages':{'1':{'data':{'team':{'name':'Клуб А'},'stats':[{'id':k,'val':v} for k,v in {'gp':games,'g':goals,'shots':shots,'pts':5,'a':2}.items()]}}}}
|
||
|
||
|
||
def old_coach():
|
||
return {'id':1,'name_ru':'Иванов Иван','profile':{'name':'Иванов Иван','fields':{'Дата рождения':'1 января 1970'},'teams':[{'title':'Чужой клуб из меню'}],'seasons':['2008/2026'], 'tables':[{'headers':['']+['Всего','Х','Г']*9,'rows':[{'section':'25/26 | рег.чемпионат','values':['Клуб А',10,5,5,4,2,2,1,1,0,1,0,1,1,1,0,1,0,1,2,1,1,14,7,7,70,70,70]}]}]}}
|
||
|
||
|
||
def old_official():
|
||
return {'id':2,'name_ru':'Петров Пётр','profile':{'fields':{'Амплуа':'Главный'},'seasons':['2026/2003','2008/2026'],'tables':[{'headers':['Сезон','№','Амп','И','Всего','Х','Г','Всего','Х','Г']+['Х','Г']*8,'rows':[{'section':'Главный судья','values':['Регулярный чемпионат 2025/2026',12,'гл.',10,200,80,120,20,8,12]+[2,3]*8}]}]}}
|
||
|
||
|
||
def config(tmp_path,**kwargs):
|
||
args=dict(output_dir=tmp_path,cache_dir=tmp_path/'html',workers=2,timeout=5,retries=0,delay=0,max_pages=4,max_people=0,force=False,official_scan_min=500,official_scan_max=1600,current_only=False)
|
||
return collector.Config(**(args|kwargs))
|
||
|
||
|
||
def test_names_never_use_global_navigation_and_preserve_given_name_order():
|
||
soup=BeautifulSoup('<html><head><title>Иванов Иван И., хоккеист: статистика КХЛ</title></head><body><nav><h1>Конференция Запад</h1></nav><h2>Иван Иванов</h2><div>Ivan Ivanov</div><p>Нападающий</p></body></html>','html.parser')
|
||
assert collector.extract_names(soup,'players')==('Иван Иванов','Ivan Ivanov')
|
||
p=player();p['profile']['list_name_ru']='Иванов Иван'
|
||
assert player_name(p)=='Иван Иванов'
|
||
assert not valid_name('Имя не загружено')
|
||
assert not valid_name('Конференция Запад')
|
||
assert valid_name('Ростислав Иванов')
|
||
assert player_name({'profile':{'name':'Конференция Запад'}})=='Имя не загружено'
|
||
|
||
|
||
def test_legacy_staff_tables_get_meaningful_stats_without_navigation():
|
||
c=staff_view(old_coach(),'coaches');r=c['rows'][0]
|
||
assert c['teams']==['Клуб А'] and c['seasons']==['2025/2026']
|
||
assert r['stats']['games']==10 and r['stats']['wins_total']==6 and r['stats']['wins_pct']==60
|
||
assert r['stats']['points_pct']==70
|
||
o=staff_view(old_official(),'officials');r=o['rows'][0]
|
||
assert o['teams']==[] and o['roles']==['Главный судья']
|
||
assert r['stats']['minor']==5 and r['stats']['penalty_avg']==20
|
||
assert r['stats']['home']['penalty_minutes']==80
|
||
assert season_name('© 2008–2026 КХЛ')=='' and season_name('2026-05-03')==''
|
||
|
||
|
||
def test_names_only_repairs_existing_statistics_without_profile_requests(tmp_path,monkeypatch):
|
||
p=player();p['identity']['full_name']='Конференция Запад';p['profile']['name']='Конференция Запад'
|
||
path=tmp_path/'khl_players_all.json';path.write_text(json.dumps({'players':[p],'stages':[{'id':1,'season':'2025/2026','title':'Регулярный чемпионат'}]}))
|
||
monkeypatch.setattr(collector,'discover_by_lists',lambda *a:({10:'https://www.khl.ru/players/10/'},[],{10:{'name_raw':'Иванов Иван'}}))
|
||
client=SimpleNamespace(fetch=lambda *a:pytest.fail('Names-only must not load 4000 profiles'))
|
||
collector.collect_kind('players',client,config(tmp_path,names_only=True))
|
||
result=json.loads(path.read_text());assert result['players'][0]['stages']==p['stages']
|
||
assert player_name(result['players'][0])=='Иван Иванов'
|
||
assert result['meta']['error_count']==0
|
||
|
||
|
||
def test_failed_refresh_retains_exact_file_and_partial_refresh_keeps_other_people(tmp_path,monkeypatch):
|
||
path=tmp_path/'khl_coaches_all.json';path.write_text(json.dumps({'people':[old_coach(),{'id':9,'name_ru':'Петров Пётр'}]}));before=path.read_bytes()
|
||
monkeypatch.setattr(collector,'discover_by_lists',lambda *a:({1:'https://www.khl.ru/coaches/1/'},[],{}))
|
||
def fail(*a):raise RuntimeError('HTTP 403')
|
||
with pytest.raises(RuntimeError):collector.collect_kind('coaches',SimpleNamespace(fetch=fail),config(tmp_path))
|
||
assert path.read_bytes()==before
|
||
monkeypatch.setattr(collector,'staff_from_profile',lambda *a:old_coach())
|
||
collector.collect_kind('coaches',SimpleNamespace(fetch=lambda *a:'ok'),config(tmp_path))
|
||
assert {p['id'] for p in json.loads(path.read_text())['people']}=={1,9}
|
||
|
||
|
||
def test_http_cache_expires_and_captcha_script_is_not_a_block_page(tmp_path):
|
||
html='<html><head><title>Игроки КХЛ</title><script src="captcha.js"></script></head><body>'+('Профиль '*150)+'</body></html>'
|
||
assert collector.HttpClient.valid_html(html)
|
||
assert not collector.HttpClient.valid_html('<html><title>Access denied</title><body>'+('x'*700)+'</body></html>')
|
||
client=collector.HttpClient(config(tmp_path,force=True));url='https://www.khl.ru/players/'
|
||
client.cache_path(url).write_text(html);calls=[]
|
||
client.session=lambda:SimpleNamespace(get=lambda *a,**k:(calls.append(a[0]) or SimpleNamespace(status_code=200,text=html)))
|
||
client.fetch(url);assert calls==[url]
|
||
|
||
|
||
def test_historical_pagination_does_not_stop_on_overlap(monkeypatch,tmp_path):
|
||
def links(ids):return ''.join(f'<a href="/players/{i}/">Иванов Иван</a>' for i in ids)
|
||
pages={'https://www.khl.ru/players/':links([1,2]),'https://www.khl.ru/players/?PAGEN_1=2':links([3,4]),'https://www.khl.ru/players/season/all/':links([1,2]),'https://www.khl.ru/players/season/all/?PAGEN_1=2':links([3,4]),'https://www.khl.ru/players/season/all/?PAGEN_1=3':links([5])}
|
||
found,_,_=collector.discover_by_lists('players',SimpleNamespace(fetch=lambda u:pages.get(u,'')),config(tmp_path))
|
||
assert set(found)=={1,2,3,4,5}
|
||
|
||
|
||
def test_career_percentages_are_ratios_and_missing_values_stay_unknown():
|
||
p=player(games=2,goals=1,shots=10);p['stages']['2']={'data':{'team':{'name':'Клуб Б'},'stats':[{'id':k,'val':v}for k,v in {'gp':8,'g':4,'shots':90,'toi_avg':'10:30'}.items()]}}
|
||
meta={'1':{'season':'2024/2025','title':'Регулярный чемпионат'},'2':{'season':'2025/2026','title':'Регулярный чемпионат'}}
|
||
values={s['id']:s['val']for s in aggregate(p,meta)['stats']}
|
||
assert values['gp']==10 and values['g']==5 and values['shot_pct']==5
|
||
assert 'pts' not in values and 'toi_avg' not in values
|
||
assert aggregate(p,meta,'playoff')['stats']==[]
|
||
|
||
|
||
@pytest.fixture
|
||
def career_env(tmp_path):
|
||
db=LocalTestDatabase(tmp_path/'career.sqlite');db.create_all();service=KhlCareerService(db)
|
||
store=SimpleNamespace(lock=RLock(),players_data={'players':[player(10),player(11,'Пётр Петров')]},coaches_data={'people':[old_coach()]},officials_data={'people':[old_official()]},player_stages={'1':{'season':'2025/2026','title':'Регулярный чемпионат'}})
|
||
service.import_store(store)
|
||
with db.session() as s:
|
||
s.add(Tournament(external_id='t'));s.add(Game(external_id='g',tournament_external_id='t',home_team_external_id='club',home_entry_external_id='entry',home_team_name_ru='Дом',away_team_name_ru='Гости'))
|
||
s.add(MatchControlLease(game_external_id='g',owner_user_id='operator',owner_participant_id='tab',generation=1,revision=4))
|
||
p=Player(external_id='external-10',first_name_ru='Иван',last_name_ru='Иванов',birth_date=date(2000,1,1),photo_url='https://img.khl.ru/players/10/160.jpg')
|
||
p2=Player(external_id='external-11',first_name_ru='Пётр',last_name_ru='Петров',birth_date=date(2000,1,1))
|
||
p3=Player(external_id='gone',first_name_ru='Алексей',last_name_ru='Ушедший',birth_date=date(2000,1,1))
|
||
s.add_all([p,p2,p3]);s.flush()
|
||
s.add(GameRoster(game_external_id='g',team_external_id='club',player_external_id=p.external_id,side='home',number='7',role='skater'))
|
||
for pl,active,t in ((p,True,'t'),(p2,True,'t'),(p3,False,'t'),(p3,True,'old')):s.add(PlayerTournament(player_id=pl.id,tournament_external_id=t,team_entry_external_id='entry',club_external_id='club',active=active,jersey_number='8'))
|
||
return db,service,store
|
||
|
||
|
||
def test_import_is_idempotent_and_roster_scope_excludes_inactive_and_other_seasons(career_env):
|
||
db,service,store=career_env
|
||
assert service.import_store(store)['updated']==0
|
||
match=service.game_career('g');full=service.game_career('g',all_members=True)
|
||
assert match['matched']==1 and len(match['home']['skaters'])==1
|
||
assert full['matched']==2 and {p['name']for p in full['home']['skaters']}=={'Иван Иванов','Пётр Петров'}
|
||
assert all(p['season_statistics']['games']==10 for p in full['home']['skaters'])
|
||
with db.session() as s:
|
||
assert s.scalar(select(MatchControlLease)).revision==4
|
||
assert len(list(s.scalars(select(KhlCareerRecord))))==4
|
||
assert len(list(s.scalars(select(KhlImportState))))==3
|
||
store.players_data={'players':[]};service.import_store(store)
|
||
assert service.game_career('g')['matched']==1
|
||
|
||
|
||
def test_ambiguous_name_and_conflicting_birth_date_do_not_attach_wrong_career(career_env):
|
||
db,service,store=career_env
|
||
with db.session() as s:
|
||
src=s.get(KhlCareerRecord,('players','11'))
|
||
s.add(KhlCareerRecord(kind='players',khl_id='99',name_key=src.name_key,birth_date=src.birth_date,payload=src.payload))
|
||
p=s.scalar(select(Player).where(Player.external_id=='external-10'));p.birth_date=date(2001,1,1)
|
||
value=service.game_career('g',all_members=True)
|
||
assert value['matched']==0 and value['missing']==2
|
||
assert value['home']['skaters'][0]['season_statistics']['available'] is False
|
||
|
||
|
||
def test_csv_includes_full_team_and_neutralizes_spreadsheet_formulas(career_env):
|
||
_,service,_=career_env
|
||
payload=service.game_career('g',all_members=True)
|
||
payload['home']['team_name']='=HYPERLINK("bad")'
|
||
result=career_csv(payload,'home')
|
||
assert result.startswith('\ufeff') and "'=HYPERLINK" in result
|
||
assert 'Иван Иванов' in result and 'Пётр Петров' in result
|
||
assert 'unmatched' not in result and 'https://www.khl.ru/players/10/' in result
|
||
|
||
|
||
def test_read_only_career_routes_work_for_viewer_and_require_login(career_env):
|
||
from hockey_data.router import create_hockey_router
|
||
db,_,_=career_env
|
||
def viewer():return SimpleNamespace(id='viewer',is_admin=False)
|
||
app=FastAPI();app.include_router(create_hockey_router(SimpleNamespace(database=db),auth_dependency=viewer))
|
||
with TestClient(app) as client:
|
||
assert client.get('/api/hockey/games/g/career?all_members=true').json()['matched']==2
|
||
assert client.get('/api/hockey/games/g/career.csv?side=home').status_code==200
|
||
assert client.get('/api/hockey/games/g/career?mode=bad').status_code==422
|
||
assert client.get('/api/hockey/games/missing/career').status_code==404
|
||
def expired():raise HTTPException(401,'login')
|
||
app.dependency_overrides[viewer]=expired
|
||
assert client.get('/api/hockey/games/g/career').status_code==401
|
||
|
||
|
||
def test_access_block_stops_further_network_requests(tmp_path):
|
||
client=collector.HttpClient(config(tmp_path));calls=[]
|
||
client.session=lambda:SimpleNamespace(get=lambda *a,**k:(calls.append(a[0]) or SimpleNamespace(status_code=403)))
|
||
for pid in (1,2,3):
|
||
with pytest.raises(RuntimeError,match='HTTP 403'):client.fetch(f'https://www.khl.ru/players/{pid}/')
|
||
assert len(calls)==1
|
||
|
||
|
||
def test_matches_failure_preserves_old_cache_and_marks_it_stale(tmp_path,monkeypatch):
|
||
path=tmp_path/'10_1.json';payload={'rows':[{'values':['2025-01-01','A — B','1:0']}],'page':1}
|
||
path.write_text(json.dumps({'fetched_at':1,'payload':payload}));before=path.read_bytes()
|
||
monkeypatch.setattr(center,'PLAYER_MATCHES_CACHE_DIR',tmp_path)
|
||
def blocked(*a):raise RuntimeError('HTTP 403')
|
||
monkeypatch.setattr(center,'_download_khl_player_page',blocked)
|
||
result=center.get_khl_site_player_matches(10)
|
||
assert result['rows']==payload['rows'] and result['stale'] is True and '403' in result['warning']
|
||
assert path.read_bytes()==before
|
||
|
||
|
||
def test_refresh_api_uses_current_python_fresh_cache_and_bounded_workers(monkeypatch):
|
||
captured=[]
|
||
monkeypatch.setattr(center,'JOBS',SimpleNamespace(start=lambda t,c:captured.extend(c),state=lambda:{'running':True}))
|
||
center.api_collect('player-names')
|
||
command=captured[0]
|
||
assert command[1]=='-u' and '--force' in command and '--names-only' in command
|
||
assert command[command.index('--workers')+1]=='3'
|
||
|
||
|
||
def test_corrupt_local_snapshot_keeps_previously_loaded_data(tmp_path,monkeypatch):
|
||
p=tmp_path/'players.json';p.write_text(json.dumps({'players':[player()]}))
|
||
monkeypatch.setattr(center,'PLAYERS_FILE',p)
|
||
monkeypatch.setattr(center,'COACHES_FILE',tmp_path/'coaches.json')
|
||
monkeypatch.setattr(center,'OFFICIALS_FILE',tmp_path/'officials.json')
|
||
store=center.DataStore();assert store.counts()['players']==1
|
||
p.write_text('broken json');store.reload()
|
||
assert store.counts()['players']==1 and store.load_warnings
|
||
|
||
|
||
def test_job_unlocks_even_if_reload_fails(monkeypatch):
|
||
def fail():raise RuntimeError('bad file')
|
||
monkeypatch.setattr(center,'STORE',SimpleNamespace(reload=fail))
|
||
monkeypatch.setattr(center.subprocess,'Popen',lambda *a,**k:SimpleNamespace(stdout=[],wait=lambda:0))
|
||
job=center.JobManager();job.running=True;job._run([['python','-u','collector.py','--kind','players']])
|
||
assert job.running is False and job.return_code==1
|
||
|
||
|
||
def test_failed_import_rolls_back_changed_records(career_env):
|
||
_,service,store=career_env
|
||
store.source_versions={'players':(99,99)}
|
||
store.players_data['players'][0]['stages']['1']['data']['stats'][0]['val']=999
|
||
store.players_data['players'][1]['profile']['unserializable']=object()
|
||
with pytest.raises(TypeError):service.import_store(store)
|
||
assert service.game_career('g')['home']['skaters'][0]['season_statistics']['games']==10
|
||
|
||
|
||
def test_roster_entry_without_directory_card_is_shown_as_missing(career_env):
|
||
db,service,_=career_env
|
||
with db.session() as s:
|
||
s.add(GameRoster(game_external_id='g',team_external_id='club',player_external_id='missing',side='home',number='99',role='goalkeeper'))
|
||
result=service.game_career('g')
|
||
assert result['matched']==1 and result['missing']==1
|
||
assert result['home']['goalkeepers'][0]['number']=='99'
|
||
assert result['home']['goalkeepers'][0]['career_status']=='unmatched'
|