import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import select
from khl_site import khl_site_collector as collector
from khl_site import khl_data_center as center
from khl_site.html_source import decode_response, detail_urls, inspect_html, person_url
from hockey_data.models import KhlCareerRecord
from tests.test_khl_refresh_and_career import config, player, career_env
def page(title='Игроки КХЛ', body='Список игроков', tail=''):
return '
'+title+''+body+''+tail+''
def response(html, status=200):
data = html.encode('utf-8')
return SimpleNamespace(status_code=status, content=data, text=data.decode('latin1'), encoding='ISO-8859-1')
@pytest.mark.parametrize('code', [401,403,429,500,503])
def test_numeric_error_title_is_not_a_normal_document_even_with_http_200(code):
html = page(str(code), str(code))
assert collector.HttpClient.valid_html(html) is False
assert inspect_html(html)['html_status'] == code
def test_utf8_bytes_override_requests_wrong_default_encoding():
html = page('Иванов Иван, хоккеист: статистика', 'Иван Иванов
')
text, encoding = decode_response(response(html))
assert 'Иван Иванов' in text and 'Ð' not in text
assert encoding.lower() == 'utf-8'
assert collector.HttpClient.valid_html(text)
from bs4 import BeautifulSoup
assert collector.extract_names(BeautifulSoup(text,'html.parser'),'players')[0] == 'Иван Иванов'
def test_normal_page_captcha_script_is_not_a_challenge():
html = page(tail='')
assert collector.HttpClient.valid_html(html)
assert not collector.HttpClient.valid_html(page('Проверка', 'Подтвердите, что вы не робот'))
def test_soft_restriction_stops_requests_and_does_not_cache_the_error(tmp_path):
client = collector.HttpClient(config(tmp_path, force=True, delay=0)); calls=[]
client.session=lambda:SimpleNamespace(get=lambda url,**kw:(calls.append(url) or response(page('403','403'))))
with pytest.raises(RuntimeError, match='HTML 403'):
client.fetch('https://www.khl.ru/players/')
with pytest.raises(RuntimeError):client.fetch('https://www.khl.ru/coaches/')
assert len(calls)==1 and not list(client.config.cache_dir.glob('*.html'))
assert client.last_problem['http_status']==200 and client.last_problem['html_status']==403
assert 'cookies' not in client.last_problem and 'content' not in client.last_problem
def test_old_cached_error_page_is_discarded_not_served_as_a_player_list(tmp_path):
client=collector.HttpClient(config(tmp_path, delay=0));url='https://www.khl.ru/players/'
client.cache_path(url).write_text(page('403','403'));calls=[]
actual=page(body='Иванов Иван')
client.session=lambda:SimpleNamespace(get=lambda *a,**k:(calls.append(1) or response(actual)))
assert collector.detail_links('players',client.fetch(url))=={10:'https://www.khl.ru/players/10/'}
assert calls==[1]
def test_links_support_relative_cards_and_embedded_urls_without_foreign_hosts():
html=page(body='''Иванов Иван
Петров Пётр
Сидоров Иван
Чужой Игрок
Чужой Игрок
''')
assert set(detail_urls('players',html))=={10,11,12,13}
hints=collector.player_list_hints(html);assert hints[10]['name_raw']=='Иванов Иван' and hints[11]['name_raw']=='Петров Пётр'
assert person_url('players','javascript:alert(1)') is None
assert person_url('players','https://x@khl.ru/players/1/') is None
def test_known_profiles_refresh_when_normal_list_layout_is_unrecognized(tmp_path,monkeypatch):
cfg=config(tmp_path,max_people=1)
path=tmp_path/'khl_players_all.json';path.write_text(json.dumps({'meta':{'generated_at':'2020-01-01'},'players':[player(10),player(11)],'stages':[]}))
calls=[]
def fetch(url):calls.append(url);return page(body='Каталог игроков обновлён')
client=SimpleNamespace(fetch=fetch)
monkeypatch.setattr(collector,'player_from_profile',lambda pid,*a:(player(pid,games=12),[]))
collector.collect_kind('players',client,cfg)
data=json.loads(path.read_text());assert data['meta']['discovery_fallback'] is True
assert data['meta']['discovery_complete'] is False and data['meta']['updated_count']==1
people={p['identity']['id']:p for p in data['players']}
assert people[10]['stages']['1']['data']['stats'][0]['val']==12
assert people[11]['source_updated_at']=='2020-01-01'
assert people[10]['source_updated_at']!='2020-01-01'
assert calls==['https://www.khl.ru/players/','https://www.khl.ru/players/season/all/','https://www.khl.ru/players/10/']
def test_blocked_directory_never_falls_back_to_other_paths_or_overwrites_archive(tmp_path):
cfg=config(tmp_path,force=True,delay=0);path=tmp_path/'khl_players_all.json'
path.write_text(json.dumps({'players':[player()]}));before=path.read_bytes()
client=collector.HttpClient(cfg);calls=[]
client.session=lambda:SimpleNamespace(get=lambda url,**kw:(calls.append(url) or response(page('403','403'))))
with pytest.raises(RuntimeError,match='ограничил доступ'):collector.collect_kind('players',client,cfg)
assert calls==['https://www.khl.ru/players/'] and path.read_bytes()==before
def test_unknown_list_without_an_archive_stays_a_failure(tmp_path):
with pytest.raises(RuntimeError,match='Не распознаны ссылки'):
collector.collect_kind('players',SimpleNamespace(fetch=lambda url:page()),config(tmp_path))
assert not (tmp_path/'khl_players_all.json').exists()
def test_independent_directory_continues_after_parse_error_and_writes_diagnostics(tmp_path,monkeypatch):
cfg=config(tmp_path);calls=[]
monkeypatch.setattr(collector,'parse_args',lambda:(cfg,['players','coaches','officials']))
def collect(kind,client,config):
calls.append(kind)
if kind=='players':raise RuntimeError('Список не распознан')
client.last_collection_meta={'updated_count':3}
monkeypatch.setattr(collector,'collect_kind',collect)
assert collector.main()==1
report=json.loads((tmp_path/'khl_update_report.json').read_text())
assert calls==['players','coaches','officials'] and report['result']=='partial'
assert [x['status'] for x in report['items']]==['error','ok','ok']
def test_main_does_not_continue_after_site_restriction(tmp_path,monkeypatch):
cfg=config(tmp_path);calls=[]
monkeypatch.setattr(collector,'parse_args',lambda:(cfg,['players','coaches','officials']))
def collect(kind,client,config):
calls.append(kind);client.blocked_error='HTTP 403';raise RuntimeError('HTTP 403')
monkeypatch.setattr(collector,'collect_kind',collect)
assert collector.main()==1
report=json.loads((tmp_path/'khl_update_report.json').read_text())
assert calls==['players'] and [x['status'] for x in report['items']]==['error','skipped','skipped']
def test_person_dates_survive_import_separately_from_archive_write_time(career_env):
db,service,store=career_env
store.players_data['meta']={'generated_at':'2026-09-27'}
store.players_data['players'][0]['source_updated_at']='2020-01-01'
service.import_store(store)
with db.session() as session:
record=session.get(KhlCareerRecord,('players','10'))
assert json.loads(record.payload)['source_updated_at']=='2020-01-01'
def test_downloadable_report_uses_json_attachment_and_clear_missing_error(tmp_path,monkeypatch):
monkeypatch.setattr(center,'DATA_DIR',tmp_path)
with TestClient(center.APP) as client:
assert client.get('/api/diagnostics').status_code==404
report={'items':[{'kind':'players','status':'error','message':'КХЛ ограничил доступ'}],'result':'error'}
(tmp_path/'khl_update_report.json').write_text(json.dumps(report))
response=client.get('/api/diagnostics')
assert response.status_code==200 and response.json()==report
assert 'attachment;' in response.headers['content-disposition']
assert response.headers['cache-control']=='no-store'