import json from pathlib import Path from types import SimpleNamespace import pytest from fastapi.testclient import TestClient from sqlalchemy import select from khl_site import khl_site_collector as collector from khl_site import khl_data_center as center from khl_site.html_source import decode_response, detail_urls, inspect_html, person_url from hockey_data.models import KhlCareerRecord from tests.test_khl_refresh_and_career import config, player, career_env def page(title='Игроки КХЛ', body='Список игроков', tail=''): return ''+title+''+body+''+tail+'' def response(html, status=200): data = html.encode('utf-8') return SimpleNamespace(status_code=status, content=data, text=data.decode('latin1'), encoding='ISO-8859-1') @pytest.mark.parametrize('code', [401,403,429,500,503]) def test_numeric_error_title_is_not_a_normal_document_even_with_http_200(code): html = page(str(code), str(code)) assert collector.HttpClient.valid_html(html) is False assert inspect_html(html)['html_status'] == code def test_utf8_bytes_override_requests_wrong_default_encoding(): html = page('Иванов Иван, хоккеист: статистика', '

Иван Иванов

') text, encoding = decode_response(response(html)) assert 'Иван Иванов' in text and 'Ð' not in text assert encoding.lower() == 'utf-8' assert collector.HttpClient.valid_html(text) from bs4 import BeautifulSoup assert collector.extract_names(BeautifulSoup(text,'html.parser'),'players')[0] == 'Иван Иванов' def test_normal_page_captcha_script_is_not_a_challenge(): html = page(tail='') assert collector.HttpClient.valid_html(html) assert not collector.HttpClient.valid_html(page('Проверка', 'Подтвердите, что вы не робот')) def test_soft_restriction_stops_requests_and_does_not_cache_the_error(tmp_path): client = collector.HttpClient(config(tmp_path, force=True, delay=0)); calls=[] client.session=lambda:SimpleNamespace(get=lambda url,**kw:(calls.append(url) or response(page('403','403')))) with pytest.raises(RuntimeError, match='HTML 403'): client.fetch('https://www.khl.ru/players/') with pytest.raises(RuntimeError):client.fetch('https://www.khl.ru/coaches/') assert len(calls)==1 and not list(client.config.cache_dir.glob('*.html')) assert client.last_problem['http_status']==200 and client.last_problem['html_status']==403 assert 'cookies' not in client.last_problem and 'content' not in client.last_problem def test_old_cached_error_page_is_discarded_not_served_as_a_player_list(tmp_path): client=collector.HttpClient(config(tmp_path, delay=0));url='https://www.khl.ru/players/' client.cache_path(url).write_text(page('403','403'));calls=[] actual=page(body='Иванов Иван') client.session=lambda:SimpleNamespace(get=lambda *a,**k:(calls.append(1) or response(actual))) assert collector.detail_links('players',client.fetch(url))=={10:'https://www.khl.ru/players/10/'} assert calls==[1] def test_links_support_relative_cards_and_embedded_urls_without_foreign_hosts(): html=page(body='''Иванов Иван
Петров Пётр
Сидоров Иван
Чужой Игрок Чужой Игрок ''') assert set(detail_urls('players',html))=={10,11,12,13} hints=collector.player_list_hints(html);assert hints[10]['name_raw']=='Иванов Иван' and hints[11]['name_raw']=='Петров Пётр' assert person_url('players','javascript:alert(1)') is None assert person_url('players','https://x@khl.ru/players/1/') is None def test_known_profiles_refresh_when_normal_list_layout_is_unrecognized(tmp_path,monkeypatch): cfg=config(tmp_path,max_people=1) path=tmp_path/'khl_players_all.json';path.write_text(json.dumps({'meta':{'generated_at':'2020-01-01'},'players':[player(10),player(11)],'stages':[]})) calls=[] def fetch(url):calls.append(url);return page(body='Каталог игроков обновлён') client=SimpleNamespace(fetch=fetch) monkeypatch.setattr(collector,'player_from_profile',lambda pid,*a:(player(pid,games=12),[])) collector.collect_kind('players',client,cfg) data=json.loads(path.read_text());assert data['meta']['discovery_fallback'] is True assert data['meta']['discovery_complete'] is False and data['meta']['updated_count']==1 people={p['identity']['id']:p for p in data['players']} assert people[10]['stages']['1']['data']['stats'][0]['val']==12 assert people[11]['source_updated_at']=='2020-01-01' assert people[10]['source_updated_at']!='2020-01-01' assert calls==['https://www.khl.ru/players/','https://www.khl.ru/players/season/all/','https://www.khl.ru/players/10/'] def test_blocked_directory_never_falls_back_to_other_paths_or_overwrites_archive(tmp_path): cfg=config(tmp_path,force=True,delay=0);path=tmp_path/'khl_players_all.json' path.write_text(json.dumps({'players':[player()]}));before=path.read_bytes() client=collector.HttpClient(cfg);calls=[] client.session=lambda:SimpleNamespace(get=lambda url,**kw:(calls.append(url) or response(page('403','403')))) with pytest.raises(RuntimeError,match='ограничил доступ'):collector.collect_kind('players',client,cfg) assert calls==['https://www.khl.ru/players/'] and path.read_bytes()==before def test_unknown_list_without_an_archive_stays_a_failure(tmp_path): with pytest.raises(RuntimeError,match='Не распознаны ссылки'): collector.collect_kind('players',SimpleNamespace(fetch=lambda url:page()),config(tmp_path)) assert not (tmp_path/'khl_players_all.json').exists() def test_independent_directory_continues_after_parse_error_and_writes_diagnostics(tmp_path,monkeypatch): cfg=config(tmp_path);calls=[] monkeypatch.setattr(collector,'parse_args',lambda:(cfg,['players','coaches','officials'])) def collect(kind,client,config): calls.append(kind) if kind=='players':raise RuntimeError('Список не распознан') client.last_collection_meta={'updated_count':3} monkeypatch.setattr(collector,'collect_kind',collect) assert collector.main()==1 report=json.loads((tmp_path/'khl_update_report.json').read_text()) assert calls==['players','coaches','officials'] and report['result']=='partial' assert [x['status'] for x in report['items']]==['error','ok','ok'] def test_main_does_not_continue_after_site_restriction(tmp_path,monkeypatch): cfg=config(tmp_path);calls=[] monkeypatch.setattr(collector,'parse_args',lambda:(cfg,['players','coaches','officials'])) def collect(kind,client,config): calls.append(kind);client.blocked_error='HTTP 403';raise RuntimeError('HTTP 403') monkeypatch.setattr(collector,'collect_kind',collect) assert collector.main()==1 report=json.loads((tmp_path/'khl_update_report.json').read_text()) assert calls==['players'] and [x['status'] for x in report['items']]==['error','skipped','skipped'] def test_person_dates_survive_import_separately_from_archive_write_time(career_env): db,service,store=career_env store.players_data['meta']={'generated_at':'2026-09-27'} store.players_data['players'][0]['source_updated_at']='2020-01-01' service.import_store(store) with db.session() as session: record=session.get(KhlCareerRecord,('players','10')) assert json.loads(record.payload)['source_updated_at']=='2020-01-01' def test_downloadable_report_uses_json_attachment_and_clear_missing_error(tmp_path,monkeypatch): monkeypatch.setattr(center,'DATA_DIR',tmp_path) with TestClient(center.APP) as client: assert client.get('/api/diagnostics').status_code==404 report={'items':[{'kind':'players','status':'error','message':'КХЛ ограничил доступ'}],'result':'error'} (tmp_path/'khl_update_report.json').write_text(json.dumps(report)) response=client.get('/api/diagnostics') assert response.status_code==200 and response.json()==report assert 'attachment;' in response.headers['content-disposition'] assert response.headers['cache-control']=='no-store'