hockey_new/tests/test_khl_html_recovery.py

import json
from pathlib import Path
from types import SimpleNamespace

import pytest
from fastapi.testclient import TestClient
from sqlalchemy import select

from khl_site import khl_site_collector as collector
from khl_site import khl_data_center as center
from khl_site.html_source import decode_response, detail_urls, inspect_html, person_url
from hockey_data.models import KhlCareerRecord
from tests.test_khl_refresh_and_career import config, player, career_env


def page(title='Игроки КХЛ', body='Список игроков', tail=''):
    return '<!doctype html><html><head><meta charset="utf-8"><title>'+title+'</title></head><body>'+body+'<!--'+' '*800+'-->'+tail+'</body></html>'


def response(html, status=200):
    data = html.encode('utf-8')
    return SimpleNamespace(status_code=status, content=data, text=data.decode('latin1'), encoding='ISO-8859-1')


@pytest.mark.parametrize('code', [401,403,429,500,503])
def test_numeric_error_title_is_not_a_normal_document_even_with_http_200(code):
    html = page(str(code), str(code))
    assert collector.HttpClient.valid_html(html) is False
    assert inspect_html(html)['html_status'] == code


def test_utf8_bytes_override_requests_wrong_default_encoding():
    html = page('Иванов Иван, хоккеист: статистика', '<h1>Иван Иванов</h1>')
    text, encoding = decode_response(response(html))
    assert 'Иван Иванов' in text and 'Ð' not in text
    assert encoding.lower() == 'utf-8'
    assert collector.HttpClient.valid_html(text)
    from bs4 import BeautifulSoup
    assert collector.extract_names(BeautifulSoup(text,'html.parser'),'players')[0] == 'Иван Иванов'


def test_normal_page_captcha_script_is_not_a_challenge():
    html = page(tail='<script src="captcha.js"></script>')
    assert collector.HttpClient.valid_html(html)
    assert not collector.HttpClient.valid_html(page('Проверка', 'Подтвердите, что вы не робот'))


def test_soft_restriction_stops_requests_and_does_not_cache_the_error(tmp_path):
    client = collector.HttpClient(config(tmp_path, force=True, delay=0)); calls=[]
    client.session=lambda:SimpleNamespace(get=lambda url,**kw:(calls.append(url) or response(page('403','403'))))
    with pytest.raises(RuntimeError, match='HTML 403'):
        client.fetch('https://www.khl.ru/players/')
    with pytest.raises(RuntimeError):client.fetch('https://www.khl.ru/coaches/')
    assert len(calls)==1 and not list(client.config.cache_dir.glob('*.html'))
    assert client.last_problem['http_status']==200 and client.last_problem['html_status']==403
    assert 'cookies' not in client.last_problem and 'content' not in client.last_problem


def test_old_cached_error_page_is_discarded_not_served_as_a_player_list(tmp_path):
    client=collector.HttpClient(config(tmp_path, delay=0));url='https://www.khl.ru/players/'
    client.cache_path(url).write_text(page('403','403'));calls=[]
    actual=page(body='<a href="/players/10/">Иванов Иван</a>')
    client.session=lambda:SimpleNamespace(get=lambda *a,**k:(calls.append(1) or response(actual)))
    assert collector.detail_links('players',client.fetch(url))=={10:'https://www.khl.ru/players/10/'}
    assert calls==[1]


def test_links_support_relative_cards_and_embedded_urls_without_foreign_hosts():
    html=page(body='''<a href="10/?view=all#stats">Иванов Иван</a>
      <div data-href="/players/11/">Петров Пётр</div><div data-url="https://khl.ru/players/12/">Сидоров Иван</div>
      <a href="https://foreign.example/players/999/">Чужой Игрок</a>
      <a href="https://khl.ru.evil.example/players/998/">Чужой Игрок</a>
      <script type="application/ld+json">{"itemListElement":[{"url":"https://www.khl.ru/players/13/"},{"id":997}]}</script>''')
    assert set(detail_urls('players',html))=={10,11,12,13}
    hints=collector.player_list_hints(html);assert hints[10]['name_raw']=='Иванов Иван' and hints[11]['name_raw']=='Петров Пётр'
    assert person_url('players','javascript:alert(1)') is None
    assert person_url('players','https://x@khl.ru/players/1/') is None


def test_known_profiles_refresh_when_normal_list_layout_is_unrecognized(tmp_path,monkeypatch):
    cfg=config(tmp_path,max_people=1)
    path=tmp_path/'khl_players_all.json';path.write_text(json.dumps({'meta':{'generated_at':'2020-01-01'},'players':[player(10),player(11)],'stages':[]}))
    calls=[]
    def fetch(url):calls.append(url);return page(body='Каталог игроков обновлён')
    client=SimpleNamespace(fetch=fetch)
    monkeypatch.setattr(collector,'player_from_profile',lambda pid,*a:(player(pid,games=12),[]))
    collector.collect_kind('players',client,cfg)
    data=json.loads(path.read_text());assert data['meta']['discovery_fallback'] is True
    assert data['meta']['discovery_complete'] is False and data['meta']['updated_count']==1
    people={p['identity']['id']:p for p in data['players']}
    assert people[10]['stages']['1']['data']['stats'][0]['val']==12
    assert people[11]['source_updated_at']=='2020-01-01'
    assert people[10]['source_updated_at']!='2020-01-01'
    assert calls==['https://www.khl.ru/players/','https://www.khl.ru/players/season/all/','https://www.khl.ru/players/10/']


def test_blocked_directory_never_falls_back_to_other_paths_or_overwrites_archive(tmp_path):
    cfg=config(tmp_path,force=True,delay=0);path=tmp_path/'khl_players_all.json'
    path.write_text(json.dumps({'players':[player()]}));before=path.read_bytes()
    client=collector.HttpClient(cfg);calls=[]
    client.session=lambda:SimpleNamespace(get=lambda url,**kw:(calls.append(url) or response(page('403','403'))))
    with pytest.raises(RuntimeError,match='ограничил доступ'):collector.collect_kind('players',client,cfg)
    assert calls==['https://www.khl.ru/players/'] and path.read_bytes()==before


def test_unknown_list_without_an_archive_stays_a_failure(tmp_path):
    with pytest.raises(RuntimeError,match='Не распознаны ссылки'):
        collector.collect_kind('players',SimpleNamespace(fetch=lambda url:page()),config(tmp_path))
    assert not (tmp_path/'khl_players_all.json').exists()


def test_independent_directory_continues_after_parse_error_and_writes_diagnostics(tmp_path,monkeypatch):
    cfg=config(tmp_path);calls=[]
    monkeypatch.setattr(collector,'parse_args',lambda:(cfg,['players','coaches','officials']))
    def collect(kind,client,config):
        calls.append(kind)
        if kind=='players':raise RuntimeError('Список не распознан')
        client.last_collection_meta={'updated_count':3}
    monkeypatch.setattr(collector,'collect_kind',collect)
    assert collector.main()==1
    report=json.loads((tmp_path/'khl_update_report.json').read_text())
    assert calls==['players','coaches','officials'] and report['result']=='partial'
    assert [x['status'] for x in report['items']]==['error','ok','ok']


def test_main_does_not_continue_after_site_restriction(tmp_path,monkeypatch):
    cfg=config(tmp_path);calls=[]
    monkeypatch.setattr(collector,'parse_args',lambda:(cfg,['players','coaches','officials']))
    def collect(kind,client,config):
        calls.append(kind);client.blocked_error='HTTP 403';raise RuntimeError('HTTP 403')
    monkeypatch.setattr(collector,'collect_kind',collect)
    assert collector.main()==1
    report=json.loads((tmp_path/'khl_update_report.json').read_text())
    assert calls==['players'] and [x['status'] for x in report['items']]==['error','skipped','skipped']


def test_person_dates_survive_import_separately_from_archive_write_time(career_env):
    db,service,store=career_env
    store.players_data['meta']={'generated_at':'2026-09-27'}
    store.players_data['players'][0]['source_updated_at']='2020-01-01'
    service.import_store(store)
    with db.session() as session:
        record=session.get(KhlCareerRecord,('players','10'))
        assert json.loads(record.payload)['source_updated_at']=='2020-01-01'


def test_downloadable_report_uses_json_attachment_and_clear_missing_error(tmp_path,monkeypatch):
    monkeypatch.setattr(center,'DATA_DIR',tmp_path)
    with TestClient(center.APP) as client:
        assert client.get('/api/diagnostics').status_code==404
        report={'items':[{'kind':'players','status':'error','message':'КХЛ ограничил доступ'}],'result':'error'}
        (tmp_path/'khl_update_report.json').write_text(json.dumps(report))
        response=client.get('/api/diagnostics')
        assert response.status_code==200 and response.json()==report
        assert 'attachment;' in response.headers['content-disposition']
        assert response.headers['cache-control']=='no-store'