425 lines
19 KiB
Python
425 lines
19 KiB
Python
"""Offline import admission contracts, with real service/HTTP/SQLite paths.
|
|
|
|
Only Stat2TV responses and the physical Agent socket are substituted. Deliberate
|
|
thread barriers stop an import before SQL, so SQLite writer locks cannot make a
|
|
passing control request look like admission protection.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import asyncio
|
|
import json
|
|
import threading
|
|
import time
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
from fastapi import HTTPException, Query
|
|
from sqlalchemy import event, select
|
|
|
|
from hockey_data import async_work
|
|
from hockey_data.agent_bridge import LiveAgent
|
|
from hockey_data.client import Stat2TVResponse
|
|
from hockey_data.models import MatchControlLease, OperatorSession, UserPreference, VmixAssignment, VmixDevice
|
|
from hockey_data.service import HockeyDataService
|
|
from tests.test_stage1_match_access import env, timer
|
|
|
|
|
|
def _workers(monkeypatch, count):
|
|
monkeypatch.setenv("HOCKEY_DB_WORKERS", str(count))
|
|
monkeypatch.setenv("HOCKEY_DB_POOL_SIZE", "10")
|
|
|
|
|
|
async def _settle():
|
|
# Give already-created coroutines time to reach their first admission wait.
|
|
for _ in range(8):
|
|
await asyncio.sleep(0)
|
|
|
|
|
|
def _offline_match_responses(service, monkeypatch):
|
|
settings = service.settings.public()
|
|
names = {name: service._resolve_match_resource_endpoint(settings[key],
|
|
tournament_external_id="t", game_external_id="g")
|
|
for name, key in {
|
|
"match_ru": "match_json_endpoint_template", "match_en": "match_json_en_endpoint_template",
|
|
"players": "players_endpoint_template", "countries": "player_countries_endpoint_template",
|
|
"referees": "referees_endpoint_template"}.items()}
|
|
match = json.dumps({"game": {"idschedule": "g", "teamA": "Home", "teamB": "Away",
|
|
"idteamA": "home", "idteamB": "away", "status": "scheduled"},
|
|
"teams": {"A": {}, "B": {}}, "players": {"A": {}, "B": {}}}).encode()
|
|
contents = {names["match_ru"]: match, names["match_en"]: match,
|
|
names["players"]: b"<PlayerStatsList/>", names["countries"]: b"<Countries/>",
|
|
names["referees"]: b"<Referees/>"}
|
|
|
|
async def fetch(endpoint, **kwargs):
|
|
return Stat2TVResponse(content=contents[endpoint], status_code=200,
|
|
content_type="application/json" if endpoint in (names["match_ru"], names["match_en"])
|
|
else "application/xml", auth_mode="offline-test", url=endpoint)
|
|
|
|
monkeypatch.setattr(service, "_fetch_cached_resource", fetch)
|
|
|
|
|
|
def _agent_target(env):
|
|
assert env.claim().status_code == 200
|
|
channel = env.service.open_session(wfl_user_id="a", tournament_external_id="t", game_external_id="g")
|
|
device_id = "GFX-ADMISSION-01"
|
|
with env.db.session() as session:
|
|
session.scalar(select(OperatorSession).where(OperatorSession.session_token == channel["token"])).vmix_device_uuid = device_id
|
|
device = VmixDevice(device_uuid=device_id, device_secret_hash="offline-test", wfl_user_id="a",
|
|
is_active_for_account=True, agent_version="1.5.3", vmix_connected=True,
|
|
current_match_external_id="g", current_assignment_key="admission-assignment")
|
|
session.add(device)
|
|
session.flush()
|
|
session.add(VmixAssignment(device_id=device.id, assignment_key="admission-assignment",
|
|
wfl_user_id="a", operator_session_token=channel["token"], game_external_id="g", active=True))
|
|
session.get(MatchControlLease, "g").owner_device_id = device_id
|
|
|
|
class Socket:
|
|
def __init__(self):
|
|
self.sent = []
|
|
|
|
async def send_json(self, payload):
|
|
self.sent.append(payload)
|
|
|
|
socket = Socket()
|
|
env.hub._live[device_id] = LiveAgent(websocket=socket, connected_at=datetime.utcnow())
|
|
return channel, device_id, socket, env.headers()
|
|
|
|
|
|
@pytest.mark.parametrize("workers", [2, 4, 8])
|
|
def test_real_match_import_backlog_preserves_timer_write_and_agent_target_resolution(env, monkeypatch, workers):
|
|
_workers(monkeypatch, workers)
|
|
_offline_match_responses(env.service, monkeypatch)
|
|
channel, device_id, socket, headers = _agent_target(env)
|
|
original_prepare = env.service._prepare_match_directories
|
|
release = threading.Event()
|
|
counter_lock = threading.Lock()
|
|
entered_count = 0
|
|
|
|
async def scenario():
|
|
entered = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def prepare(*args, **kwargs):
|
|
nonlocal entered_count
|
|
with counter_lock:
|
|
entered_count += 1
|
|
# A real RLock blocks competing prepare calls in BUILD191. Admission
|
|
# in BUILD192 must keep those contenders outside the DB worker pool.
|
|
with env.service._import_lock:
|
|
loop.call_soon_threadsafe(entered.set)
|
|
assert release.wait(10), "test import barrier timed out"
|
|
return original_prepare(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(env.service, "_prepare_match_directories", prepare)
|
|
imports = [asyncio.create_task(env.service.sync_match_details(
|
|
tournament_external_id="t", game_external_id="g", skip_optional_shots=True))
|
|
for _ in range(max(9, workers + 2))]
|
|
requests = None
|
|
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=env.app), base_url="http://test") as client:
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 2)
|
|
await _settle()
|
|
requests = asyncio.gather(
|
|
client.put("/api/hockey/games/g/control/timers", headers=headers, json=timer(345000, False)),
|
|
client.post("/api/hockey/vmix/sequence", headers=headers,
|
|
json={"session_token": channel["token"], "device_id": device_id,
|
|
"delivery_mode": "timer-fast-ordered", "commands": [
|
|
{"Function": "SuspendCountdown", "Input": "Score", "SelectedName": "Clock.Text"}]}))
|
|
# Shield protects cleanup: cancelling a running DB job correctly
|
|
# waits for its thread, so wait_for cancellation itself could wait
|
|
# until the held import is released.
|
|
started = time.monotonic()
|
|
try:
|
|
responses = await asyncio.wait_for(asyncio.shield(requests), 5)
|
|
finally:
|
|
print("QA import interactive " + json.dumps({"workers": workers,
|
|
"elapsed_seconds": round(time.monotonic() - started, 4),
|
|
"barrier_released": release.is_set(), "prepare_entered": entered_count,
|
|
"requests_done": requests.done()}))
|
|
assert all(response.status_code == 200 for response in responses), [r.text for r in responses]
|
|
assert not release.is_set()
|
|
assert entered_count == 1, "bulk RLock waiters must not occupy DB workers"
|
|
assert env.service.game_control("g")["timers"]["main_timer"]["current_ms"] == 345000
|
|
commands = [frame["command"] for frame in socket.sent if frame.get("type") == "vmix.command"]
|
|
assert commands == [{"Function": "SuspendCountdown", "Input": "Score", "Value": "", "SelectedName": "Clock.Text"}]
|
|
finally:
|
|
release.set()
|
|
outcomes = await asyncio.gather(*imports, return_exceptions=True)
|
|
if requests is not None:
|
|
await asyncio.gather(requests, return_exceptions=True)
|
|
assert not [item for item in outcomes if isinstance(item, BaseException)], outcomes
|
|
assert all(item["game"]["external_id"] == "g" for item in outcomes)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_waiting_cancellation_never_dispatches_and_does_not_leak_admission(monkeypatch):
|
|
_workers(monkeypatch, 2)
|
|
release = threading.Event()
|
|
calls = []
|
|
|
|
async def scenario():
|
|
entered = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def holder():
|
|
calls.append("holder")
|
|
loop.call_soon_threadsafe(entered.set)
|
|
assert release.wait(3)
|
|
|
|
first = asyncio.create_task(async_work.run_import_db(holder))
|
|
waiting = None
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 1)
|
|
waiting = asyncio.create_task(async_work.run_import_db(lambda: calls.append("cancelled-waiter")))
|
|
await _settle()
|
|
waiting.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await asyncio.wait_for(asyncio.shield(waiting), .5)
|
|
assert calls == ["holder"] and not release.is_set()
|
|
assert await asyncio.wait_for(async_work.run_db(lambda: "interactive"), .5) == "interactive"
|
|
finally:
|
|
release.set()
|
|
await asyncio.gather(first, *([waiting] if waiting else []), return_exceptions=True)
|
|
assert await async_work.run_import_db(lambda: "next") == "next"
|
|
assert calls == ["holder"]
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_running_cancellation_holds_admission_until_atomic_commit(env, monkeypatch):
|
|
_workers(monkeypatch, 2)
|
|
release = threading.Event()
|
|
second_started = threading.Event()
|
|
|
|
async def scenario():
|
|
entered = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def transaction():
|
|
with env.db.atomic_session() as session:
|
|
row = UserPreference(wfl_user_id="cancelled-import", runtime_ui_json='{"stage":"before"}')
|
|
session.add(row)
|
|
loop.call_soon_threadsafe(entered.set)
|
|
assert release.wait(3)
|
|
row.runtime_ui_json = '{"stage":"committed"}'
|
|
|
|
first = asyncio.create_task(async_work.run_import_db(transaction))
|
|
second = None
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 1)
|
|
first.cancel()
|
|
await _settle()
|
|
first.cancel() # repeated cancellation must not detach the thread
|
|
second = asyncio.create_task(async_work.run_import_db(second_started.set))
|
|
await _settle()
|
|
assert not first.done() and not second_started.is_set()
|
|
assert await asyncio.wait_for(async_work.run_db(lambda: "control"), .5) == "control"
|
|
assert not first.done() and not second_started.is_set() and not release.is_set(), (
|
|
"cancellation must retain admission until the running transaction finishes")
|
|
finally:
|
|
release.set()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await first
|
|
if second is not None:
|
|
await second
|
|
assert second_started.is_set()
|
|
with env.db.session() as session:
|
|
assert session.get(UserPreference, "cancelled-import").runtime_ui_json == '{"stage":"committed"}'
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_exception_rolls_back_and_atomic_context_is_rejected_before_admission(env, monkeypatch):
|
|
_workers(monkeypatch, 2)
|
|
release = threading.Event()
|
|
|
|
async def scenario():
|
|
def failing():
|
|
with env.db.atomic_session() as session:
|
|
session.add(UserPreference(wfl_user_id="failed-import"))
|
|
session.flush()
|
|
raise ValueError("offline rollback sentinel")
|
|
|
|
with pytest.raises(ValueError, match="offline rollback sentinel"):
|
|
await async_work.run_import_db(failing)
|
|
with env.db.session() as session:
|
|
assert session.get(UserPreference, "failed-import") is None
|
|
assert await async_work.run_import_db(lambda: "recovered") == "recovered"
|
|
entered = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def hold():
|
|
loop.call_soon_threadsafe(entered.set)
|
|
assert release.wait(3)
|
|
|
|
holder = asyncio.create_task(async_work.run_import_db(hold))
|
|
rejected = None
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 1)
|
|
with env.db.atomic_session():
|
|
rejected = asyncio.create_task(async_work.run_import_db(lambda: pytest.fail("atomic job dispatched")))
|
|
with pytest.raises(RuntimeError, match="cannot cross"):
|
|
await asyncio.wait_for(asyncio.shield(rejected), .5)
|
|
assert not release.is_set(), "atomic misuse must fail before waiting for admission"
|
|
finally:
|
|
release.set()
|
|
await asyncio.gather(holder, *([rejected] if rejected else []), return_exceptions=True)
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_global_service_admission_is_recreated_for_each_event_loop(env, monkeypatch, tmp_path):
|
|
_workers(monkeypatch, 2)
|
|
other = HockeyDataService(base_dir=tmp_path / "other-service", database=env.db, settings=env.service.settings)
|
|
|
|
async def scenario():
|
|
release = threading.Event()
|
|
entered = asyncio.Event()
|
|
second_entered = threading.Event()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def first():
|
|
with env.service._import_lock:
|
|
loop.call_soon_threadsafe(entered.set)
|
|
assert release.wait(3)
|
|
|
|
def second():
|
|
with other._import_lock:
|
|
second_entered.set()
|
|
return other._prepare_match_directories("t", {key: key for key in ("players", "countries", "referees")},
|
|
{key: None for key in ("players", "countries", "referees")})
|
|
|
|
first_task = asyncio.create_task(async_work.run_import_db(first))
|
|
second_task = None
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 1)
|
|
second_task = asyncio.create_task(async_work.run_import_db(second))
|
|
await _settle()
|
|
assert await asyncio.wait_for(async_work.run_db(lambda: 42), .5) == 42
|
|
assert not second_entered.is_set(), "independent services share one admission budget"
|
|
finally:
|
|
release.set()
|
|
await asyncio.gather(first_task, *([second_task] if second_task else []))
|
|
assert second_entered.is_set()
|
|
|
|
# Contention in both runs binds each limiter to its loop. A process-global
|
|
# asyncio semaphore or leaked token fails on the second loop.
|
|
asyncio.run(scenario())
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_single_worker_stays_bounded_when_no_headroom_is_available(monkeypatch):
|
|
_workers(monkeypatch, 1)
|
|
release = threading.Event()
|
|
interactive_entered = threading.Event()
|
|
|
|
async def scenario():
|
|
entered = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def hold():
|
|
loop.call_soon_threadsafe(entered.set)
|
|
assert release.wait(3)
|
|
|
|
holder = asyncio.create_task(async_work.run_import_db(hold))
|
|
interactive = None
|
|
try:
|
|
await asyncio.wait_for(entered.wait(), 1)
|
|
interactive = asyncio.create_task(async_work.run_db(interactive_entered.set))
|
|
await _settle()
|
|
assert not interactive_entered.is_set(), "admission must not bypass the configured one-worker bound"
|
|
finally:
|
|
release.set()
|
|
await asyncio.gather(holder, *([interactive] if interactive else []))
|
|
assert interactive_entered.is_set()
|
|
|
|
asyncio.run(scenario())
|
|
|
|
|
|
def test_actual_cold_schedule_request_keeps_event_loop_responsive(env, monkeypatch):
|
|
_workers(monkeypatch, 2)
|
|
source = Path(__file__).resolve().parents[1] / "app.py"
|
|
module = ast.parse(source.read_text(encoding="utf-8"))
|
|
handler = next(node for node in module.body if isinstance(node, ast.AsyncFunctionDef)
|
|
and node.name == "tournament_live_schedule")
|
|
handler.decorator_list = []
|
|
namespace = {"hockey_service": env.service, "run_db": async_work.run_db,
|
|
"date": date, "Query": Query, "Any": Any, "HTTPException": HTTPException}
|
|
exec(compile(ast.fix_missing_locations(ast.Module(body=[handler], type_ignores=[])), str(source), "exec"), namespace)
|
|
env.app.get("/qa-cold-schedule/{external_id}")(namespace[handler.name])
|
|
|
|
@env.app.get("/qa-admission-health")
|
|
async def health():
|
|
return {"ok": True}
|
|
|
|
# Production DDL targets PostgreSQL. Translate only its two dialect tokens,
|
|
# retaining the actual table creation, index creation and subsequent query.
|
|
def sqlite_ddl(conn, cursor, statement, parameters, context, executemany):
|
|
if "CREATE TABLE IF NOT EXISTS schedule_live" in statement:
|
|
statement = statement.replace("BIGSERIAL PRIMARY KEY", "INTEGER PRIMARY KEY").replace("DEFAULT NOW()", "DEFAULT CURRENT_TIMESTAMP")
|
|
return statement, parameters
|
|
|
|
event.listen(env.db.engine, "before_cursor_execute", sqlite_ddl, retval=True)
|
|
release = threading.Event()
|
|
held = threading.Event()
|
|
|
|
def hold_import_lock():
|
|
with env.service._import_lock:
|
|
held.set()
|
|
release.wait(10)
|
|
|
|
owner = threading.Thread(target=hold_import_lock)
|
|
owner.start()
|
|
if not held.wait(1):
|
|
release.set()
|
|
owner.join(2)
|
|
event.remove(env.db.engine, "before_cursor_execute", sqlite_ddl)
|
|
pytest.fail("test import lock holder did not start")
|
|
# A watchdog releases the raw RLock even if a regression blocks the event
|
|
# loop itself; the test then fails rather than hanging its pytest process.
|
|
watchdog = threading.Timer(5, release.set)
|
|
watchdog.start()
|
|
original_ensure = env.service.ensure_schedule_live_table
|
|
|
|
async def scenario():
|
|
entering = asyncio.Event()
|
|
loop = asyncio.get_running_loop()
|
|
|
|
def ensure():
|
|
loop.call_soon_threadsafe(entering.set)
|
|
return original_ensure()
|
|
|
|
monkeypatch.setattr(env.service, "ensure_schedule_live_table", ensure)
|
|
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=env.app), base_url="http://test") as client:
|
|
request = asyncio.create_task(client.get("/qa-cold-schedule/t?on_date=2026-10-21"))
|
|
try:
|
|
started = time.monotonic()
|
|
try:
|
|
await asyncio.wait_for(entering.wait(), 3)
|
|
finally:
|
|
print("QA cold ensure " + json.dumps({"elapsed_seconds": round(time.monotonic() - started, 4),
|
|
"barrier_released": release.is_set(), "request_done": request.done()}))
|
|
assert not request.done() and not release.is_set(), "cold schedule initialization blocked the event loop"
|
|
response = await asyncio.wait_for(client.get("/qa-admission-health"), 1.5)
|
|
assert response.json() == {"ok": True}
|
|
assert not request.done() and not env.service._schedule_live_table_ready
|
|
finally:
|
|
release.set()
|
|
result = await request
|
|
assert result.status_code == 200, result.text
|
|
assert env.service._schedule_live_table_ready
|
|
|
|
try:
|
|
asyncio.run(scenario())
|
|
finally:
|
|
release.set()
|
|
watchdog.cancel()
|
|
owner.join(2)
|
|
event.remove(env.db.engine, "before_cursor_execute", sqlite_ddl)
|
|
assert not owner.is_alive(), "test import holder leaked a thread"
|