211 lines
8.5 KiB
Python
211 lines
8.5 KiB
Python
"""Small persistent helpers for the KHL collector and its daily scheduler."""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import tempfile
|
||
import threading
|
||
from datetime import datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
|
||
try:
|
||
from zoneinfo import ZoneInfo
|
||
MOSCOW = ZoneInfo("Europe/Moscow")
|
||
except (ImportError, KeyError): # Windows without the optional tzdata package.
|
||
MOSCOW = timezone(timedelta(hours=3), "Europe/Moscow")
|
||
|
||
KINDS = ("players", "coaches", "officials")
|
||
DEFAULT_CONFIG = {"enabled": True, "time": "03:00", "workers": 1,
|
||
"request_delay_seconds": 0.5, "window_minutes": 60}
|
||
|
||
|
||
def timestamp():
|
||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||
|
||
|
||
def read_json(path, default):
|
||
try:
|
||
return json.loads(Path(path).read_text(encoding="utf-8"))
|
||
except FileNotFoundError:
|
||
return default
|
||
|
||
|
||
def atomic_json(path, value):
|
||
path = Path(path)
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
name = None
|
||
try:
|
||
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent,
|
||
prefix=path.name + ".", suffix=".tmp", delete=False) as fh:
|
||
name = fh.name
|
||
json.dump(value, fh, ensure_ascii=False, indent=2)
|
||
fh.write("\n")
|
||
os.replace(name, path)
|
||
finally:
|
||
if name:
|
||
Path(name).unlink(missing_ok=True)
|
||
|
||
|
||
class FileLock:
|
||
"""OS lock: released on exit, also prevents duplicate server workers/jobs."""
|
||
def __init__(self, path):
|
||
self.path, self.handle = Path(path), None
|
||
|
||
def acquire(self):
|
||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||
handle = self.path.open("a+b")
|
||
try:
|
||
if os.name == "nt":
|
||
import msvcrt
|
||
if handle.tell() == 0:
|
||
handle.write(b"0"); handle.flush()
|
||
handle.seek(0)
|
||
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
||
else:
|
||
import fcntl
|
||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
except (BlockingIOError, OSError):
|
||
handle.close()
|
||
return False
|
||
self.handle = handle
|
||
return True
|
||
|
||
def close(self):
|
||
if self.handle is not None:
|
||
if os.name == "nt":
|
||
import msvcrt
|
||
self.handle.seek(0)
|
||
msvcrt.locking(self.handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||
self.handle.close()
|
||
self.handle = None
|
||
|
||
|
||
def load_config(path, *, create=False):
|
||
path = Path(path)
|
||
if create and not path.exists():
|
||
# Exclusive creation must never replace a studio's saved configuration.
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
try:
|
||
with path.open("x", encoding="utf-8") as fh:
|
||
json.dump(DEFAULT_CONFIG, fh, ensure_ascii=False, indent=2)
|
||
fh.write("\n")
|
||
except FileExistsError:
|
||
pass
|
||
raw = read_json(path, {})
|
||
value = {**DEFAULT_CONFIG, **raw}
|
||
hour, minute = map(int, str(value["time"]).split(":"))
|
||
if not (0 <= hour <= 23 and 0 <= minute <= 59):
|
||
raise ValueError("Время обновления КХЛ должно быть в формате ЧЧ:ММ")
|
||
value["time"] = f"{hour:02}:{minute:02}"
|
||
value["workers"] = max(1, min(4, int(value["workers"])))
|
||
value["request_delay_seconds"] = max(0.1, min(10.0, float(value["request_delay_seconds"])))
|
||
value["window_minutes"] = max(1, min(60, int(value["window_minutes"])))
|
||
value["enabled"] = bool(value["enabled"])
|
||
return value
|
||
|
||
|
||
class RetryQueue:
|
||
"""Only remove pending IDs after their merged archive has been committed."""
|
||
def __init__(self, data_dir):
|
||
self.data_dir = Path(data_dir)
|
||
self.path = self.data_dir / "khl_update_pending.json"
|
||
self.data = read_json(self.path, {"version": 1, "kinds": {}})
|
||
if not isinstance(self.data, dict) or not isinstance(self.data.get("kinds"), dict):
|
||
raise ValueError("Повреждён список незагруженных данных КХЛ")
|
||
# Recover failures from earlier builds without rewriting the old archive.
|
||
if not self.path.exists():
|
||
for kind in KINDS:
|
||
old = read_json(self.data_dir / f"khl_{kind}_all.json", {})
|
||
part = self.part(kind)
|
||
for item in old.get("errors", []):
|
||
pid = str(item.get("id", ""))
|
||
if pid.isdigit():
|
||
part["profiles"][pid] = {"id": pid, "error": str(item.get("error", "Ошибка загрузки"))}
|
||
for item in old.get("players" if kind == "players" else "people", []):
|
||
identity, profile = item.get("identity") or {}, item.get("profile") or {}
|
||
pid = str(identity.get("id") or profile.get("id") or item.get("id") or "")
|
||
warning = profile.get("refresh_warning") or item.get("refresh_warning")
|
||
if pid.isdigit() and warning:
|
||
part["profiles"][pid] = {"id": pid, "error": str(warning)}
|
||
elif pid.isdigit():
|
||
part["completed_ids"].append(pid)
|
||
if old.get("discovery_errors"):
|
||
part["discovery"] = "Не удалось полностью прочитать список КХЛ"
|
||
|
||
def part(self, kind):
|
||
return self.data["kinds"].setdefault(kind, {"profiles": {}, "completed_ids": [], "discovery": "", "current_only": False})
|
||
|
||
def save(self):
|
||
atomic_json(self.path, self.data)
|
||
|
||
def pending_kinds(self):
|
||
return [kind for kind in KINDS if self.part(kind)["profiles"] or self.part(kind)["discovery"]]
|
||
|
||
def summary(self):
|
||
profiles, directories, items = 0, 0, []
|
||
for kind in KINDS:
|
||
part = self.part(kind)
|
||
profiles += len(part["profiles"])
|
||
if part["discovery"]:
|
||
directories += 1
|
||
items.append({"kind": kind, "id": "", "error": part["discovery"]})
|
||
for pid, value in part["profiles"].items():
|
||
if len(items) < 50:
|
||
items.append({"kind": kind, "id": pid, "error": value.get("error", "Не завершено")})
|
||
return {"profiles": profiles, "directories": directories, "items": items,
|
||
"kinds": self.pending_kinds()}
|
||
|
||
|
||
class DailyScheduler:
|
||
def __init__(self, config_path, jobs, launch):
|
||
self.config_path, self.jobs, self.launch = Path(config_path), jobs, launch
|
||
self.stop_event = threading.Event()
|
||
self.thread = None
|
||
self.error = ""
|
||
|
||
def status(self, now=None):
|
||
now = (now or datetime.now(MOSCOW)).astimezone(MOSCOW)
|
||
try:
|
||
config = load_config(self.config_path)
|
||
hour, minute = map(int, config["time"].split(":"))
|
||
due = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||
last = self.jobs.state().get("last_scheduled_date", "")
|
||
if now >= due + timedelta(minutes=config["window_minutes"]) or last == now.date().isoformat():
|
||
due += timedelta(days=1)
|
||
return {**config, "timezone": "Europe/Moscow", "next_run_at": due.isoformat() if config["enabled"] else "", "error": self.error}
|
||
except Exception as exc:
|
||
return {"enabled": False, "timezone": "Europe/Moscow", "error": str(exc)}
|
||
|
||
def tick(self, now=None):
|
||
now = (now or datetime.now(MOSCOW)).astimezone(MOSCOW)
|
||
config = load_config(self.config_path)
|
||
hour, minute = map(int, config["time"].split(":"))
|
||
due = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||
if config["enabled"] and due <= now < due + timedelta(minutes=config["window_minutes"]):
|
||
self.launch(now.date().isoformat())
|
||
|
||
def start(self):
|
||
if self.thread and self.thread.is_alive():
|
||
return
|
||
try:
|
||
load_config(self.config_path, create=True)
|
||
except Exception as exc:
|
||
self.error = str(exc)
|
||
self.stop_event.clear()
|
||
self.thread = threading.Thread(target=self._loop, name="khl-nightly-scheduler", daemon=True)
|
||
self.thread.start()
|
||
|
||
def stop(self):
|
||
self.stop_event.set()
|
||
if self.thread:
|
||
self.thread.join(timeout=2)
|
||
|
||
def _loop(self):
|
||
while not self.stop_event.is_set():
|
||
try:
|
||
self.tick()
|
||
self.error = ""
|
||
except Exception as exc:
|
||
self.error = str(exc)
|
||
self.stop_event.wait(30)
|