190 build - улучшение работы всего сервиса. часть 1
This commit is contained in:
51
hockey_data/snapshot_storage.py
Normal file
51
hockey_data/snapshot_storage.py
Normal file
@@ -0,0 +1,51 @@
|
||||
"""Bound debug reports and reuse identical XML without breaking historical paths."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def save_xml_snapshot(directory: Path, content: bytes, *, resource: str, tournament: str = "") -> Path:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
safe_resource = re.sub(r"[^A-Za-z0-9_-]", "_", str(resource))[:64] or "resource"
|
||||
safe_tournament = re.sub(r"[^A-Za-z0-9_-]", "_", str(tournament))[:48]
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
path = directory / f"{digest}_{safe_tournament}_{safe_resource}.xml"
|
||||
if path.is_file() and not path.is_symlink() and path.read_bytes() == content:
|
||||
return path
|
||||
temporary = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(dir=directory, prefix="snapshot_", suffix=".tmp", delete=False) as stream:
|
||||
temporary = Path(stream.name)
|
||||
stream.write(content)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
if temporary is not None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def trim_discovery_reports(directory: Path, *, keep: int = 40, days: int = 30) -> int:
|
||||
"""Only standalone debug JSON is disposable; XML paths may be stored in PostgreSQL."""
|
||||
pattern = re.compile(r"^\d{8}_\d{6}_\d{6}_[A-Za-z0-9_-]+_games_discovery\.json$")
|
||||
reports = sorted((path for path in directory.glob("*_games_discovery.json")
|
||||
if pattern.fullmatch(path.name) and path.is_file() and not path.is_symlink()),
|
||||
key=lambda path: path.name, reverse=True)
|
||||
cutoff = time.time() - max(1, days) * 86400
|
||||
removed = 0
|
||||
for path in reports[max(1, keep):]:
|
||||
try:
|
||||
if path.stat().st_mtime < cutoff:
|
||||
path.unlink(missing_ok=True)
|
||||
removed += 1
|
||||
except OSError:
|
||||
# Concurrent maintenance or a read-only diagnostic directory must
|
||||
# not turn a successful data import into a failed synchronization.
|
||||
continue
|
||||
return removed
|
||||
Reference in New Issue
Block a user