BUILD148 — исправление OOM и запуска
This commit is contained in:
8
app.py
8
app.py
@@ -29,7 +29,7 @@ from ui_builder import install_ui_builder
|
|||||||
from khl_site.khl_data_center import APP as khl_site_app
|
from khl_site.khl_data_center import APP as khl_site_app
|
||||||
|
|
||||||
BASE_DIR = Path(__file__).resolve().parent
|
BASE_DIR = Path(__file__).resolve().parent
|
||||||
BUILD_VERSION = "2026.09.18.5"
|
BUILD_VERSION = "2026.09.18.6"
|
||||||
# compatibility: BUILD_VERSION = "2026.08.26.1"
|
# compatibility: BUILD_VERSION = "2026.08.26.1"
|
||||||
# compatibility: BUILD_VERSION = "2026.08.25.1"
|
# compatibility: BUILD_VERSION = "2026.08.25.1"
|
||||||
# compatibility: BUILD_VERSION = "2026.08.24.15"
|
# compatibility: BUILD_VERSION = "2026.08.24.15"
|
||||||
@@ -67,6 +67,12 @@ configure_hockey_vmix_settings(BASE_DIR)
|
|||||||
app = FastAPI(title=f"Хоккейная панель управления / Stat2TV — build {BUILD_VERSION}")
|
app = FastAPI(title=f"Хоккейная панель управления / Stat2TV — build {BUILD_VERSION}")
|
||||||
|
|
||||||
print(f"[Hockey Build] {BUILD_VERSION}")
|
print(f"[Hockey Build] {BUILD_VERSION}")
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/health", include_in_schema=False)
|
||||||
|
async def hockey_public_health() -> dict[str, Any]:
|
||||||
|
"""Unauthenticated liveness probe for systemd/deploy scripts."""
|
||||||
|
return {"ok": True, "build": BUILD_VERSION}
|
||||||
hockey_settings = HockeySettingsStore(BASE_DIR / "settings")
|
hockey_settings = HockeySettingsStore(BASE_DIR / "settings")
|
||||||
hockey_database = HockeyDatabase(BASE_DIR)
|
hockey_database = HockeyDatabase(BASE_DIR)
|
||||||
hockey_auth = HockeyAuthDependencies(create_auth_adapter(hockey_database))
|
hockey_auth = HockeyAuthDependencies(create_auth_adapter(hockey_database))
|
||||||
|
|||||||
261
deploy_hockey.sh
Normal file
261
deploy_hockey.sh
Normal file
@@ -0,0 +1,261 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
# Hockey Web / Stat2TV deploy
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
|
REPO_URL="https://git.tvstart.ru/ychernenko/hockey_new.git"
|
||||||
|
TARGET_DIR="/root/KHL"
|
||||||
|
SERVICE_NAME="khl-data.service"
|
||||||
|
ENV_FILE="/mnt/khl/.env"
|
||||||
|
PORT="8000"
|
||||||
|
RELEASE="main"
|
||||||
|
|
||||||
|
log() { printf '\033[0;32m[INFO]\033[0m %s\n' "$*"; }
|
||||||
|
warn() { printf '\033[1;33m[WARN]\033[0m %s\n' "$*" >&2; }
|
||||||
|
err() { printf '\033[0;31m[ERROR]\033[0m %s\n' "$*" >&2; }
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<USAGE
|
||||||
|
Использование:
|
||||||
|
sudo $0 [-r branch|tag|commit]
|
||||||
|
|
||||||
|
Примеры:
|
||||||
|
sudo $0 -r main
|
||||||
|
sudo $0 -r Barabanov_TEST
|
||||||
|
sudo $0 -r v1.2.3
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
while getopts ":r:h" opt; do
|
||||||
|
case "$opt" in
|
||||||
|
r) RELEASE="$OPTARG" ;;
|
||||||
|
h) usage; exit 0 ;;
|
||||||
|
:) err "Для -$OPTARG требуется значение"; exit 2 ;;
|
||||||
|
\?) err "Неизвестный параметр: -$OPTARG"; usage; exit 2 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ ${EUID:-$(id -u)} -ne 0 ]]; then
|
||||||
|
err "Запусти deploy от root: sudo $0 -r $RELEASE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
install_system_dependencies() {
|
||||||
|
local missing=()
|
||||||
|
for cmd in git python3 curl systemctl; do
|
||||||
|
command -v "$cmd" >/dev/null 2>&1 || missing+=("$cmd")
|
||||||
|
done
|
||||||
|
|
||||||
|
if ! python3 -m venv --help >/dev/null 2>&1; then
|
||||||
|
missing+=("python3-venv")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ((${#missing[@]} > 0)); then
|
||||||
|
log "Устанавливаю системные зависимости: ${missing[*]}"
|
||||||
|
apt-get update
|
||||||
|
DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
||||||
|
git python3 python3-venv python3-pip curl ca-certificates
|
||||||
|
else
|
||||||
|
log "Системные зависимости уже установлены"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_env() {
|
||||||
|
if [[ ! -f "$ENV_FILE" ]]; then
|
||||||
|
err "Не найден файл окружения: $ENV_FILE"
|
||||||
|
err "Создай его до deploy. Минимально нужны HOCKEY_DATABASE_URL и WFL_DATABASE_URL."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
chmod 600 "$ENV_FILE" || true
|
||||||
|
}
|
||||||
|
|
||||||
|
checkout_release() {
|
||||||
|
if [[ -d "$TARGET_DIR/.git" ]]; then
|
||||||
|
log "Обновляю существующий репозиторий в $TARGET_DIR"
|
||||||
|
git -C "$TARGET_DIR" remote set-url origin "$REPO_URL"
|
||||||
|
git -C "$TARGET_DIR" fetch origin --prune --tags
|
||||||
|
else
|
||||||
|
log "Клонирую $REPO_URL -> $TARGET_DIR"
|
||||||
|
mkdir -p "$(dirname "$TARGET_DIR")"
|
||||||
|
git clone "$REPO_URL" "$TARGET_DIR"
|
||||||
|
git -C "$TARGET_DIR" fetch origin --prune --tags
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Ветка origin/<release>
|
||||||
|
if git -C "$TARGET_DIR" show-ref --verify --quiet "refs/remotes/origin/$RELEASE"; then
|
||||||
|
log "Deploy ветки: $RELEASE"
|
||||||
|
git -C "$TARGET_DIR" checkout -B "$RELEASE" "origin/$RELEASE"
|
||||||
|
git -C "$TARGET_DIR" reset --hard "origin/$RELEASE"
|
||||||
|
|
||||||
|
# Git tag
|
||||||
|
elif git -C "$TARGET_DIR" show-ref --verify --quiet "refs/tags/$RELEASE"; then
|
||||||
|
log "Deploy тега: $RELEASE"
|
||||||
|
git -C "$TARGET_DIR" checkout --detach "refs/tags/$RELEASE"
|
||||||
|
|
||||||
|
# SHA / локально разрешимый commit-ish
|
||||||
|
elif git -C "$TARGET_DIR" rev-parse --verify --quiet "$RELEASE^{commit}" >/dev/null; then
|
||||||
|
log "Deploy commit: $RELEASE"
|
||||||
|
git -C "$TARGET_DIR" checkout --detach "$RELEASE"
|
||||||
|
|
||||||
|
else
|
||||||
|
err "Не найдена ветка, тег или commit: $RELEASE"
|
||||||
|
echo ""
|
||||||
|
echo "Удалённые ветки:"
|
||||||
|
git -C "$TARGET_DIR" branch -r || true
|
||||||
|
echo ""
|
||||||
|
echo "Теги:"
|
||||||
|
git -C "$TARGET_DIR" tag -l | tail -50 || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Удаляем только обычные untracked-файлы. Игнорируемые .env/settings не трогаются.
|
||||||
|
git -C "$TARGET_DIR" clean -fd
|
||||||
|
|
||||||
|
log "Commit: $(git -C "$TARGET_DIR" log -1 --oneline)"
|
||||||
|
}
|
||||||
|
|
||||||
|
check_project_files() {
|
||||||
|
local required=("app.py" "requirements.txt" "check_remote_database.py" "hockey_data/__init__.py" "ui_builder/__init__.py" "khl_site/khl_data_center.py")
|
||||||
|
local f
|
||||||
|
for f in "${required[@]}"; do
|
||||||
|
if [[ ! -f "$TARGET_DIR/$f" ]]; then
|
||||||
|
err "В релизе отсутствует обязательный файл: $f"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
prepare_env_link() {
|
||||||
|
# Приложение само читает .env из корня проекта. Оставляем секреты вне Git,
|
||||||
|
# но делаем ссылку, чтобы одинаково работал systemd и ручной запуск.
|
||||||
|
if [[ -e "$TARGET_DIR/.env" && ! -L "$TARGET_DIR/.env" ]]; then
|
||||||
|
warn "$TARGET_DIR/.env уже существует обычным файлом — оставляю его как есть"
|
||||||
|
else
|
||||||
|
ln -sfn "$ENV_FILE" "$TARGET_DIR/.env"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
prepare_venv() {
|
||||||
|
if [[ ! -x "$TARGET_DIR/.venv/bin/python" ]]; then
|
||||||
|
log "Создаю virtualenv"
|
||||||
|
python3 -m venv "$TARGET_DIR/.venv"
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Обновляю Python-зависимости"
|
||||||
|
"$TARGET_DIR/.venv/bin/python" -m pip install --upgrade pip setuptools wheel
|
||||||
|
"$TARGET_DIR/.venv/bin/python" -m pip install -r "$TARGET_DIR/requirements.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_code() {
|
||||||
|
log "Проверяю Python-код"
|
||||||
|
"$TARGET_DIR/.venv/bin/python" -m py_compile \
|
||||||
|
"$TARGET_DIR/app.py" \
|
||||||
|
"$TARGET_DIR/check_remote_database.py"
|
||||||
|
|
||||||
|
# Проверка БД тем же кодом, который использует проект.
|
||||||
|
log "Проверяю PostgreSQL"
|
||||||
|
(
|
||||||
|
cd "$TARGET_DIR"
|
||||||
|
"$TARGET_DIR/.venv/bin/python" "$TARGET_DIR/check_remote_database.py"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
write_systemd_service() {
|
||||||
|
log "Создаю systemd unit: /etc/systemd/system/$SERVICE_NAME"
|
||||||
|
|
||||||
|
cat > "/etc/systemd/system/$SERVICE_NAME" <<UNIT
|
||||||
|
[Unit]
|
||||||
|
Description=Hockey Stat2TV / UI Builder
|
||||||
|
After=network-online.target
|
||||||
|
Wants=network-online.target
|
||||||
|
StartLimitIntervalSec=300
|
||||||
|
StartLimitBurst=5
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=root
|
||||||
|
WorkingDirectory=$TARGET_DIR
|
||||||
|
EnvironmentFile=$ENV_FILE
|
||||||
|
Environment=PYTHONUNBUFFERED=1
|
||||||
|
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||||
|
|
||||||
|
# Перед стартом убеждаемся, что PostgreSQL доступен.
|
||||||
|
ExecStartPre=$TARGET_DIR/.venv/bin/python $TARGET_DIR/check_remote_database.py
|
||||||
|
|
||||||
|
# ГЛАВНАЯ КОМАНДА ЗАПУСКА ПРОЕКТА:
|
||||||
|
ExecStart=$TARGET_DIR/.venv/bin/python -m uvicorn app:app --host 0.0.0.0 --port $PORT
|
||||||
|
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5
|
||||||
|
TimeoutStopSec=20
|
||||||
|
KillSignal=SIGINT
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
SyslogIdentifier=KHL
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
UNIT
|
||||||
|
|
||||||
|
chmod 644 "/etc/systemd/system/$SERVICE_NAME"
|
||||||
|
systemctl daemon-reload
|
||||||
|
systemctl enable "$SERVICE_NAME" >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
restart_service() {
|
||||||
|
log "Перезапускаю $SERVICE_NAME"
|
||||||
|
systemctl restart "$SERVICE_NAME"
|
||||||
|
|
||||||
|
log "Жду HTTP на 127.0.0.1:$PORT"
|
||||||
|
local ok=0
|
||||||
|
for _ in $(seq 1 30); do
|
||||||
|
if curl -fsS --max-time 2 "http://127.0.0.1:$PORT/api/health" >/tmp/khl-health.json 2>/dev/null; then
|
||||||
|
ok=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ "$ok" -ne 1 ]]; then
|
||||||
|
err "Приложение не прошло HTTP-проверку"
|
||||||
|
systemctl status "$SERVICE_NAME" --no-pager -l || true
|
||||||
|
echo ""
|
||||||
|
journalctl -u "$SERVICE_NAME" -n 100 --no-pager || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "HTTP OK: $(cat /tmp/khl-health.json)"
|
||||||
|
rm -f /tmp/khl-health.json
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
log "Deploy Hockey: release=$RELEASE"
|
||||||
|
install_system_dependencies
|
||||||
|
check_env
|
||||||
|
checkout_release
|
||||||
|
check_project_files
|
||||||
|
|
||||||
|
# Не меняем активный venv, пока код ещё не скачан/проверен по структуре.
|
||||||
|
systemctl stop "$SERVICE_NAME" 2>/dev/null || true
|
||||||
|
|
||||||
|
prepare_env_link
|
||||||
|
prepare_venv
|
||||||
|
validate_code
|
||||||
|
write_systemd_service
|
||||||
|
restart_service
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
log "Deploy завершён"
|
||||||
|
log "Service: $SERVICE_NAME"
|
||||||
|
log "URL: http://<SERVER_IP>:$PORT/"
|
||||||
|
log "Status: systemctl status $SERVICE_NAME"
|
||||||
|
log "Logs: journalctl -u $SERVICE_NAME -f"
|
||||||
|
log "Restart: systemctl restart $SERVICE_NAME"
|
||||||
|
echo ""
|
||||||
|
log "Команда запуска приложения:"
|
||||||
|
echo " $TARGET_DIR/.venv/bin/python -m uvicorn app:app --host 0.0.0.0 --port $PORT"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
@@ -237,7 +237,9 @@ class HockeyDataService:
|
|||||||
# tournament cache up to date. A deep pass runs on startup and once per
|
# tournament cache up to date. A deep pass runs on startup and once per
|
||||||
# calendar day; the remaining hourly passes only recheck recent matches.
|
# calendar day; the remaining hourly passes only recheck recent matches.
|
||||||
first_run = True
|
first_run = True
|
||||||
await asyncio.sleep(4)
|
# BUILD148: give Uvicorn/deploy health checks time to settle before any
|
||||||
|
# network-heavy catalogue maintenance begins.
|
||||||
|
await asyncio.sleep(15)
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
today_key = date.today().isoformat()
|
today_key = date.today().isoformat()
|
||||||
@@ -368,8 +370,15 @@ class HockeyDataService:
|
|||||||
# every game shown on that date, not only the selected game.
|
# every game shown on that date, not only the selected game.
|
||||||
include = row.game_date == on_date
|
include = row.game_date == on_date
|
||||||
elif deep:
|
elif deep:
|
||||||
# Future matches have no meaningful score to reconcile yet.
|
# BUILD148: a daily/background deep pass must not walk the
|
||||||
include = row.game_date is None or row.game_date <= today
|
# whole historical database. Old finished matches are stable;
|
||||||
|
# reconcile only the recent window where late score/status
|
||||||
|
# corrections are still realistic. This keeps startup bounded.
|
||||||
|
deep_from = today - timedelta(days=30)
|
||||||
|
include = (
|
||||||
|
row.game_date is not None
|
||||||
|
and deep_from <= row.game_date <= today
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
include = (
|
include = (
|
||||||
row.game_date is not None
|
row.game_date is not None
|
||||||
@@ -439,11 +448,16 @@ class HockeyDataService:
|
|||||||
errors.append(f"{game_id}: {str(error)[:180]}")
|
errors.append(f"{game_id}: {str(error)[:180]}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
results = (
|
# BUILD148: keep the number of simultaneously-created coroutine/result
|
||||||
await asyncio.gather(*(guarded(game_id) for game_id in selected_game_ids))
|
# objects bounded. The inner semaphore limits network concurrency to four,
|
||||||
if selected_game_ids
|
# while batching also prevents a tournament with hundreds of games from
|
||||||
else []
|
# producing a large temporary allocation during startup maintenance.
|
||||||
)
|
results: list[dict[str, bool] | None] = []
|
||||||
|
batch_size = 16
|
||||||
|
for offset in range(0, len(selected_game_ids), batch_size):
|
||||||
|
batch = selected_game_ids[offset:offset + batch_size]
|
||||||
|
results.extend(await asyncio.gather(*(guarded(game_id) for game_id in batch)))
|
||||||
|
|
||||||
failed = sum(1 for value in results if value is None)
|
failed = sum(1 for value in results if value is None)
|
||||||
updated = sum(1 for value in results if value and value.get("changed"))
|
updated = sum(1 for value in results if value and value.get("changed"))
|
||||||
score_updated = sum(1 for value in results if value and value.get("score_changed"))
|
score_updated = sum(1 for value in results if value and value.get("score_changed"))
|
||||||
@@ -2309,12 +2323,40 @@ class HockeyDataService:
|
|||||||
cache_seconds: float,
|
cache_seconds: float,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
) -> Any:
|
) -> Any:
|
||||||
|
"""Fetch a Stat2TV resource with a small bounded in-memory cache.
|
||||||
|
|
||||||
|
BUILD148 fixes an important leak introduced by per-game reconciliation:
|
||||||
|
cache_seconds=0 means *do not cache*. Previously every unique game JSON
|
||||||
|
response was still retained forever in ``_resource_response_cache``, so a
|
||||||
|
deep pass could keep hundreds of megabytes alive until the OOM killer
|
||||||
|
terminated Uvicorn. Positive-TTL resources remain cached, but the cache is
|
||||||
|
capped so changing tournaments cannot grow it without bound.
|
||||||
|
"""
|
||||||
now = time.monotonic()
|
now = time.monotonic()
|
||||||
cached = self._resource_response_cache.get(endpoint)
|
ttl = max(0.0, float(cache_seconds or 0.0))
|
||||||
if not force and cached and now - cached[0] < max(0.0, cache_seconds):
|
|
||||||
return cached[1]
|
if ttl > 0.0:
|
||||||
|
cached = self._resource_response_cache.get(endpoint)
|
||||||
|
if not force and cached and now - cached[0] < ttl:
|
||||||
|
return cached[1]
|
||||||
|
else:
|
||||||
|
# A zero-TTL caller explicitly requests a fresh, non-retained value.
|
||||||
|
self._resource_response_cache.pop(endpoint, None)
|
||||||
|
|
||||||
response = await self.client.fetch_xml(endpoint)
|
response = await self.client.fetch_xml(endpoint)
|
||||||
self._resource_response_cache[endpoint] = (now, response)
|
|
||||||
|
if ttl > 0.0:
|
||||||
|
self._resource_response_cache[endpoint] = (now, response)
|
||||||
|
max_entries = 96
|
||||||
|
overflow = len(self._resource_response_cache) - max_entries
|
||||||
|
if overflow > 0:
|
||||||
|
oldest = sorted(
|
||||||
|
self._resource_response_cache.items(),
|
||||||
|
key=lambda item: item[1][0],
|
||||||
|
)[:overflow]
|
||||||
|
for key, _value in oldest:
|
||||||
|
self._resource_response_cache.pop(key, None)
|
||||||
|
|
||||||
return response
|
return response
|
||||||
|
|
||||||
def _save_snapshot(
|
def _save_snapshot(
|
||||||
|
|||||||
Reference in New Issue
Block a user