From cc58ef3fff7902174896153fbb225dfb171e9ac5 Mon Sep 17 00:00:00 2001 From: Stream Date: Sun, 26 Apr 2026 02:10:01 +0300 Subject: [PATCH] auto-sync: 2026-04-26 02:10:01 --- .../ingest/tracks_fr24/config.py | 3 + .../ingest/tracks_fr24/fr24_worker.py | 8 +++ .../flightradar24/scripts/audit_containers.sh | 72 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100755 tasks/flightradar24/scripts/audit_containers.sh diff --git a/tasks/flightradar24/ingest/tracks_fr24/config.py b/tasks/flightradar24/ingest/tracks_fr24/config.py index 756e297..aaf56ad 100644 --- a/tasks/flightradar24/ingest/tracks_fr24/config.py +++ b/tasks/flightradar24/ingest/tracks_fr24/config.py @@ -33,6 +33,9 @@ class Config: # Max pages to paginate through (safety cap). 200 × 20 = 4000 flights max MAX_PAGES: int = int(os.getenv("FR24_MAX_PAGES", "200")) + # Credit guard: warn if estimated max flights exceeds this threshold + CREDIT_GUARD_MAX_FLIGHTS: int = int(os.getenv("FR24_CREDIT_GUARD", "2000")) + @property def DB_DSN(self) -> str: return ( diff --git a/tasks/flightradar24/ingest/tracks_fr24/fr24_worker.py b/tasks/flightradar24/ingest/tracks_fr24/fr24_worker.py index ea80a8f..d7dc27e 100644 --- a/tasks/flightradar24/ingest/tracks_fr24/fr24_worker.py +++ b/tasks/flightradar24/ingest/tracks_fr24/fr24_worker.py @@ -457,6 +457,14 @@ def supplement_schedule(conn, target_date: date) -> int: def run(target_date: date, conn) -> Dict: """Main entry: load flight summaries + optionally tracks. Returns stats dict.""" log.info("FR24 worker: starting for %s", target_date) + estimated_max = config.MAX_PAGES * config.PAGE_SIZE + log.info("FR24 worker: MAX_PAGES=%d, PAGE_SIZE=%d, estimated max flights=%d", + config.MAX_PAGES, config.PAGE_SIZE, estimated_max) + if estimated_max > config.CREDIT_GUARD_MAX_FLIGHTS: + log.critical("CREDIT GUARD: estimated max flights %d exceeds limit %d " + "(верифицируй MAX_PAGES=%d и PAGE_SIZE=%d перед запуском!)", + estimated_max, config.CREDIT_GUARD_MAX_FLIGHTS, + config.MAX_PAGES, config.PAGE_SIZE) stats = { "date": str(target_date), "flights_found": 0, diff --git a/tasks/flightradar24/scripts/audit_containers.sh b/tasks/flightradar24/scripts/audit_containers.sh new file mode 100755 index 0000000..65ef9f0 --- /dev/null +++ b/tasks/flightradar24/scripts/audit_containers.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# audit_containers.sh — сравнивает код в Docker-контейнерах с локальным репо +# Запускать перед любым ручным запуском воркеров FR24! +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(dirname "$SCRIPT_DIR")" +SSH="$HOME/.openclaw/skills/installer/scripts/ssh_exec.sh" + +declare -A CONTAINERS +CONTAINERS["fr24-tracks-fr24"]="ingest/tracks_fr24" +CONTAINERS["fr24-schedule"]="ingest/schedule" +CONTAINERS["fr24-mart"]="ingest/mart" +CONTAINERS["fr24-tracks-fa"]="ingest/tracks_fa" + +ERRORS=0 +WARNINGS=0 + +for container in "${!CONTAINERS[@]}"; do + local_dir="$REPO/${CONTAINERS[$container]}" + echo "=== $container ===" + + if [[ ! -d "$local_dir" ]]; then + echo " ⚠️ Локальная папка не найдена: $local_dir" + WARNINGS=$((WARNINGS + 1)) + echo "" + continue + fi + + remote_md5=$("$SSH" --host fr24 --cmd "docker exec $container sh -c 'md5sum /app/*.py 2>/dev/null'" 2>/dev/null || echo "") + + if [[ -z "$remote_md5" ]]; then + echo " ⚠️ Контейнер недоступен или нет .py файлов" + WARNINGS=$((WARNINGS + 1)) + echo "" + continue + fi + + while IFS= read -r line; do + [[ -z "$line" ]] && continue + remote_hash=$(echo "$line" | awk '{print $1}') + remote_file=$(basename "$(echo "$line" | awk '{print $2}')") + local_file="$local_dir/$remote_file" + + if [[ ! -f "$local_file" ]]; then + echo " ⚠️ $remote_file — нет локального файла (только в контейнере)" + WARNINGS=$((WARNINGS + 1)) + continue + fi + + local_hash=$(md5sum "$local_file" | awk '{print $1}') + if [[ "$remote_hash" == "$local_hash" ]]; then + echo " ✅ $remote_file" + else + echo " ❌ MISMATCH: $remote_file" + echo " container: $remote_hash" + echo " local: $local_hash" + ERRORS=$((ERRORS + 1)) + fi + done <<< "$remote_md5" + echo "" +done + +echo "========================================" +if [[ $ERRORS -eq 0 && $WARNINGS -eq 0 ]]; then + echo "✅ Всё совпадает — можно запускать воркеры" +elif [[ $ERRORS -eq 0 ]]; then + echo "⚠️ Предупреждений: $WARNINGS (ошибок нет)" +else + echo "❌ Расхождений: $ERRORS — задеплоить актуальные версии ПЕРЕД запуском воркеров!" + exit 1 +fi