From 3577ff32ac107b1b3c2d1253fd8eb34eeb9f6a72 Mon Sep 17 00:00:00 2001 From: claude-bot Date: Mon, 1 Jun 2026 19:38:55 +0000 Subject: [PATCH] feat(ET-009): activate EnduroRussia + Wikiloc GPS sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Конфиг-only активация двух новых источников GPS-треков поверх pipeline ET-008. Не вводит новых компонентов, БД-таблиц, endpoint'ов. Config: - config/gps_sources.yaml: enduro_russia enabled=true, base_url исправлен на endurorussia.ru (без дефиса); добавлена запись wikiloc с max_tracks_per_run=50, activity_filter=[motorcycle, enduro]. - config/gps_regions.yaml: wikiloc добавлен в tsfo_plus_chuvashia.sources. Parser: - wikiloc.py: добавлен soft-cap max_tracks_per_run в collect(), извлечение created_at из GPX metadata/первого trkpt — для корректной межисточниковой дедупликации с EnduroRussia. UI (src/web/gps_tracks.js): - GPS_SOURCE_COLORS: добавлен цвет wikiloc (#4363d8). - Дефолтный фильтр sources включает wikiloc. - GPS_SOURCE_ATTRIBUTIONS: маппинг source_id → строка атрибуции; _updateGpsAttribution() подтягивает /api/gps-tracks/health и выставляет attribution с теми источниками, у которых tracks > 0. - _buildGpsFiltersUI: чекбокс «Wikiloc» в #gps-source-grid. Tests: - Fixtures: 7 файлов в tests/fixtures/gps-tracks/. - Unit: 10 UT-ER + 10 UT-WL — парсеры, MAPPING, bbox-фильтр, pagination, 429/403 graceful-stop, rate-limit, max_tracks_per_run. - Integration: IT-ER-01, IT-WL-01, IT-WL-02, IT-DEDUP-01, IT-LIC-01 через scripts.gps_collect.main + httpx.MockTransport. - Contract: 2 CT-ER с маркером @pytest.mark.network (nightly only). - JS: 2 новых теста на наличие wikiloc в SOURCE_COLORS и в фильтрах. Linters/Tests: ruff clean (новые файлы), 166 pytest passed, 24 JS-tests passed. Refs: ET-009 Acceptance: AC-01..AC-08, AC-14..AC-17 (для AC-09..AC-13 — продакшн-прогон) Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 13 + config/gps_regions.yaml | 2 +- config/gps_sources.yaml | 19 +- pyproject.toml | 4 + src/api/gps_tracks/sources/wikiloc.py | 54 ++- src/web/gps_tracks.js | 66 +++- tests/contract/__init__.py | 0 tests/contract/test_endurorussia_api_smoke.py | 64 ++++ .../enduro-russia-api-tracks-page1.json | 46 +++ .../gps-tracks/enduro-russia-track-1.gpx | 25 ++ .../gps-tracks/enduro-russia-track-2.gpx | 12 + .../gps-tracks/enduro-russia-track-3.gpx | 18 + .../gps-tracks/wikiloc-search-page1.html | 32 ++ tests/fixtures/gps-tracks/wikiloc-track.gpx | 25 ++ .../gps-tracks/wikiloc-trail-page.html | 20 + tests/integration/test_pipeline_et009.py | 360 ++++++++++++++++++ tests/unit/test_gps_tracks_enduro_russia.py | 264 +++++++++++++ tests/unit/test_gps_tracks_wikiloc.py | 262 +++++++++++++ tests/web/gps_tracks.test.js | 20 +- 19 files changed, 1287 insertions(+), 19 deletions(-) create mode 100644 tests/contract/__init__.py create mode 100644 tests/contract/test_endurorussia_api_smoke.py create mode 100644 tests/fixtures/gps-tracks/enduro-russia-api-tracks-page1.json create mode 100644 tests/fixtures/gps-tracks/enduro-russia-track-1.gpx create mode 100644 tests/fixtures/gps-tracks/enduro-russia-track-2.gpx create mode 100644 tests/fixtures/gps-tracks/enduro-russia-track-3.gpx create mode 100644 tests/fixtures/gps-tracks/wikiloc-search-page1.html create mode 100644 tests/fixtures/gps-tracks/wikiloc-track.gpx create mode 100644 tests/fixtures/gps-tracks/wikiloc-trail-page.html create mode 100644 tests/integration/test_pipeline_et009.py create mode 100644 tests/unit/test_gps_tracks_enduro_russia.py create mode 100644 tests/unit/test_gps_tracks_wikiloc.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a3f71c..a1c58f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,19 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ## [Unreleased] ### Added +- ET-009: Активация GPS-источников EnduroRussia и Wikiloc — `config/gps_sources.yaml` + включает оба источника (`enabled: true`), для Wikiloc добавлен soft-cap + `max_tracks_per_run: 50` и activity-фильтр; `config/gps_regions.yaml` подписывает + `wikiloc` на регион `tsfo_plus_chuvashia`. Парсер `wikiloc.py` извлекает время из + GPX-metadata (для корректной дедупликации) и поддерживает `max_tracks_per_run` + cap. UI: цвет `wikiloc`, чекбокс источника, динамическая атрибуция + (`GPS_SOURCE_ATTRIBUTIONS`) подтягивается с `/api/gps-tracks/health`. + Тесты: 10 unit ER + 10 unit WL + 5 integration + 2 contract (nightly only). + +### Fixed +- ET-009: исправлен URL `enduro_russia` в `config/gps_sources.yaml` + (`https://enduro-russia.ru` → `https://endurorussia.ru`, без дефиса). + - Initial project structure - CLAUDE.md project passport - Agent system prompts (architect, developer, reviewer, tester, deployer) diff --git a/config/gps_regions.yaml b/config/gps_regions.yaml index ba80554..dd276ed 100644 --- a/config/gps_regions.yaml +++ b/config/gps_regions.yaml @@ -3,7 +3,7 @@ regions: name: "ЦФО + Чувашия" bbox: [29.0, 49.5, 47.5, 60.0] enabled: true - sources: [osm, enduro_russia, ttrails] + sources: [osm, enduro_russia, wikiloc, ttrails] - id: north_caucasus name: "Северный Кавказ" diff --git a/config/gps_sources.yaml b/config/gps_sources.yaml index 98dc559..a8fd78c 100644 --- a/config/gps_sources.yaml +++ b/config/gps_sources.yaml @@ -13,14 +13,29 @@ sources: - id: enduro_russia name: "EnduroRussia.ru" - enabled: false + enabled: true license_adr: "docs/work-items/ET-008/06-adr/ADR-010-enduro-russia-licensing.md" - base_url: "https://enduro-russia.ru" + base_url: "https://endurorussia.ru" rate_limit_sec: 5 user_agent: "enduro-trails/1.0 (+https://openclaw.mva154.duckdns.org/enduro/)" attribution: "EnduroRussia.ru" parser_module: "src.api.gps_tracks.sources.enduro_russia" save_user_field: false + source_priority: 80 + + - id: wikiloc + name: "Wikiloc" + enabled: true + license_adr: "docs/work-items/ET-008/06-adr/ADR-012-wikiloc-licensing.md" + base_url: "https://www.wikiloc.com" + rate_limit_sec: 10 + user_agent: "enduro-trails/1.0 (+https://openclaw.mva154.duckdns.org/enduro/)" + attribution: "© Wikiloc contributors" + parser_module: "src.api.gps_tracks.sources.wikiloc" + save_user_field: false + source_priority: 70 + activity_filter: [motorcycle, enduro] + max_tracks_per_run: 50 - id: ttrails name: "Тропинки.ру" diff --git a/pyproject.toml b/pyproject.toml index 9e1a6c9..03bb965 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,3 +35,7 @@ line-length = 120 [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] +markers = [ + "network: contract smoke tests that hit live HTTP endpoints (deselect with '-m \"not network\"')", +] +addopts = "-m 'not network'" diff --git a/src/api/gps_tracks/sources/wikiloc.py b/src/api/gps_tracks/sources/wikiloc.py index a26dbe6..b839100 100644 --- a/src/api/gps_tracks/sources/wikiloc.py +++ b/src/api/gps_tracks/sources/wikiloc.py @@ -62,6 +62,8 @@ class WikilocParser(SourceParser): source_id = self.config.get("id", "wikiloc") source_priority = self.config.get("source_priority", 70) activity_filter = self.config.get("activity_filter", ["motorcycle", "enduro"]) + max_tracks = self.config.get("max_tracks_per_run") + yielded = 0 headers = { "User-Agent": user_agent, @@ -188,7 +190,15 @@ class WikilocParser(SourceParser): ): continue + if max_tracks is not None and yielded >= max_tracks: + logger.info( + "Wikiloc: reached max_tracks_per_run=%d, stopping", + max_tracks, + ) + return + yield track + yielded += 1 page += 1 @@ -260,16 +270,40 @@ def _parse_gpx( if tag.startswith("{"): ns = tag.split("}")[0] + "}" - # Извлекаем название из GPX metadata если нет из HTML - if not name: - for child in root: - local = child.tag.replace(ns, "") if ns else child.tag - if local == "metadata": - for meta_child in child: - local2 = meta_child.tag.replace(ns, "") if ns else meta_child.tag - if local2 == "name": - name = meta_child.text + # Извлекаем название и время из GPX metadata + created_at = None + for child in root: + local = child.tag.replace(ns, "") if ns else child.tag + if local == "metadata": + for meta_child in child: + local2 = meta_child.tag.replace(ns, "") if ns else meta_child.tag + if local2 == "name" and not name: + name = meta_child.text + elif local2 == "time" and meta_child.text: + created_at = meta_child.text.strip() + break + + # Fallback: первая