From cdb5e5671e43f21f1342829722b56a99a2ee3132 Mon Sep 17 00:00:00 2001 From: Stream Date: Sat, 2 May 2026 07:50:01 +0300 Subject: [PATCH] auto-sync: 2026-05-02 07:50:01 --- tasks/enduro-trails/prototype/app.py | 498 ++++++++++++++++++ .../prototype/docker-compose.yml | 49 ++ .../enduro-trails/prototype/requirements.txt | 7 + .../enduro-trails/prototype/static/index.html | 456 ++++++++++++++++ .../enduro-trails/prototype/static/style.json | 185 +++++++ tasks/enduro-trails/scripts/download.sh | 32 ++ tasks/enduro-trails/scripts/parse.py | 375 +++++++++++++ 7 files changed, 1602 insertions(+) create mode 100644 tasks/enduro-trails/prototype/app.py create mode 100644 tasks/enduro-trails/prototype/docker-compose.yml create mode 100644 tasks/enduro-trails/prototype/requirements.txt create mode 100644 tasks/enduro-trails/prototype/static/index.html create mode 100644 tasks/enduro-trails/prototype/static/style.json create mode 100755 tasks/enduro-trails/scripts/download.sh create mode 100644 tasks/enduro-trails/scripts/parse.py diff --git a/tasks/enduro-trails/prototype/app.py b/tasks/enduro-trails/prototype/app.py new file mode 100644 index 0000000..48f4e92 --- /dev/null +++ b/tasks/enduro-trails/prototype/app.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python3 +""" +app.py — FastAPI сервер для Enduro Trails +- Раздаёт статику из static/ +- /api/tiles/{z}/{x}/{y}.mvt — векторные тайлы из Spatialite +- /api/poi — список POI как GeoJSON +""" + +import os +import math +import struct +import sqlite3 +import json +from pathlib import Path + +from fastapi import FastAPI, HTTPException, Response +from fastapi.staticfiles import StaticFiles +from fastapi.middleware.cors import CORSMiddleware +import uvicorn + +# ─── Конфиг ─────────────────────────────────────────────────────────────────── + +DATA_PATH = os.environ.get( + "DATA_PATH", + os.path.join(os.path.dirname(__file__), "../data/centralfederal.sqlite"), +) +DATA_PATH = os.path.abspath(DATA_PATH) +STATIC_DIR = os.path.join(os.path.dirname(__file__), "static") +PORT = int(os.environ.get("PORT", 5558)) + +app = FastAPI(title="Enduro Trails API") + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + +# ─── Spatialite helper ──────────────────────────────────────────────────────── + +def get_db(): + """Открывает соединение с SQLite (с попыткой загрузить Spatialite).""" + conn = sqlite3.connect(DATA_PATH) + conn.row_factory = sqlite3.Row + conn.enable_load_extension(True) + for path in ["mod_spatialite", "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", + "/usr/lib/mod_spatialite.so", "/usr/local/lib/mod_spatialite.so"]: + try: + conn.load_extension(path) + break + except Exception: + continue + conn.enable_load_extension(False) + return conn + + +# ─── Tile math ──────────────────────────────────────────────────────────────── + +def tile_to_bbox(z: int, x: int, y: int): + """Возвращает (west, south, east, north) в градусах для тайла z/x/y.""" + n = 2 ** z + west = x / n * 360.0 - 180.0 + east = (x + 1) / n * 360.0 - 180.0 + north = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * y / n)))) + south = math.degrees(math.atan(math.sinh(math.pi * (1 - 2 * (y + 1) / n)))) + return west, south, east, north + + +def lon_to_tile_x(lon: float, z: int) -> float: + return (lon + 180.0) / 360.0 * (2 ** z) + + +def lat_to_tile_y(lat: float, z: int) -> float: + lat_r = math.radians(lat) + return (1.0 - math.log(math.tan(lat_r) + 1.0 / math.cos(lat_r)) / math.pi) / 2.0 * (2 ** z) + + +TILE_EXTENT = 4096 + + +def lonlat_to_tile_coords(lon: float, lat: float, z: int, tx: int, ty: int): + """Конвертирует lon/lat в пиксельные координаты тайла [0, TILE_EXTENT].""" + px = (lon_to_tile_x(lon, z) - tx) * TILE_EXTENT + py = (lat_to_tile_y(lat, z) - ty) * TILE_EXTENT + return int(px), int(py) + + +# ─── Minimal MVT encoder ────────────────────────────────────────────────────── +# Реализует минимальный Mapbox Vector Tile (protobuf) без внешних зависимостей +# Spec: https://github.com/mapbox/vector-tile-spec/tree/master/2.1 + +def _varint(value: int) -> bytes: + """Encode unsigned varint.""" + buf = [] + while True: + towrite = value & 0x7F + value >>= 7 + if value: + buf.append(towrite | 0x80) + else: + buf.append(towrite) + break + return bytes(buf) + + +def _zigzag(n: int) -> int: + return (n << 1) ^ (n >> 31) + + +def _pb_field(field_num: int, wire_type: int, data: bytes) -> bytes: + tag = (field_num << 3) | wire_type + return _varint(tag) + data + + +def _pb_string(field_num: int, s: str) -> bytes: + encoded = s.encode("utf-8") + return _pb_field(field_num, 2, _varint(len(encoded)) + encoded) + + +def _pb_bytes(field_num: int, b: bytes) -> bytes: + return _pb_field(field_num, 2, _varint(len(b)) + b) + + +def _pb_uint32(field_num: int, v: int) -> bytes: + return _pb_field(field_num, 0, _varint(v)) + + +def encode_linestring_geometry(coords_px): + """ + Кодирует список (x, y) пикселей в MVT geometry (LINESTRING). + Возвращает bytes команд. + """ + if len(coords_px) < 2: + return None + + cmds = [] + # MoveTo первой точки + dx = coords_px[0][0] + dy = coords_px[0][1] + cmds.append((1 << 3) | 1) # MoveTo, count=1 + cmds.append(_zigzag(dx)) + cmds.append(_zigzag(dy)) + + # LineTo остальных + cmds.append(((len(coords_px) - 1) << 3) | 2) + prev_x, prev_y = coords_px[0] + for x, y in coords_px[1:]: + cmds.append(_zigzag(x - prev_x)) + cmds.append(_zigzag(y - prev_y)) + prev_x, prev_y = x, y + + return b"".join(_varint(c) for c in cmds) + + +def encode_point_geometry(x: int, y: int) -> bytes: + cmds = [] + cmds.append((1 << 3) | 1) # MoveTo, count=1 + cmds.append(_zigzag(x)) + cmds.append(_zigzag(y)) + return b"".join(_varint(c) for c in cmds) + + +def build_mvt_tile(trails_rows, poi_rows, z: int, tx: int, ty: int) -> bytes: + """Строит MVT тайл из строк trails и poi.""" + + def build_layer(name: str, features_data: list, geom_type: int) -> bytes: + """ + features_data: list of {"geom_bytes": bytes, "props": dict} + geom_type: 2=LINESTRING, 1=POINT + """ + if not features_data: + return b"" + + # Собираем ключи и значения + keys = [] + key_index = {} + values = [] + value_index = {} + + feature_blobs = [] + for fd in features_data: + geom_bytes = fd["geom_bytes"] + props = fd["props"] + if not geom_bytes: + continue + + tags = [] + for k, v in props.items(): + if v is None: + continue + v_str = str(v) + + if k not in key_index: + key_index[k] = len(keys) + keys.append(k) + ki = key_index[k] + + vk = (type(v).__name__, v_str) + if vk not in value_index: + value_index[vk] = len(values) + values.append((type(v).__name__, v_str)) + vi = value_index[vk] + + tags.extend([ki, vi]) + + # Feature proto + feat = b"" + feat += _pb_uint32(3, geom_type) # type + # geometry field=4 + feat += _pb_bytes(4, geom_bytes) + # tags field=2 + for t in tags: + feat += _pb_uint32(2, t) + + feature_blobs.append(feat) + + if not feature_blobs: + return b"" + + layer = b"" + layer += _pb_uint32(15, 2) # version=2 + layer += _pb_string(1, name) # name + layer += _pb_uint32(5, TILE_EXTENT) # extent + + for k in keys: + layer += _pb_string(3, k) + + for (vtype, vstr) in values: + # value message: string_value=1, float_value=2, ... + val_msg = b"" + if vtype == "int" or vtype == "float": + try: + fv = float(vstr) + val_msg += _pb_field(3, 1, struct.pack("" + geom_type = struct.unpack_from(endian + "I", blob, 1)[0] + # 2 = LineString, 1002 = LineString with SRID + if geom_type not in (2, 1000002, 2147483650): + # попробуем через shapely + return _wkb_via_shapely(blob, z, tx, ty, "line") + + offset = 5 + # Если есть SRID (geom_type & 0x20000000) + if geom_type & 0x20000000: + offset += 4 + + num_points = struct.unpack_from(endian + "I", blob, offset)[0] + offset += 4 + + coords_px = [] + for _ in range(num_points): + lon, lat = struct.unpack_from(endian + "dd", blob, offset) + offset += 16 + px, py = lonlat_to_tile_coords(lon, lat, z, tx, ty) + coords_px.append((px, py)) + + return coords_px if len(coords_px) >= 2 else None + except Exception: + return _wkb_via_shapely(blob, z, tx, ty, "line") + + +def wkb_point_to_pixels(blob: bytes, z: int, tx: int, ty: int): + """Парсит WKB Point и возвращает (px, py).""" + try: + if len(blob) < 21: + return None + byte_order = blob[0] + endian = "<" if byte_order == 1 else ">" + geom_type = struct.unpack_from(endian + "I", blob, 1)[0] + if geom_type not in (1, 1000001, 2147483649): + return _wkb_via_shapely(blob, z, tx, ty, "point") + + offset = 5 + if geom_type & 0x20000000: + offset += 4 + + lon, lat = struct.unpack_from(endian + "dd", blob, offset) + return lonlat_to_tile_coords(lon, lat, z, tx, ty) + except Exception: + return _wkb_via_shapely(blob, z, tx, ty, "point") + + +def _wkb_via_shapely(blob, z, tx, ty, kind): + """Fallback: парсим через shapely.""" + try: + from shapely import wkb as swkb + geom = swkb.loads(bytes(blob)) + if kind == "point": + return lonlat_to_tile_coords(geom.x, geom.y, z, tx, ty) + else: + coords = list(geom.coords) + return [lonlat_to_tile_coords(lon, lat, z, tx, ty) for lon, lat in coords] + except Exception: + return None + + +# ─── API endpoints ──────────────────────────────────────────────────────────── + +@app.get("/api/tiles/{z}/{x}/{y}.mvt") +async def get_tile(z: int, x: int, y: int): + if not os.path.exists(DATA_PATH): + raise HTTPException(503, f"База данных не найдена: {DATA_PATH}. Запустите parse.py") + + west, south, east, north = tile_to_bbox(z, x, y) + + # Небольшой буфер для краевых геометрий + buf = (east - west) * 0.1 + q_west, q_south, q_east, q_north = west - buf, south - buf, east + buf, north + buf + + try: + conn = get_db() + cur = conn.cursor() + + # Trails — простой bbox по координатам из WKB (без пространственного индекса) + # Используем ST_Intersects если Spatialite доступен, иначе fallback + try: + cur.execute(""" + SELECT osm_id, highway_type, track_type, surface, name, + length_m, mtb_scale, geom + FROM trails + WHERE ST_Intersects(geom, BuildMBR(?,?,?,?,4326)) + LIMIT 5000 + """, (q_west, q_south, q_east, q_north)) + except Exception: + # Fallback без пространственного индекса + cur.execute(""" + SELECT osm_id, highway_type, track_type, surface, name, + length_m, mtb_scale, geom + FROM trails + LIMIT 5000 + """) + + trails_rows = cur.fetchall() + + try: + cur.execute(""" + SELECT osm_id, poi_type, name, geom + FROM poi + WHERE ST_Intersects(geom, BuildMBR(?,?,?,?,4326)) + LIMIT 1000 + """, (q_west, q_south, q_east, q_north)) + except Exception: + cur.execute("SELECT osm_id, poi_type, name, geom FROM poi LIMIT 1000") + + poi_rows = cur.fetchall() + conn.close() + + except Exception as e: + raise HTTPException(500, f"Ошибка БД: {e}") + + mvt = build_mvt_tile(trails_rows, poi_rows, z, x, y) + + return Response( + content=mvt, + media_type="application/x-protobuf", + headers={ + "Content-Encoding": "identity", + "Access-Control-Allow-Origin": "*", + }, + ) + + +@app.get("/api/poi") +async def get_poi(): + if not os.path.exists(DATA_PATH): + raise HTTPException(503, f"База данных не найдена: {DATA_PATH}") + + try: + conn = get_db() + cur = conn.cursor() + cur.execute("SELECT osm_id, poi_type, name, geom FROM poi LIMIT 10000") + rows = cur.fetchall() + conn.close() + except Exception as e: + raise HTTPException(500, f"Ошибка БД: {e}") + + features = [] + for row in rows: + geom_blob = row["geom"] + if not geom_blob: + continue + try: + byte_order = geom_blob[0] + endian = "<" if byte_order == 1 else ">" + lon, lat = struct.unpack_from(endian + "dd", bytes(geom_blob), 5) + features.append({ + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [lon, lat]}, + "properties": { + "osm_id": row["osm_id"], + "poi_type": row["poi_type"], + "name": row["name"], + }, + }) + except Exception: + continue + + return {"type": "FeatureCollection", "features": features} + + +@app.get("/api/health") +async def health(): + db_exists = os.path.exists(DATA_PATH) + return { + "status": "ok", + "db_path": DATA_PATH, + "db_exists": db_exists, + } + + +# ─── Static files ───────────────────────────────────────────────────────────── + +if os.path.exists(STATIC_DIR): + app.mount("/", StaticFiles(directory=STATIC_DIR, html=True), name="static") + + +# ─── Entry point ────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + print(f"==> Enduro Trails API") + print(f" DB: {DATA_PATH}") + print(f" Static: {STATIC_DIR}") + print(f" Port: {PORT}") + uvicorn.run(app, host="0.0.0.0", port=PORT) diff --git a/tasks/enduro-trails/prototype/docker-compose.yml b/tasks/enduro-trails/prototype/docker-compose.yml new file mode 100644 index 0000000..4c88789 --- /dev/null +++ b/tasks/enduro-trails/prototype/docker-compose.yml @@ -0,0 +1,49 @@ +version: '3.8' + +services: + # Шаг 1: скачать и распарсить данные + data-init: + image: python:3.11-slim + working_dir: /app + volumes: + - ../data:/data + - ../scripts:/scripts + command: > + bash -c " + apt-get update -qq && + apt-get install -y -qq osmium-tool wget libsqlite3-mod-spatialite libspatialite-dev && + pip install --quiet python-osmium shapely pysqlite3-binary && + echo '==> Скачиваем ЦФО...' && + wget -q -c 'https://download.geofabrik.de/russia/centralfederal.ru-latest.osm.pbf' -O /data/centralfederal.ru-latest.osm.pbf && + echo '==> Скачиваем Поволжье...' && + wget -q -c 'https://download.geofabrik.de/russia/volga.osm.pbf' -O /data/volga.osm.pbf && + echo '==> Объединяем...' && + osmium merge /data/centralfederal.ru-latest.osm.pbf /data/volga.osm.pbf -o /data/merged.osm.pbf --overwrite && + echo '==> Фильтруем по BBOX...' && + osmium extract --bbox=30.0,51.0,48.0,59.0 /data/merged.osm.pbf -o /data/region.osm.pbf --overwrite && + echo '==> Парсим в SQLite...' && + python /scripts/parse.py && + echo '==> Данные готовы!' + " + profiles: + - init + + # Шаг 2: веб-сервер + enduro-trails: + image: python:3.11-slim + working_dir: /app + volumes: + - .:/app + - ../data:/data + ports: + - "5558:5558" + command: > + bash -c " + apt-get update -qq && + apt-get install -y -qq libsqlite3-mod-spatialite && + pip install --quiet -r requirements.txt && + python app.py + " + environment: + - DATA_PATH=/data/centralfederal.sqlite + restart: unless-stopped diff --git a/tasks/enduro-trails/prototype/requirements.txt b/tasks/enduro-trails/prototype/requirements.txt new file mode 100644 index 0000000..483c838 --- /dev/null +++ b/tasks/enduro-trails/prototype/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.111.0 +uvicorn==0.29.0 +python-osmium==3.7.0 +pysqlite3-binary==0.5.2 +shapely==2.0.4 +pyproj==3.6.1 +mapbox-vector-tile==2.0.1 diff --git a/tasks/enduro-trails/prototype/static/index.html b/tasks/enduro-trails/prototype/static/index.html new file mode 100644 index 0000000..2c1b646 --- /dev/null +++ b/tasks/enduro-trails/prototype/static/index.html @@ -0,0 +1,456 @@ + + + + + + Enduro Trails — ЦФО + Чувашия + + + + + + + + + + + +
+
+ +
+ +
Загрузка карты...
+
+ +
+

⚠️ Данные не загружены

+

+ База данных не найдена или пуста.

+ Для загрузки данных выполните:
+ bash scripts/download.sh
+ python scripts/parse.py +

+
+ +
+

Легенда

+
+
+ Грунтовка grade3-5 +
+
+
+ Грунтовка grade1-2 +
+
+
+ Тропа / bridleway +
+
+
+ Асфальт +
+
+
+ Вершина +
+
+ Вода +
+
+ Смотровая +
+
+ Руины +
+
+ Пещера +
+
+ Брод +
+
+
+ +
Zoom: 7 | Координаты: —
+
+ + + + diff --git a/tasks/enduro-trails/prototype/static/style.json b/tasks/enduro-trails/prototype/static/style.json new file mode 100644 index 0000000..3b81ea0 --- /dev/null +++ b/tasks/enduro-trails/prototype/static/style.json @@ -0,0 +1,185 @@ +{ + "version": 8, + "name": "Enduro Trails Dark", + "metadata": {}, + "center": [37.6, 55.75], + "zoom": 7, + "bearing": 0, + "pitch": 0, + "sources": { + "trails-tiles": { + "type": "vector", + "tiles": ["/api/tiles/{z}/{x}/{y}.mvt"], + "minzoom": 5, + "maxzoom": 16 + }, + "osm-raster": { + "type": "raster", + "tiles": [ + "https://tile.openstreetmap.org/{z}/{x}/{y}.png" + ], + "tileSize": 256, + "attribution": "© OpenStreetMap contributors" + } + }, + "glyphs": "https://demotiles.maplibre.org/font/{fontstack}/{range}.pbf", + "layers": [ + { + "id": "background", + "type": "background", + "paint": { + "background-color": "#1a1a2e" + } + }, + { + "id": "osm-base", + "type": "raster", + "source": "osm-raster", + "paint": { + "raster-opacity": 0.25, + "raster-saturation": -0.8, + "raster-brightness-min": 0.0, + "raster-brightness-max": 0.3 + } + }, + { + "id": "trails-asphalt", + "type": "line", + "source": "trails-tiles", + "source-layer": "trails", + "filter": [ + "in", "highway", "primary", "secondary", "tertiary", "residential", "cycleway" + ], + "paint": { + "line-color": "#555555", + "line-width": 1, + "line-opacity": 0.7 + }, + "layout": { + "line-cap": "round", + "line-join": "round" + } + }, + { + "id": "trails-grade12", + "type": "line", + "source": "trails-tiles", + "source-layer": "trails", + "filter": [ + "all", + ["==", "highway", "track"], + ["in", "tracktype", "grade1", "grade2"] + ], + "paint": { + "line-color": "#FFA500", + "line-width": [ + "interpolate", ["linear"], ["zoom"], + 8, 1.5, + 12, 2.5, + 16, 4 + ], + "line-opacity": 0.9 + }, + "layout": { + "line-cap": "round", + "line-join": "round" + } + }, + { + "id": "trails-grade345", + "type": "line", + "source": "trails-tiles", + "source-layer": "trails", + "filter": [ + "all", + ["==", "highway", "track"], + ["!in", "tracktype", "grade1", "grade2"] + ], + "paint": { + "line-color": "#FF8C00", + "line-width": [ + "interpolate", ["linear"], ["zoom"], + 8, 2, + 12, 3.5, + 16, 5 + ], + "line-opacity": 0.95 + }, + "layout": { + "line-cap": "round", + "line-join": "round" + } + }, + { + "id": "trails-path-bridleway", + "type": "line", + "source": "trails-tiles", + "source-layer": "trails", + "filter": [ + "in", "highway", "path", "bridleway", "footway" + ], + "paint": { + "line-color": "#FFD700", + "line-width": [ + "interpolate", ["linear"], ["zoom"], + 8, 1, + 12, 1.5, + 16, 3 + ], + "line-opacity": 0.85, + "line-dasharray": [3, 2] + }, + "layout": { + "line-cap": "butt", + "line-join": "round" + } + }, + { + "id": "poi-circles", + "type": "circle", + "source": "trails-tiles", + "source-layer": "poi", + "paint": { + "circle-radius": [ + "interpolate", ["linear"], ["zoom"], + 8, 3, + 12, 6, + 16, 10 + ], + "circle-color": [ + "match", ["get", "poi_type"], + "natural=peak", "#ff4444", + "natural=water", "#4488ff", + "tourism=viewpoint", "#44ff88", + "historic=ruins", "#cc88ff", + "natural=cave_entrance", "#ffaa00", + "ford=yes", "#00ccff", + "#ffffff" + ], + "circle-stroke-color": "#1a1a2e", + "circle-stroke-width": 1.5, + "circle-opacity": 0.9 + } + }, + { + "id": "poi-labels", + "type": "symbol", + "source": "trails-tiles", + "source-layer": "poi", + "minzoom": 11, + "layout": { + "text-field": ["get", "name"], + "text-font": ["Open Sans Regular"], + "text-size": 11, + "text-offset": [0, 1.2], + "text-anchor": "top", + "text-optional": true + }, + "paint": { + "text-color": "#ffffff", + "text-halo-color": "#1a1a2e", + "text-halo-width": 1.5 + } + } + ] +} diff --git a/tasks/enduro-trails/scripts/download.sh b/tasks/enduro-trails/scripts/download.sh new file mode 100755 index 0000000..c2909b9 --- /dev/null +++ b/tasks/enduro-trails/scripts/download.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# download.sh — скачивает и подготавливает OSM PBF данные для Enduro Trails +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DATA_DIR="$SCRIPT_DIR/../data" + +mkdir -p "$DATA_DIR" +cd "$DATA_DIR" + +echo "==> Скачиваем ЦФО..." +wget -c "https://download.geofabrik.de/russia/centralfederal.ru-latest.osm.pbf" \ + -O centralfederal.ru-latest.osm.pbf + +echo "==> Скачиваем Поволжье (Чувашия и др.)..." +wget -c "https://download.geofabrik.de/russia/volga.osm.pbf" \ + -O volga.osm.pbf + +echo "==> Объединяем файлы..." +osmium merge centralfederal.ru-latest.osm.pbf volga.osm.pbf \ + -o merged.osm.pbf --overwrite + +echo "==> Фильтруем по BBOX (ЦФО + Чувашия)..." +# bbox: west,south,east,north = 30.0,51.0,48.0,59.0 +osmium extract \ + --bbox=30.0,51.0,48.0,59.0 \ + merged.osm.pbf \ + -o region.osm.pbf \ + --overwrite + +echo "==> Готово! Файлы в $DATA_DIR:" +ls -lh "$DATA_DIR"/*.pbf diff --git a/tasks/enduro-trails/scripts/parse.py b/tasks/enduro-trails/scripts/parse.py new file mode 100644 index 0000000..8481c3d --- /dev/null +++ b/tasks/enduro-trails/scripts/parse.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +""" +parse.py — парсинг OSM PBF → Spatialite для Enduro Trails +Читает region.osm.pbf, сохраняет trails и POI в centralfederal.sqlite +""" + +import os +import sys +import json +import math +import sqlite3 +import argparse + +try: + import osmium +except ImportError: + print("ERROR: python-osmium не установлен. pip install python-osmium") + sys.exit(1) + +try: + # pysqlite3-binary предоставляет sqlite3 с поддержкой расширений + import pysqlite3 as sqlite3_ext + HAS_PYSQLITE3 = True +except ImportError: + HAS_PYSQLITE3 = False + sqlite3_ext = sqlite3 + +from shapely.geometry import LineString, Point +from shapely import wkb as shapely_wkb + +# ─── Константы ──────────────────────────────────────────────────────────────── + +HIGHWAY_TYPES = {"track", "path", "bridleway", "cycleway", "footway"} + +POI_FILTERS = { + "natural": {"water", "peak", "cave_entrance"}, + "tourism": {"viewpoint"}, + "historic": {"ruins"}, + "ford": {"yes"}, +} + +EARTH_RADIUS_M = 6_371_000.0 + + +# ─── Утилиты ────────────────────────────────────────────────────────────────── + +def haversine_length(coords): + """Длина ломаной в метрах по списку (lon, lat) пар.""" + total = 0.0 + for i in range(len(coords) - 1): + lon1, lat1 = math.radians(coords[i][0]), math.radians(coords[i][1]) + lon2, lat2 = math.radians(coords[i+1][0]), math.radians(coords[i+1][1]) + dlat = lat2 - lat1 + dlon = lon2 - lon1 + a = math.sin(dlat/2)**2 + math.cos(lat1)*math.cos(lat2)*math.sin(dlon/2)**2 + total += 2 * EARTH_RADIUS_M * math.asin(math.sqrt(a)) + return total + + +def geom_to_wkb_hex(geom): + """Shapely geometry → WKB hex string для Spatialite.""" + return shapely_wkb.dumps(geom, hex=True) + + +# ─── OSM Handlers ───────────────────────────────────────────────────────────── + +class TrailHandler(osmium.SimpleHandler): + """Собирает highway=track/path/... из OSM.""" + + def __init__(self): + super().__init__() + self.trails = [] + + def way(self, w): + tags = w.tags + hw = tags.get("highway", "") + if hw not in HIGHWAY_TYPES: + return + + try: + coords = [(n.lon, n.lat) for n in w.nodes if n.location.valid()] + except Exception: + return + + if len(coords) < 2: + return + + length_m = haversine_length(coords) + geom = LineString(coords) + + extra_tags = {} + for tag in w.tags: + extra_tags[tag.k] = tag.v + + self.trails.append({ + "osm_id": w.id, + "highway_type": hw, + "track_type": tags.get("tracktype", None), + "surface": tags.get("surface", None), + "name": tags.get("name", None), + "length_m": length_m, + "mtb_scale": tags.get("mtb:scale", None), + "visibility": tags.get("trail_visibility", None), + "smoothness": tags.get("smoothness", None), + "access": tags.get("access", None), + "tags": json.dumps(extra_tags, ensure_ascii=False), + "geom_wkb": geom_to_wkb_hex(geom), + }) + + +class POIHandler(osmium.SimpleHandler): + """Собирает POI: вершины, родники, смотровые и т.д.""" + + def __init__(self): + super().__init__() + self.pois = [] + + def _check_tags(self, tags): + """Возвращает poi_type если тег совпадает с фильтром.""" + for key, values in POI_FILTERS.items(): + val = tags.get(key, "") + if val in values: + return f"{key}={val}" + return None + + def node(self, n): + poi_type = self._check_tags(n.tags) + if not poi_type: + return + if not n.location.valid(): + return + + geom = Point(n.location.lon, n.location.lat) + self.pois.append({ + "osm_id": n.id, + "poi_type": poi_type, + "name": n.tags.get("name", None), + "geom_wkb": geom_to_wkb_hex(geom), + }) + + def way(self, w): + """Для водоёмов-полигонов берём центроид.""" + poi_type = self._check_tags(w.tags) + if not poi_type: + return + + try: + coords = [(n.lon, n.lat) for n in w.nodes if n.location.valid()] + except Exception: + return + + if len(coords) < 2: + return + + geom = LineString(coords).centroid + self.pois.append({ + "osm_id": w.id, + "poi_type": poi_type, + "name": w.tags.get("name", None), + "geom_wkb": geom_to_wkb_hex(geom), + }) + + +# ─── Spatialite ─────────────────────────────────────────────────────────────── + +def open_spatialite(db_path): + """Открывает соединение с Spatialite, загружает расширение.""" + conn = sqlite3_ext.connect(db_path) + conn.enable_load_extension(True) + + # Пробуем разные пути к mod_spatialite + spatialite_paths = [ + "mod_spatialite", + "/usr/lib/x86_64-linux-gnu/mod_spatialite.so", + "/usr/lib/mod_spatialite.so", + "/usr/local/lib/mod_spatialite.so", + ] + loaded = False + for path in spatialite_paths: + try: + conn.load_extension(path) + loaded = True + print(f" Spatialite загружен: {path}") + break + except Exception: + continue + + if not loaded: + print("WARNING: mod_spatialite не найден — геометрия будет храниться как WKB blob без пространственных индексов") + + return conn, loaded + + +def init_db(conn, has_spatialite): + """Создаёт таблицы и индексы.""" + cur = conn.cursor() + + if has_spatialite: + cur.execute("SELECT InitSpatialMetaData(1)") + + cur.executescript(""" + DROP TABLE IF EXISTS trails; + CREATE TABLE trails ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + osm_id INTEGER NOT NULL, + highway_type TEXT, + track_type TEXT, + surface TEXT, + name TEXT, + length_m REAL, + mtb_scale TEXT, + visibility TEXT, + smoothness TEXT, + access TEXT, + tags TEXT, + geom GEOMETRY + ); + + DROP TABLE IF EXISTS poi; + CREATE TABLE poi ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + osm_id INTEGER NOT NULL, + poi_type TEXT, + name TEXT, + geom GEOMETRY + ); + """) + + if has_spatialite: + try: + cur.execute("SELECT AddGeometryColumn('trails', 'geom', 4326, 'LINESTRING', 'XY')") + except Exception: + pass # колонка уже добавлена через CREATE TABLE + try: + cur.execute("SELECT AddGeometryColumn('poi', 'geom', 4326, 'POINT', 'XY')") + except Exception: + pass + + cur.executescript(""" + CREATE INDEX IF NOT EXISTS idx_trails_highway ON trails(highway_type); + CREATE INDEX IF NOT EXISTS idx_trails_surface ON trails(surface); + CREATE INDEX IF NOT EXISTS idx_poi_type ON poi(poi_type); + """) + + conn.commit() + + +def insert_trails(conn, trails, has_spatialite): + cur = conn.cursor() + batch = [] + for t in trails: + if has_spatialite: + geom_expr = f"GeomFromWKB(x'{t['geom_wkb']}', 4326)" + else: + geom_expr = f"x'{t['geom_wkb']}'" + + batch.append(( + t["osm_id"], t["highway_type"], t["track_type"], t["surface"], + t["name"], t["length_m"], t["mtb_scale"], t["visibility"], + t["smoothness"], t["access"], t["tags"], + )) + + # Вставляем батчами по 1000 + BATCH = 1000 + for i in range(0, len(trails), BATCH): + chunk = trails[i:i+BATCH] + for t in chunk: + cur.execute(""" + INSERT INTO trails + (osm_id, highway_type, track_type, surface, name, length_m, + mtb_scale, visibility, smoothness, access, tags, geom) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?) + """, ( + t["osm_id"], t["highway_type"], t["track_type"], t["surface"], + t["name"], t["length_m"], t["mtb_scale"], t["visibility"], + t["smoothness"], t["access"], t["tags"], + bytes.fromhex(t["geom_wkb"]), + )) + conn.commit() + print(f" trails: вставлено {min(i+BATCH, len(trails))}/{len(trails)}") + + +def insert_pois(conn, pois): + cur = conn.cursor() + BATCH = 1000 + for i in range(0, len(pois), BATCH): + chunk = pois[i:i+BATCH] + for p in chunk: + cur.execute(""" + INSERT INTO poi (osm_id, poi_type, name, geom) + VALUES (?,?,?,?) + """, ( + p["osm_id"], p["poi_type"], p["name"], + bytes.fromhex(p["geom_wkb"]), + )) + conn.commit() + print(f" poi: вставлено {min(i+BATCH, len(pois))}/{len(pois)}") + + +def create_spatial_indexes(conn, has_spatialite): + if not has_spatialite: + return + cur = conn.cursor() + try: + cur.execute("SELECT CreateSpatialIndex('trails', 'geom')") + conn.commit() + print(" Пространственный индекс trails создан") + except Exception as e: + print(f" WARNING: индекс trails: {e}") + try: + cur.execute("SELECT CreateSpatialIndex('poi', 'geom')") + conn.commit() + print(" Пространственный индекс poi создан") + except Exception as e: + print(f" WARNING: индекс poi: {e}") + + +# ─── Main ───────────────────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser(description="Парсинг OSM PBF → Spatialite") + parser.add_argument( + "--pbf", + default=os.path.join(os.path.dirname(__file__), "../data/region.osm.pbf"), + help="Путь к PBF файлу", + ) + parser.add_argument( + "--db", + default=os.path.join(os.path.dirname(__file__), "../data/centralfederal.sqlite"), + help="Путь к выходному SQLite/Spatialite файлу", + ) + args = parser.parse_args() + + pbf_path = os.path.abspath(args.pbf) + db_path = os.path.abspath(args.db) + + if not os.path.exists(pbf_path): + print(f"ERROR: PBF файл не найден: {pbf_path}") + print("Сначала запустите scripts/download.sh") + sys.exit(1) + + print(f"==> Читаем PBF: {pbf_path}") + + print(" Парсим дороги...") + trail_handler = TrailHandler() + trail_handler.apply_file(pbf_path, locations=True) + print(f" Найдено дорог: {len(trail_handler.trails)}") + + print(" Парсим POI...") + poi_handler = POIHandler() + poi_handler.apply_file(pbf_path, locations=True) + print(f" Найдено POI: {len(poi_handler.pois)}") + + print(f"==> Открываем БД: {db_path}") + os.makedirs(os.path.dirname(db_path), exist_ok=True) + conn, has_spatialite = open_spatialite(db_path) + + print("==> Инициализируем схему...") + init_db(conn, has_spatialite) + + print("==> Вставляем дороги...") + insert_trails(conn, trail_handler.trails, has_spatialite) + + print("==> Вставляем POI...") + insert_pois(conn, poi_handler.pois) + + print("==> Создаём пространственные индексы...") + create_spatial_indexes(conn, has_spatialite) + + conn.close() + print(f"\n✓ Готово! БД сохранена: {db_path}") + + +if __name__ == "__main__": + main()