add backend server setup with environment configuration, tile caching, and bus stop fetching functionality

2026-03-31 16:37:34 +01:00
parent 618f3dd3ed
commit 49fc04591b
27 changed files with 1536 additions and 374 deletions
@@ -0,0 +1,2 @@
+MAPBOX_ACCESS_TOKEN=pk.eyJ1IjoiaW1iZW5qaW5ldCIsImEiOiJjbW01azQ1bTcwODJ5MnBzOG9tMTUxdGRoIn0.z22taz_5I_8D5GuDlser_Q
+PORT=8080
@@ -0,0 +1,2 @@
+MAPBOX_ACCESS_TOKEN=pk.your_token_here
+PORT=8080
@@ -0,0 +1,52 @@
+import 'dart:io';
+
+import 'package:dotenv/dotenv.dart';
+import 'package:shelf/shelf.dart' show Middleware, Response;
+import 'package:rra_backend/rra_backend.dart';
+import 'package:shelf/shelf.dart';
+import 'package:shelf/shelf_io.dart' as io;
+
+Middleware _logger() => (handler) => (request) async {
+  final sw = Stopwatch()..start();
+  final response = await handler(request);
+  sw.stop();
+  final ms = sw.elapsedMilliseconds;
+  print('${request.method} ${response.statusCode} ${ms}ms ${request.url.path}');
+  return response;
+};
+
+Middleware _cors() => (handler) => (request) async {
+  if (request.method == 'OPTIONS') {
+    return Response.ok('', headers: _corsHeaders);
+  }
+  final response = await handler(request);
+  return response.change(headers: _corsHeaders);
+};
+
+const _corsHeaders = {
+  'Access-Control-Allow-Origin': '*',
+  'Access-Control-Allow-Methods': 'GET, OPTIONS',
+  'Access-Control-Allow-Headers': 'Origin, Content-Type',
+};
+
+void main() async {
+  final env = DotEnv(includePlatformEnvironment: true)..load();
+
+  final mapboxToken = env['MAPBOX_ACCESS_TOKEN'];
+  if (mapboxToken == null || mapboxToken.isEmpty) {
+    stderr.writeln('ERROR: MAPBOX_ACCESS_TOKEN not set');
+    exit(1);
+  }
+
+  final port = int.tryParse(env['PORT'] ?? '8080') ?? 8080;
+
+  await TileCache.init();
+
+  final handler = const Pipeline()
+      .addMiddleware(_logger())
+      .addMiddleware(_cors())
+      .addHandler(buildRouter(mapboxToken).call);
+
+  final server = await io.serve(handler, InternetAddress.anyIPv4, port);
+  print('rra_backend listening on :${server.port}');
+}
@@ -0,0 +1,2 @@
+export 'src/cache/tile_cache.dart';
+export 'src/router.dart';
@@ -0,0 +1,75 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:sqlite3/sqlite3.dart';
+
+// disk cache for raw .pbf tile bytes
+// stale after 14 days, max 50k tiles before oldest get pruned
+class TileCache {
+  TileCache._();
+
+  static const _staleMs = 14 * 24 * 60 * 60 * 1000;
+  static const _maxTiles = 50000;
+
+  static late final Database _db;
+
+  static Future<void> init({String? path}) async {
+    final dbPath = path ?? '${Directory.current.path}/tile_cache.db';
+    _db = sqlite3.open(dbPath);
+
+    _db.execute('''
+      CREATE TABLE IF NOT EXISTS tiles (
+        key   TEXT PRIMARY KEY,
+        ts    INTEGER NOT NULL,
+        bytes BLOB    NOT NULL
+      )
+    ''');
+
+    _db.execute('CREATE INDEX IF NOT EXISTS idx_tiles_ts ON tiles(ts)');
+
+    // WAL mode — much better for concurrent reads
+    _db.execute('PRAGMA journal_mode=WAL');
+    _db.execute('PRAGMA synchronous=NORMAL');
+  }
+
+  static Uint8List? get(String key) {
+    final now = DateTime.now().millisecondsSinceEpoch;
+    final result = _db.select(
+      'SELECT bytes, ts FROM tiles WHERE key = ?',
+      [key],
+    );
+
+    if (result.isEmpty) return null;
+
+    final row = result.first;
+    final ts = row['ts'] as int;
+    if (now - ts > _staleMs) {
+      _db.execute('DELETE FROM tiles WHERE key = ?', [key]);
+      return null;
+    }
+
+    return row['bytes'] as Uint8List;
+  }
+
+  static void put(String key, Uint8List bytes) {
+    final now = DateTime.now().millisecondsSinceEpoch;
+    _db.execute(
+      'INSERT OR REPLACE INTO tiles (key, ts, bytes) VALUES (?, ?, ?)',
+      [key, now, bytes],
+    );
+    _pruneIfNeeded();
+  }
+
+  static void _pruneIfNeeded() {
+    final count = (_db.select('SELECT COUNT(*) as c FROM tiles').first['c'] as int);
+    if (count <= _maxTiles) return;
+
+    final excess = count - _maxTiles;
+
+    _db.execute('''
+      DELETE FROM tiles WHERE key IN (
+        SELECT key FROM tiles ORDER BY ts ASC LIMIT ?
+      )
+    ''', [excess]);
+  }
+}
@@ -0,0 +1,116 @@
+import 'dart:async';
+import 'dart:convert';
+import 'dart:math' as math;
+
+import 'package:http/http.dart' as http;
+import 'package:shelf/shelf.dart';
+
+final Map<String, _BusStopEntry> _cache = {};
+
+const _ttlMs = 5 * 60 * 1000;
+
+class _BusStopEntry {
+  _BusStopEntry(this.body, this.ts);
+  final String body;
+  final int ts;
+}
+
+class BusStopsHandler {
+
+  Future<Response> handle(Request req) async {
+    final params = req.url.queryParameters;
+    final minLat = double.tryParse(params['minLat'] ?? '');
+    final minLon = double.tryParse(params['minLon'] ?? '');
+    final maxLat = double.tryParse(params['maxLat'] ?? '');
+    final maxLon = double.tryParse(params['maxLon'] ?? '');
+
+    if (minLat == null || minLon == null || maxLat == null || maxLon == null) {
+      return Response.badRequest(body: 'minLat, minLon, maxLat, maxLon required');
+    }
+
+    String r2(double v) => ((v * 100).round() / 100).toString();
+    final key = '${r2(minLat)},${r2(minLon)},${r2(maxLat)},${r2(maxLon)}';
+
+    final now = DateTime.now().millisecondsSinceEpoch;
+    final cached = _cache[key];
+    final isStale = cached == null || now - cached.ts >= _ttlMs;
+
+    final controller = StreamController<List<int>>();
+
+    () async {
+      // emit cache immediately if we have it
+      if (cached != null) {
+        controller.add(_sseBytes(cached.body));
+      }
+
+      // always fetch fresh from TfL if stale
+      if (isStale) {
+        final fresh = await _fetchFromTfl(minLat, minLon, maxLat, maxLon);
+        if (fresh != null) {
+          _cache[key] = _BusStopEntry(fresh, DateTime.now().millisecondsSinceEpoch);
+          if (_cache.length > 2000) {
+            _cache.removeWhere((_, v) =>
+                DateTime.now().millisecondsSinceEpoch - v.ts > _ttlMs);
+          }
+          // only emit if different from what we already sent
+          if (fresh != cached?.body) {
+            controller.add(_sseBytes(fresh));
+          }
+        }
+      }
+
+      await controller.close();
+    }();
+
+    return Response.ok(
+      controller.stream,
+      headers: {
+        'Content-Type': 'text/event-stream',
+        'Cache-Control': 'no-cache',
+        'X-Accel-Buffering': 'no',
+      },
+    );
+  }
+
+  static List<int> _sseBytes(String json) {
+    return utf8.encode('data: $json\n\n');
+  }
+
+  static Future<String?> _fetchFromTfl(
+    double minLat,
+    double minLon,
+    double maxLat,
+    double maxLon,
+  ) async {
+    final centerLat = (minLat + maxLat) / 2;
+    final centerLon = (minLon + maxLon) / 2;
+    final radius = _haversineM(minLat, minLon, maxLat, maxLon) / 2;
+    final r = radius.clamp(200, 1600).toInt();
+
+    final uri = Uri.parse(
+      'https://api.tfl.gov.uk/StopPoint'
+      '?stopTypes=NaptanPublicBusCoachTram'
+      '&lat=$centerLat&lon=$centerLon&radius=$r&modes=bus',
+    );
+
+    try {
+      final upstream = await http.get(uri, headers: {'Accept': 'application/json'});
+      if (upstream.statusCode != 200) return null;
+      return upstream.body;
+    } catch (_) {
+      return null;
+    }
+  }
+
+  static double _haversineM(double lat1, double lon1, double lat2, double lon2) {
+    const r = 6371000.0;
+    final dLat = (lat2 - lat1) * math.pi / 180;
+    final dLon = (lon2 - lon1) * math.pi / 180;
+    final a = math.sin(dLat / 2) * math.sin(dLat / 2) +
+        math.cos(lat1 * math.pi / 180) *
+            math.cos(lat2 * math.pi / 180) *
+            math.sin(dLon / 2) *
+            math.sin(dLon / 2);
+    return 2 * r * math.atan2(math.sqrt(a), math.sqrt(1 - a));
+  }
+}
@@ -0,0 +1,60 @@
+import 'package:http/http.dart' as http;
+import 'package:shelf/shelf.dart';
+
+// cache routes by waypoints+profile — routes dont change often
+final Map<String, _RouteEntry> _routeCache = {};
+const _routeTtlMs = 60 * 60 * 1000; // 1 hour
+
+class _RouteEntry {
+  _RouteEntry(this.body, this.ts);
+  final String body;
+  final int ts;
+}
+
+class RouteHandler {
+
+  static const _osrm = 'https://router.project-osrm.org';
+
+  Future<Response> handle(Request req) async {
+    final params = req.url.queryParameters;
+    final waypoints = params['waypoints'];
+    final profile = params['profile'] ?? 'driving';
+
+    if (waypoints == null || waypoints.isEmpty) {
+      return Response.badRequest(body: 'waypoints required');
+    }
+
+    final cacheKey = '$profile|$waypoints';
+    final now = DateTime.now().millisecondsSinceEpoch;
+    final cached = _routeCache[cacheKey];
+
+    if (cached != null && now - cached.ts < _routeTtlMs) {
+      return Response.ok(
+        cached.body,
+        headers: {'Content-Type': 'application/json', 'X-Cache': 'HIT'},
+      );
+    }
+
+    final uri = Uri.parse(
+      '$_osrm/route/v1/$profile/$waypoints'
+      '?overview=full&geometries=geojson',
+    );
+
+    final upstream = await http.get(uri);
+
+    if (upstream.statusCode != 200) {
+      return Response(upstream.statusCode, body: 'OSRM error');
+    }
+
+    _routeCache[cacheKey] = _RouteEntry(upstream.body, now);
+
+    if (_routeCache.length > 500) {
+      _routeCache.removeWhere((_, v) => now - v.ts > _routeTtlMs);
+    }
+
+    return Response.ok(
+      upstream.body,
+      headers: {'Content-Type': 'application/json', 'X-Cache': 'MISS'},
+    );
+  }
+}
@@ -0,0 +1,60 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:http/http.dart' as http;
+import 'package:shelf/shelf.dart';
+import 'package:shelf_router/shelf_router.dart';
+
+import '../cache/tile_cache.dart';
+
+class TileHandler {
+  TileHandler({required this.mapboxToken});
+
+  final String mapboxToken;
+
+  static const _upstreamTemplate =
+      'https://api.mapbox.com/v4/mapbox.mapbox-streets-v8/{z}/{x}/{y}.vector.pbf?access_token={token}';
+
+  Future<Response> handle(Request req) async {
+    final z = req.params['z']!;
+    final x = req.params['x']!;
+    final y = req.params['y']!;
+
+    final cacheKey = 'tile/$z/$x/$y';
+
+    final cached = TileCache.get(cacheKey);
+    if (cached != null) {
+      return _tileResponse(cached, fromCache: true);
+    }
+
+    final url = _upstreamTemplate
+        .replaceAll('{z}', z)
+        .replaceAll('{x}', x)
+        .replaceAll('{y}', y)
+        .replaceAll('{token}', mapboxToken);
+
+    final upstream = await http.get(Uri.parse(url));
+
+    if (upstream.statusCode != 200) {
+      return Response(upstream.statusCode, body: 'upstream error');
+    }
+
+    final bytes = upstream.bodyBytes;
+    TileCache.put(cacheKey, bytes);
+
+    return _tileResponse(bytes, fromCache: false);
+  }
+
+  static Response _tileResponse(Uint8List bytes, {required bool fromCache}) {
+    final compressed = gzip.encode(bytes);
+    return Response.ok(
+      compressed,
+      headers: {
+        'Content-Type': 'application/x-protobuf',
+        'Content-Encoding': 'gzip',
+        'Cache-Control': 'public, max-age=86400',
+        'X-Cache': fromCache ? 'HIT' : 'MISS',
+      },
+    );
+  }
+}
@@ -0,0 +1,24 @@
+import 'package:shelf_router/shelf_router.dart';
+
+import 'handlers/bus_stops_handler.dart';
+import 'handlers/route_handler.dart';
+import 'handlers/tile_handler.dart';
+
+Router buildRouter(String mapboxToken) {
+  final router = Router();
+
+  final tileHandler = TileHandler(mapboxToken: mapboxToken);
+  final busStopsHandler = BusStopsHandler();
+  final routeHandler = RouteHandler();
+
+  // GET /tiles/{z}/{x}/{y}
+  router.get('/tiles/<z>/<x>/<y>', tileHandler.handle);
+
+  // GET /stops?lat=51.5&lon=-0.1&radius=600
+  router.get('/stops', busStopsHandler.handle);
+
+  // GET /route?waypoints=lon,lat;lon,lat&profile=driving
+  router.get('/route', routeHandler.handle);
+
+  return router;
+}
@@ -0,0 +1,173 @@
+# Generated by pub
+# See https://dart.dev/tools/pub/glossary#lockfile
+packages:
+  args:
+    dependency: transitive
+    description:
+      name: args
+      sha256: d0481093c50b1da8910eb0bb301626d4d8eb7284aa739614d2b394ee09e3ea04
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.7.0"
+  async:
+    dependency: transitive
+    description:
+      name: async
+      sha256: e2eb0491ba5ddb6177742d2da23904574082139b07c1e33b8503b9f46f3e1a37
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.13.1"
+  collection:
+    dependency: transitive
+    description:
+      name: collection
+      sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.19.1"
+  dotenv:
+    dependency: "direct main"
+    description:
+      name: dotenv
+      sha256: "379e64b6fc82d3df29461d349a1796ecd2c436c480d4653f3af6872eccbc90e1"
+      url: "https://pub.dev"
+    source: hosted
+    version: "4.2.0"
+  ffi:
+    dependency: transitive
+    description:
+      name: ffi
+      sha256: "6d7fd89431262d8f3125e81b50d3847a091d846eafcd4fdb88dd06f36d705a45"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.2.0"
+  http:
+    dependency: "direct main"
+    description:
+      name: http
+      sha256: "87721a4a50b19c7f1d49001e51409bddc46303966ce89a65af4f4e6004896412"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.6.0"
+  http_methods:
+    dependency: transitive
+    description:
+      name: http_methods
+      sha256: "6bccce8f1ec7b5d701e7921dca35e202d425b57e317ba1a37f2638590e29e566"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.1"
+  http_parser:
+    dependency: transitive
+    description:
+      name: http_parser
+      sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571"
+      url: "https://pub.dev"
+    source: hosted
+    version: "4.1.2"
+  lints:
+    dependency: "direct dev"
+    description:
+      name: lints
+      sha256: cbf8d4b858bb0134ef3ef87841abdf8d63bfc255c266b7bf6b39daa1085c4290
+      url: "https://pub.dev"
+    source: hosted
+    version: "3.0.0"
+  meta:
+    dependency: transitive
+    description:
+      name: meta
+      sha256: df0c643f44ad098eb37988027a8e2b2b5a031fd3977f06bbfd3a76637e8df739
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.18.2"
+  path:
+    dependency: transitive
+    description:
+      name: path
+      sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.9.1"
+  shelf:
+    dependency: "direct main"
+    description:
+      name: shelf
+      sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.4.2"
+  shelf_router:
+    dependency: "direct main"
+    description:
+      name: shelf_router
+      sha256: f5e5d492440a7fb165fe1e2e1a623f31f734d3370900070b2b1e0d0428d59864
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.4"
+  source_span:
+    dependency: transitive
+    description:
+      name: source_span
+      sha256: "56a02f1f4cd1a2d96303c0144c93bd6d909eea6bee6bf5a0e0b685edbd4c47ab"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.10.2"
+  sqlite3:
+    dependency: "direct main"
+    description:
+      name: sqlite3
+      sha256: "3145bd74dcdb4fd6f5c6dda4d4e4490a8087d7f286a14dee5d37087290f0f8a2"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.9.4"
+  stack_trace:
+    dependency: transitive
+    description:
+      name: stack_trace
+      sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.12.1"
+  stream_channel:
+    dependency: transitive
+    description:
+      name: stream_channel
+      sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d"
+      url: "https://pub.dev"
+    source: hosted
+    version: "2.1.4"
+  string_scanner:
+    dependency: transitive
+    description:
+      name: string_scanner
+      sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.4.1"
+  term_glyph:
+    dependency: transitive
+    description:
+      name: term_glyph
+      sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.2.2"
+  typed_data:
+    dependency: transitive
+    description:
+      name: typed_data
+      sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.4.0"
+  web:
+    dependency: transitive
+    description:
+      name: web
+      sha256: "868d88a33d8a87b18ffc05f9f030ba328ffefba92d6c127917a2ba740f9cfe4a"
+      url: "https://pub.dev"
+    source: hosted
+    version: "1.1.1"
+sdks:
+  dart: ">=3.7.0 <4.0.0"
@@ -0,0 +1,17 @@
+name: rra_backend
+description: Backend proxy/cache server for rra_app
+version: 1.0.0
+publish_to: none
+
+environment:
+  sdk: '>=3.0.0 <4.0.0'
+
+dependencies:
+  shelf: ^1.4.0
+  shelf_router: ^1.1.0
+  http: ^1.2.0
+  sqlite3: ^2.4.0
+  dotenv: ^4.0.0
+
+dev_dependencies:
+  lints: ^3.0.0