feat(live): worldwide providers as data — 73 feeds, 14 countries, licence-checked

packages/backend/src/config/live.ts

@@ -98,6 +98,14 @@ export const LIVE_POLICY: LivePolicy = {
 export interface LiveConfig extends LivePolicy {
   /** Source ids, as listed, de-duplicated. Checked against the catalogue by the caller. */
   sources: readonly string[];
+  /**
+   * `LIVE_SOURCES=all`: every catalogue source — each one licence-checked —
+   * whose key, when it needs one, is configured. A keyed source without its
+   * key is then skipped quietly rather than warned about one by one.
+   */
+  all: boolean;
+  /** Ids listed by name beside `all` (`all,adsb-lol`): a source still `awaiting` a permission joins only this way. */
+  named: readonly string[];
   settings: Readonly<Record<string, LiveSourceSettings>>;
   /**
    * Valkey shared by the API tasks (`live/cluster.ts`): one task polls each
@@ -149,7 +157,10 @@ export function parseLiveConfig(
   }
 
   const settings: Record<string, LiveSourceSettings> = {};
-  const listed = result.success ? result.data : [];
+  const parsed = result.success ? result.data : [];
+  // `all`: the whole catalogue (only meaningful against it).
+  const all = parsed.includes('all') && knownSourceIds.length > 0;
+  const listed = all ? [...knownSourceIds] : parsed;
   for (const id of listed) {
     if (knownSourceIds.length > 0 && !knownSourceIds.includes(id)) {
       problems.push(
@@ -191,7 +202,8 @@ export function parseLiveConfig(
   if (problems.length > 0) {
     throw new Error(`Invalid live configuration:\n${problems.join('\n')}`);
   }
-  return { ...LIVE_POLICY, sources: listed, settings, valkeyUrl, valkeyUrlProblem };
+  const named = parsed.filter((id) => id !== 'all');
+  return { ...LIVE_POLICY, sources: listed, all, named, settings, valkeyUrl, valkeyUrlProblem };
 }
 
 /** The one parse for this process, against the catalogue's ids. */

packages/backend/src/live/__tests__/areas.test.ts

@@ -5,6 +5,7 @@
 import { describe, expect, it } from 'bun:test';
 import { liveTileOf } from '@goway/contracts';
 import { areasOf, sourceAreas } from '../areas';
+import { LIVE_SOURCES_UNREAD } from '../areas.generated';
 import { LIVE_SOURCES } from '../catalogue';
 import { LiveHub } from '../hub';
 import type { LiveSourceDefinition } from '../types';
@@ -35,7 +36,11 @@ describe('areas from stops', () => {
 
   it('exist for every source whose stops are published, and are its coverage', () => {
     for (const source of LIVE_SOURCES) {
-      const stops = Boolean(source.staticGtfs) || source.kind === 'bikeshare';
+      // Its stops are published, and were read when `live:areas` last ran
+      // (an operator's server down that day is listed, not hidden).
+      const stops =
+        (Boolean(source.staticGtfs) || source.kind === 'bikeshare') &&
+        !LIVE_SOURCES_UNREAD.includes(source.id);
       const derived = sourceAreas(source.id);
       expect({ id: source.id, derived: Boolean(derived) }).toEqual({
         id: source.id,

packages/backend/src/live/__tests__/cluster.test.ts

@@ -70,6 +70,7 @@ function task(
     completeness: 'network',
     defaultUrl: 'https://example.org/feed',
     requiresKey: false,
+    licenceCheck: { verifiedOn: '2026-10-12', source: 'https://example.org', quote: 'test' },
     pollIntervalSeconds: 10,
     staleAfterSeconds: 60,
     dropAfterSeconds: 120,

packages/backend/src/live/__tests__/combined.test.ts

@@ -105,3 +105,36 @@ describe('a static GTFS without shapes', () => {
     expect(trip?.shapeId).toBeNull();
   });
 });
+
+describe('a route type names the mode', () => {
+  it('basic and extended GTFS types', async () => {
+    const { modeOfRouteType } = await import('../staticGtfs');
+    expect(['0', '1', '2', '3', '4', '6', '11', '12'].map(modeOfRouteType)).toEqual([
+      'tram',
+      'subway',
+      'rail',
+      'bus',
+      'ferry',
+      'cable',
+      'bus',
+      'subway',
+    ]);
+    expect(['109', '401', '700', '900', '1000', '1400', 'x', '-1'].map(modeOfRouteType)).toEqual([
+      'rail',
+      'subway',
+      'bus',
+      'tram',
+      'ferry',
+      'cable',
+      null,
+      null,
+    ]);
+    // Entur's airport routes are 1102, "Domestic Air Service": not a boat.
+    expect(['1004', '1102', '1200', '1501'].map(modeOfRouteType)).toEqual([
+      'ferry',
+      null,
+      'ferry',
+      null,
+    ]);
+  });
+});

packages/backend/src/live/__tests__/hub.test.ts

@@ -18,7 +18,7 @@ import {
   liveTilesFor,
   liveTilesOf,
 } from '@goway/contracts';
-import { LiveHub, MAX_BACKOFF_MS } from '../hub';
+import { LiveHub, MAX_BACKOFF_MS, STATIC_GTFS_IDLE_MS } from '../hub';
 import { contains, LiveTileIndex } from '../tileIndex';
 import type { LiveFetchContext, LivePollResult, LiveSourceDefinition } from '../types';
 import { LiveUpstreamError } from '../types';
@@ -208,6 +208,7 @@ function harness(overrides: Partial<LiveSourceDefinition> = {}): Harness {
     completeness: 'network',
     defaultUrl: 'https://example.org/feed',
     requiresKey: false,
+    licenceCheck: { verifiedOn: '2026-10-12', source: 'https://example.org', quote: 'test' },
     pollIntervalSeconds: 10,
     staleAfterSeconds: 60,
     dropAfterSeconds: 120,
@@ -484,6 +485,136 @@ describe('the hub', () => {
     expect(h.hub.shape('test-bus', 'nope')).toBeNull();
   });
 
+  it('lets a static GTFS go once nobody has looked for a while, and reads it again when they do', async () => {
+    const h = harness({
+      staticGtfs: {
+        routes: 'https://gtfs.example/routes.txt',
+        trips: 'https://gtfs.example/trips.txt',
+        shapes: 'https://gtfs.example/shapes.txt',
+      },
+    });
+    const room = { tile: BARCELONA_TILE, kind: 'transit' as const };
+    h.hub.watch('a', [room]);
+    await h.hub.tick();
+    expect(h.hub.shape('test-bus', 'line')).not.toBeNull();
+    h.hub.unwatch('a');
+    h.clock.now += STATIC_GTFS_IDLE_MS - 1_000;
+    await h.hub.tick();
+    await h.hub.tick();
+    expect(h.hub.shape('test-bus', 'line')).not.toBeNull();
+    h.clock.now += STATIC_GTFS_IDLE_MS;
+    await h.hub.tick();
+    expect(h.hub.shape('test-bus', 'line')).toBeNull();
+    h.hub.watch('a', [room]);
+    await h.hub.tick();
+    expect(h.hub.shape('test-bus', 'line')).not.toBeNull();
+    h.hub.stop();
+  });
+
+  it("reads a big archive's routes by range, unencoded, and names each vehicle's line and mode from it", async () => {
+    // A stored (uncompressed) zip of one routes.txt and a large stop_times.txt.
+    const files: [string, string][] = [
+      ['routes.txt', 'route_id,route_short_name,route_type\n100001,7,0\n'],
+      ['stop_times.txt', 'x'.repeat(400_000)],
+    ];
+    const encoder = new TextEncoder();
+    const parts: Uint8Array[] = [];
+    const central: Uint8Array[] = [];
+    let offset = 0;
+    for (const [name, text] of files) {
+      const data = encoder.encode(text);
+      const nameBytes = encoder.encode(name);
+      const local = new Uint8Array(30);
+      const lv = new DataView(local.buffer);
+      lv.setUint32(0, 0x04034b50, true);
+      lv.setUint32(18, data.length, true);
+      lv.setUint32(22, data.length, true);
+      lv.setUint16(26, nameBytes.length, true);
+      const head = new Uint8Array(46);
+      const hv = new DataView(head.buffer);
+      hv.setUint32(0, 0x02014b50, true);
+      hv.setUint32(20, data.length, true);
+      hv.setUint32(24, data.length, true);
+      hv.setUint16(28, nameBytes.length, true);
+      hv.setUint32(42, offset, true);
+      parts.push(local, nameBytes, data);
+      central.push(head, nameBytes);
+      offset += 30 + nameBytes.length + data.length;
+    }
+    const directorySize = central.reduce((sum, part) => sum + part.length, 0);
+    const end = new Uint8Array(22);
+    const ev = new DataView(end.buffer);
+    ev.setUint32(0, 0x06054b50, true);
+    ev.setUint16(10, files.length, true);
+    ev.setUint32(12, directorySize, true);
+    ev.setUint32(16, offset, true);
+    const archive = new Uint8Array(Buffer.concat([...parts, ...central, end]));
+    const requests: Record<string, string>[] = [];
+    let served = 0;
+    const definition: LiveSourceDefinition = {
+      id: 'test-bus',
+      kind: 'transit',
+      name: 'Test buses',
+      attribution: { label: 'Test', href: 'https://example.org' },
+      licence: { label: 'CC BY 4.0', href: 'https://creativecommons.org/licenses/by/4.0/' },
+      coverage: { west: 1.9, south: 41.2, east: 2.4, north: 41.6 },
+      completeness: 'network',
+      defaultUrl: 'https://example.org/feed',
+      requiresKey: false,
+      licenceCheck: { verifiedOn: '2026-10-12', source: 'https://example.org', quote: 'test' },
+      pollIntervalSeconds: 10,
+      staleAfterSeconds: 60,
+      dropAfterSeconds: 120,
+      staticGtfs: { zip: 'https://gtfs.example/big.zip', only: 'routes' },
+      create: () => ({
+        poll: async () => ({
+          entities: [{ ...bus('1', 41.39, 2.17, T0), line: null, mode: 'other' }],
+          rejected: 0,
+          hints: new Map([['test-bus:1', { route: '100001' }]]),
+        }),
+      }),
+    };
+    const clock = { now: T0 };
+    const hub = new LiveHub({
+      sources: [
+        {
+          definition,
+          settings: { url: definition.defaultUrl, userAgent: 'test', maxRegionsPerPoll: 1 },
+        },
+      ],
+      fetch: async (_url, init) => {
+        requests.push(init.headers);
+        const range = /^bytes=(?:-(\d+)|(\d+)-(\d+))$/.exec(init.headers.range ?? '');
+        if (!range) throw new Error('the whole archive was asked for');
+        const from = range[1] ? archive.length - Number(range[1]) : Number(range[2]);
+        const to = range[1] ? archive.length - 1 : Math.min(archive.length - 1, Number(range[3]));
+        const part = archive.slice(Math.max(0, from), to + 1);
+        served += part.length;
+        return {
+          status: 206,
+          headers: { get: () => null },
+          arrayBuffer: async () => part.buffer as ArrayBuffer,
+        };
+      },
+      now: () => clock.now,
+      random: () => 0.5,
+      publish: () => {},
+      logger: { info: () => {}, warn: () => {} },
+    });
+    hub.watch('a', [{ tile: BARCELONA_TILE, kind: 'transit' }]);
+    await hub.tick(); // loads the routes, and polls before they arrive
+    clock.now += 10_000;
+    await hub.tick();
+    expect(requests.length).toBeGreaterThan(0);
+    for (const headers of requests) expect(headers['accept-encoding']).toBe('identity');
+    expect(served).toBeLessThan(archive.length / 2);
+    expect(hub.snapshot(BARCELONA_TILE, 'transit').entities[0]).toMatchObject({
+      line: '7',
+      mode: 'tram',
+    });
+    hub.stop();
+  });
+
   it('counts room watchers, so one viewer leaving keeps the room for another', async () => {
     const h = harness();
     const room = { tile: BARCELONA_TILE, kind: 'transit' as const };

packages/backend/src/live/__tests__/licence.test.ts

@@ -0,0 +1,80 @@
+/**
+ * AGENTS.md: GoWay Live re-serves only feeds whose licence allows public,
+ * commercial redistribution. Every catalogue entry must say how that was
+ * established — the day, the page, the words — and a keyed one where the
+ * owner registers. An entry without them fails here.
+ */
+import { describe, expect, it } from 'bun:test';
+import { liveLinkSchema, liveSourceIdSchema } from '@goway/contracts';
+import { LIVE_SOURCES } from '../catalogue';
+import { LIVE_PROVIDERS } from '../providers';
+
+const https = (value: string) => /^https:\/\/[^\s]+$/.test(value);
+
+describe('every live source has a verified licence', () => {
+  for (const source of LIVE_SOURCES) {
+    it(source.id, () => {
+      const check = source.licenceCheck;
+      expect({ id: source.id, date: /^\d{4}-\d{2}-\d{2}$/.test(check?.verifiedOn ?? '') }).toEqual({
+        id: source.id,
+        date: true,
+      });
+      expect(Date.parse(check.verifiedOn)).toBeLessThanOrEqual(Date.now());
+      expect(check.quote.trim().length).toBeGreaterThanOrEqual(10);
+      expect(https(check.source)).toBe(true);
+      expect(https(source.licence.href)).toBe(true);
+      expect(source.licence.label.trim().length).toBeGreaterThan(0);
+      expect(source.attribution.label.trim().length).toBeGreaterThan(0);
+      if (source.requiresKey) expect(https(source.keyRegistrationUrl ?? '')).toBe(true);
+    });
+  }
+
+  it('every credit, licence, name and id is one the public contract accepts', () => {
+    for (const source of LIVE_SOURCES) {
+      const problems = [
+        liveSourceIdSchema.safeParse(source.id).success ? null : 'id',
+        liveLinkSchema.safeParse(source.attribution).success ? null : 'attribution',
+        liveLinkSchema.safeParse(source.licence).success ? null : 'licence',
+        source.name.length <= 80 ? null : 'name',
+      ].filter(Boolean);
+      expect({ id: source.id, problems }).toEqual({ id: source.id, problems: [] });
+    }
+  });
+
+  it('a provider is keyless or names where its key is registered — and never carries one', () => {
+    for (const provider of LIVE_PROVIDERS) {
+      if (provider.auth.kind !== 'none') expect(https(provider.auth.registrationUrl)).toBe(true);
+      // A key goes in LIVE_SOURCE_<ID>_KEY, never in the catalogue (Bordeaux's
+      // public, published-for-everyone key is the one exception, named here).
+      const query = new URL(provider.url).searchParams;
+      for (const name of ['key', 'apikey', 'api_key', 'api_token', 'appID', 'token'])
+        expect({ id: provider.id, [name]: query.get(name) }).toEqual({
+          id: provider.id,
+          [name]: null,
+        });
+    }
+  });
+
+  it('ids are unique', () => {
+    const ids = LIVE_SOURCES.map((source) => source.id);
+    expect(new Set(ids).size).toBe(ids.length);
+  });
+
+  it('a feed estimated from trip updates names the static GTFS it estimates along', () => {
+    for (const provider of LIVE_PROVIDERS) {
+      if (provider.format === 'gtfs-rt-tu')
+        expect({ id: provider.id, static: Boolean(provider.staticGtfs) }).toEqual({
+          id: provider.id,
+          static: true,
+        });
+    }
+  });
+
+  it('every provider polls no faster than every 10 s, and covers a real box', () => {
+    for (const provider of LIVE_PROVIDERS) {
+      expect(provider.pollIntervalSeconds).toBeGreaterThanOrEqual(10);
+      const { west, south, east, north } = provider.coverage;
+      expect(west < east && south < north).toBe(true);
+    }
+  });
+});