diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d6325a0..14149f16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -384,6 +384,18 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 + valkey: + # GoWay Live's bus between API tasks (`live/cluster.ts`): its leases, + # demand and fan-out are tested against a real server, the engine + # production runs (ElastiCache Valkey 8.0), not only an in-memory fake. + image: valkey/valkey:8.0 + ports: + - 6379:6379 + options: >- + --health-cmd "valkey-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -432,7 +444,22 @@ jobs: -v ON_ERROR_STOP=1 -tAc "SELECT postgis_lib_version();") echo "PostGIS $version" case "$version" in 3.5.*) ;; *) echo "::error::expected PostGIS 3.5.x, got $version"; exit 1 ;; esac + - name: Install a Redis server binary for the live bus outage test + # `live/__tests__/cluster.test.ts` stops and restarts a server of its + # own (on a random port) to prove the tasks recover from a Valkey + # outage; the service container above cannot be stopped from a test. + # policy-rc.d keeps apt from starting it on 6379, which the Valkey + # service already holds. + run: | + set -euo pipefail + printf '#!/bin/sh\nexit 101\n' | sudo tee /usr/sbin/policy-rc.d >/dev/null + sudo chmod +x /usr/sbin/policy-rc.d + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends redis-server + redis-server --version - name: Run the backend test suite + env: + TEST_VALKEY_URL: redis://127.0.0.1:6379 run: bun run test:backend # THE one status check `main` should require. Everything above reports here. diff --git a/docs/MAP_FEATURES.md b/docs/MAP_FEATURES.md index a122b253..f6269031 100644 --- a/docs/MAP_FEATURES.md +++ b/docs/MAP_FEATURES.md @@ -58,7 +58,7 @@ starts off. Shipping it is deleting that line. | `cityLife` | stable | **on** | web (needs `city3d`; the road archive where published, the basemap's roads everywhere else) | `?cityLife=0/1` | live (re-attach) | | `cityStreets` | stable | **on** | web (needs `city3d`; draws only where the road archive is published) | `?cityStreets=0/1` | live (re-attach) — [`MAP_STREETS.md`](./MAP_STREETS.md) | | `landmarks` | stable | **on** | web (needs `city3d`; a model draws, and the tour offers it, only over published city tiles) | `?landmarks=0/1` | live (re-attach; the Landmarks control and "Landmark tour" come and go, a running tour ends) | -| `live` | labs | off | web, **development builds only** (needs a backend that lists `LIVE_SOURCES`; `docs/MAP_LIVE.md`) | `?live=1` | live (mount/unmount: the panel in the top-right stack, the layer, its credits and the coverage channel come and go) | +| `live` | stable | **on** | web (draws only where the backend lists `LIVE_SOURCES`: `GET /live/sources` 503 → nothing, asked again while open; `docs/MAP_LIVE.md` → Production) | `?live=0/1` | live (mount/unmount: the panel in the top-right stack, the layer, its credits and the coverage channel come and go) | | `flyover` | stable | **on** | web, native | `?flyover=0/1` | live ("Fly the route", `/studio`) | **Everything is on on goway.to** (owner, 2026-10-12: "everything should be @@ -156,10 +156,9 @@ page with nothing to list leaves the rail. map's quality profile — [`MAP_PERFORMANCE.md`](./MAP_PERFORMANCE.md); it applies live). - **Layers**: Street 3D, the 3D city, its tiles (and the official building - data they carry), trees, simulated traffic, landmarks, and the flyover with - "Open Camera Studio" while it is on — each with what it shows. Once Labs is - revealed, the woods fill joins, and live data in development builds (until - goway.to's backend serves it). + data they carry), trees, simulated traffic, landmarks, live data, and the + flyover with "Open Camera Studio" while it is on — each with what it shows. + Once Labs is revealed, the woods fill joins. - **Labs**, only once revealed: what Labs is, and "Hide Labs". - **General**: what GoWay already lets a person set — the theme (light, dark, system: Bloom's own mode, persisted by `BloomProvider` under `goway.theme`) @@ -180,7 +179,7 @@ back first returns to the section list. **Revealing Labs** is deliberately not an `.env` decision: `?labs=1` on any goway.to address once (remembered on the device; `?labs=0` forgets), or press and hold the settings button, which opens the modal on the Labs page. It works in production — it only SHOWS switches -(the woods fill; live data in development builds), and every Labs feature +(the woods fill), and every Labs feature still starts off, so a shared link can never switch an experimental renderer on for somebody else. diff --git a/docs/MAP_LIVE.md b/docs/MAP_LIVE.md index 2940a723..20cc4758 100644 --- a/docs/MAP_LIVE.md +++ b/docs/MAP_LIVE.md @@ -1,8 +1,9 @@ # GoWay Live: real things moving on the map Aircraft, public-transport vehicles, bike-share stations and ships, drawn on -the map from open real-time feeds, moving as the feeds report them. **Built, -off by default, web only, served nowhere but a developer's machine.** +the map from open real-time feeds, moving as the feeds report them. **On by +default on the web, drawn wherever the backend serves it** — and goway.to's +backend serves it once the owner lists `LIVE_SOURCES` (→ Production). Unlike a simulation, this is real data, so three things decide every design choice below: **licence** (may GoWay re-serve it, publicly, from a service that @@ -213,14 +214,33 @@ accepted value is re-checked with `liveEntitySchema` before it leaves ### The hub (`hub.ts`, `tileIndex.ts`) - **In-memory spatial index**: the latest fix per entity, bucketed by **room** - = a z7 web-mercator tile × a kind. Rooms ARE the index: a subscription is + = a web-mercator tile × a kind, at each of the kind's levels + (`LIVE_ROOM_ZOOMS`: aircraft and ships z7, ~300 km; transit z7, z9 and z11 — + a region, ~60 km, ~15 km; bike share z13, ~3.7 km). Every entity is in one + room per level — except that **only rail is regional**: a train or a metro + (`LIVE_REGIONAL_TRANSIT_MODES`) is in all three transit levels, a bus, tram, + ferry, cable car or vehicle of unknown mode in the z11 rooms alone + (`liveRoomZoomsOf`); a vehicle whose mode its operator's `routes.txt` + changes moves between levels. A subscription takes, per kind, the finest + level that covers the view in at most 20 rooms — a borough at a city zoom + with its buses; all of Catalonia (every Rodalies train, Girona's and + Tarragona's included) from the z7 rooms at a regional one, and no bus. Rooms ARE the index: a subscription is answered from the buckets its tiles cover, a delta is addressed to the bucket an entity is in, a box query walks only the buckets it touches. Replacing an entry discards the old fix: **positions are never stored as history**, and nothing is written to a database. - **Demand-driven polling**: a source is polled only while somebody watches a - room of its kind inside its coverage. No viewers in New York, no requests to - New York's feeds. A source unwatched for 60 s drops what it holds. + room of its kind that meets its **areas** — boxes round its operator's own + stops (`areas.ts`, derived by `bun run --cwd packages/backend live:areas` + into `areas.generated.ts`: 0.5° cells holding stops, each row's run of cells + one box shrunk to its stops and padded 5 km). Renfe's one box ran from + Galicia to Catalonia and took in Toulouse; its 31 areas follow its lines, + and a street view of Toulouse wakes nothing (pinned by a test). A source's + `coverage` is the box round its areas; a source with no published stops + (adsb.lol, AISA, Digitraffic) keeps a hand-drawn one. A regional room wakes + only a source that may carry rail (`transitModes`): New York's buses are + not polled for a regional view. No viewers in New York, no requests to New + York's feeds. A source unwatched for 60 s drops what it holds. - **Freshness is the feed's**: anything older than its source's `dropAfterSeconds` (by its own `observedAt`) is removed on the next tick, whatever the feed still lists. Full-snapshot feeds also remove whatever a poll @@ -231,8 +251,16 @@ accepted value is re-checked with `liveEntitySchema` before it leaves the source and the reason, never a URL (keys are redacted) and never a viewport. - **Deltas** per room, only for rooms somebody watches: upserts (new or - changed, by value) and removes. An entity crossing a room boundary is a - remove in one and an upsert in the other; the client resolves the race. + changed, by value), **moves** and removes. A vehicle whose only change is + its motion (position, heading, speed, time, place along its track) travels + as a compact `LiveMove` tuple — ~80 bytes instead of ~450. An entity crossing + a room boundary is a remove in one and an upsert in the other; the client + resolves the race. +- **Packed**: a message over `LIVE_PACKED_MIN_BYTES` (2 KB) is sent as gzip + bytes (a Socket.IO binary attachment) and inflated by the client + (`unpackLivePayload`). In the application because Bun's WebSocket server + does not negotiate permessage-deflate (measured: `socket.io` under Node + answers `permessage-deflate`, under Bun nothing). ### The socket (`socket.ts`) @@ -243,9 +271,11 @@ are public tiles × kinds, computed **by the server** from a box it validated. The default namespace keeps its Oxy auth; a test proves both. - `live:subscribe` `{ west, south, east, north, zoom, kinds }` → the server - computes the z7 tiles (refusing more than 20 as `too_wide`: below ~z8 the - map is a continent and tens of thousands of dots are noise), joins/leaves - rooms, answers `live:subscription` with the rooms it is in, then one + computes each kind's tiles at its zoom, leaves out a kind covering more than + 20 (all of them: `too_wide` — below ~z8 the map is a continent and tens of + thousands of dots are noise), joins/leaves rooms, answers + `live:subscription` with the rooms it is in, per kind (`tiles: { transit: + [...], aircraft: [...] }`; a kind left out is absent), then one `live:snapshot` per new room. Empty `kinds` leaves everything. - Limits (`LIVE_POLICY`, `config/live.ts`): 2,000 sockets per process (handshake refused with `live_capacity`), and a token bucket per socket for @@ -262,7 +292,7 @@ The default namespace keeps its Oxy auth; a test proves both. | Switch | The map-feature registry's `live` (`lib/map/mapFeatures.ts`, `docs/MAP_FEATURES.md`): the person's choice in the settings dialog (development builds), and `?live=1` in development only | | Gate | `features/explore/ExploreScreen.tsx`: `useMapFeature('live')` mounts the lazy `LiveHost` in the top-right stack, under the settings button, the time of day and the Landmarks control; off, it unmounts and the layer, its credits and the coverage go with it | | HTTP | `lib/live/client.ts`: one SDK client for `live.sources()` and `live.shape()`, always the real API (never the fixture transport) — shared by the shape cache and the settings dialog's **Live data sources** | -| Socket client | `features/live/liveClient.ts` (`socket.io-client`, debounced subscriptions, honours `rate_limited`) | +| Socket client | `features/live/liveClient.ts` (`socket.io-client`, WebSocket only, debounced subscriptions, honours `rate_limited`; bike share asked for from z15 only; packed messages inflated in arrival order) | | State and motion (pure, tested) | `features/live/liveStore.ts`, `motion.ts` | | Panel and copy | `features/live/LivePanel.tsx`, `format.ts`, `messages.ts` (en/es), `kinds.ts` (data colours) | | Map seam | `MapCanvasProps.live` (`MapLiveLayer`, a pull interface: no React render per frame), `MapAttribution` `extra` credits | @@ -467,10 +497,24 @@ empty. Kinds come from the transit mode (`rail` → `train`, `subway` → `metro - **Backend** `LIVE_SOURCES` (each feed opted into by id; an unknown id fails the boot parse). There is no other switch: live runs when a usable source is listed. None: no poller, no `/live` namespace, `GET /live/sources` → - `503 service_unavailable`. -- **Frontend** the registry's `live` (`docs/MAP_FEATURES.md`): the switch in - the settings dialog's Labs (offered in development builds only, until - goway.to has a live backend), or `?live=1` in a development build. + `503 service_unavailable`, and no other live setting (the Valkey URL + included) is read — none can stop the backend from booting. +- **Frontend** the registry's `live` (`docs/MAP_FEATURES.md`): **on by + default**, a stable switch on the settings' Layers page (`?live=0` turns it + off for one tab in a development build). Safe on by default because the backend is the + gate: `LiveHost` asks `GET /live/sources` first, and a deployment without + live (503) draws nothing and opens no socket — and asks again (30 s, 1, 2, + then every 5 min), so a map left open finds live when the backend turns it + on. The panel starts folded to its pill; the quality profile's + `live.maxEntities` caps what is drawn. + +### What it costs on first paint + +`live` is on by default, so on the web the lazy `LiveHost` chunk (~48 KB), +the shared `client` chunk and — only once `GET /live/sources` answers 200 — +`socket.io-client` (~54 KB, the chunk `@oxy.so/core` already lazy-loads) and +the `liveLayer` adapter (~8 KB) load AFTER first paint. First paint itself +carries only the seam measured below; `__common` is unchanged. ### What the flag costs when off @@ -508,6 +552,8 @@ importing the engine folders type-only). LIVE_SOURCES=adsb-lol,renfe-cercanias,fgc-geotren,mta-bus,mta-lirr,mta-mnr,mta-subway,amb-bus,aisa-bus,emt-fuenlabrada,nyc-bikeshare,digitraffic-ais \ CORS_APP_ORIGINS=http://localhost:8081 \ bun run --cwd packages/backend dev +# Optional: REDIS_URL=redis://127.0.0.1:6379 shares polling with other local +# backends on that Valkey/Redis (two processes, one poller). # Frontend against it EXPO_PUBLIC_API_URL=http://localhost:3000 bun run --cwd packages/frontend web @@ -532,42 +578,240 @@ EXPO_PUBLIC_API_URL=http://localhost:3000 bun run --cwd packages/frontend web | AISA, EMT Fuenlabrada | 1 / 20 s each | none published | | Static GTFS | each file once a day per process (≤ 120 MB) | — | -Polling is per PROCESS and demand-driven, so cost scales with the number of -backend tasks and the areas watched, not with viewers. Upstream data is free. -Egress is the cost: a dense room (all of New York: ~1,400 buses, ~2,500 -stations) is ~1.3 MB per snapshot and ~25–30 KB/s of deltas per viewer -uncompressed. Bandwidth, not CPU, is the first limit. - -## What production needs - -1. **Licence follow-ups** — email adsb.lol before production use and plan a - feeder (their key requirement is coming); register an MTA Bus Time key; get - Lyft's (or counsel's) answer on `nyc-bikeshare` over a public socket, or do - not enable it; get an Open Data BCN token for Bicing and add the - no-discovery GBFS mode; publish the ODbL notice for re-served aircraft data - alongside the map's (the `GET /live/sources` licence link is the start). -2. **One poller, many servers** — today every backend task polls on its own and - holds its own index. With more than one task, move polling to one worker (or - elect one) and fan out through the Socket.IO Redis/Valkey adapter - (ElastiCache Valkey is already in the stack) — otherwise N tasks multiply - upstream load by N and rate limits by N. -3. **Bandwidth** — enable per-message deflate for `/live` (or send deltas as a - compact binary), and thin dense rooms by zoom (a z12 view does not need - 2,500 bike stations; the client already hides them below z15 but still - receives them). -4. **CORS / embeds** — the socket honours `CORS_APP_ORIGINS` for long-polling; - WebSocket upgrades are not CORS-checked by browsers. Decide whether - third-party embeds get `/live` (it needs the public lane and the licence - answers above). -5. **Native** — `MapCanvas.native.tsx` ignores `live` today (and - the registry offers `live` on web only). MapLibre Native can draw the same - GeoJSON symbol layer with a `ShapeSource`; the pull interface would need a - ~2–4 Hz React update there instead of a frame loop. The 3D pass stays web - only, as the city engine does. -6. **Observability** — per-source poll latency, failure counts and entity - counts as metrics (they are in `GET /live/sources` and the logs today). -7. **Turn it on**: set `LIVE_SOURCES` to the vetted subset, then drop - `offer: 'development'` from the registry's `live` entry. +Polling is per DEPLOYMENT (one leader per source, → Production) and +demand-driven, so cost scales with the areas watched, not with viewers or +tasks. Upstream data is free. Egress was the cost — a whole-New-York view was +1.2 MB of snapshot and 18 KB/s of deltas per viewer — and is now ~25 KB and +<1 KB/s (→ Production, Bandwidth). + +## Production + +### Several API tasks, one poller (`live/cluster.ts`) + +The API runs as ECS tasks behind one ALB. Each task serves its own sockets; +the tasks share GoWay's Valkey (`REDIS_URL`, the fleet's +`/oxy/_shared/REDIS_URL`): + +- **Demand** — every 2 s each task decides, from its sockets' exact rooms + (which never leave the process), which SOURCES it wants, and writes only + that under `goway:live:demand:` with a 10 s TTL: source ids + (`mta-subway`, `renfe-cercanias`), plus, for the one regional feed polled + per region (aircraft), `adsb-lol@` (~470 km). A source id says no + more than the source's coverage — a city or a country — so nothing stored is + a viewport, and nothing names a viewer or a socket (`demandEntries` in + `hub.ts`). Because the decision is made on the exact rooms, a viewer in + Philadelphia turns on no New York feed even though both lie in one z6 tile + (pinned by a test). Every task reads the union; a source is wanted while any + task wants it. +- **One leader per source** — a lease (`SET NX PX 15000`, renewed by its + holder every 2 s through a compare-and-renew script) that carries an + **epoch**: the next value of the source's counter (`INCR + goway:live:epoch:`, no TTL) when this holder took it — never a + clock, which a failover to a replica whose clock lags would turn backwards. + Every message is stamped with it and a follower drops one older than the + newest it has heard — a publish abandoned at its deadline and delivered + after the lease moved on is never applied — unless that newest has been + silent for a whole lease: then the counter itself went backwards (a + failover that lost it), and the follower takes the lower mark rather than + stay deaf to the only leader there is. Only the holder polls; + it publishes each answer (the enriched entities, gzip + base64) on one + pub/sub channel, and every other task applies it to its own index exactly as + if it had polled, and tells its own sockets. Tasks that stop wanting a + source release its lease. +- **Failover** — a leader that dies stops renewing; within 15 s another task + takes the lease and polls at once. Measured with two real backend processes + on one Valkey: the holder killed with `SIGKILL` at 11:24:58, the lease + moved at 11:25:14, and the surviving task went on serving New York. +- **A task that starts** asks the leaders to republish what they hold + (`sync`), so its first viewer gets a full snapshot, not an empty one. +- **Valkey down** — every task polls on its own (duplicate upstream requests, + logged once) until Valkey answers; then the leases decide again. Three + things make "answers again" true in practice, each measured on Bun 1.4.3 + against a real `redis-server`: + - **Deadlines.** Every bus call is bounded at a third of the lease (5 s): a + call that does not answer counts as the bus being down, so the task polls + on its own at once instead of waiting out the client (~10.7 s per command + with Valkey stopped; minutes on a silent partition, long enough for the + lease to pass to a second poller while the first still polls). + - **Fresh connections.** After an outage of 8 s or more Bun's client gives up + for good: every later command fails `ERR_REDIS_CONNECTION_CLOSED` even + once Valkey is back, while a new client answers at once. A client whose + call failed on its connection, or that connected and then dropped, is + replaced (`LiveBus.recover`); one still making its first connection is + left alone. + - **Listening first.** A task stops polling only once it is subscribed again + — a reconnected client does not restore subscriptions, and a subscriber + that closed (`onclose`) or is not connected counts as lost — so the + subscription is remade on a fresh connection, with a `sync`, before the + lease decides. + + Pinned by a test that stops a real server for 10 s and starts it again: one + leader per source and the follower hearing the leader within a few rounds + (the previous code kept both tasks polling). Stopping closes the bus's + connections whatever its state. +- Nothing about a position is STORED in Valkey: positions pass through + pub/sub; the only keys are the leases, the task list and the coarse demand + above, all with TTLs, so `volatile-lru` may evict them under pressure (worst + case: a second poller for a few seconds) — and one epoch counter per + source, an integer with no TTL, which it never evicts. +- Static GTFS is read by the leader (to put vehicles on their tracks) and by + a task whose own viewers are close enough to ask it for shapes — the + finest transit rooms, where a view from `LIVE_SHAPES_MIN_ZOOM` (z12) is — + not by every task the source is wanted somewhere for, nor for a regional + view. +- The client uses WebSocket only (no long-polling), so no ALB stickiness is + needed: one connection is one task, whichever it is. + +Pinned in `live/__tests__/cluster.test.ts`: one upstream request for three +tasks, every task serving the answer, failover after the lease, a late task +synced, local polling while the bus is down or silent, coarse demand — in +memory; against a real Valkey when `TEST_VALKEY_URL` is set (CI runs +`valkey/valkey:8.0`); and through a real 10 s outage of a server the test +starts itself when `valkey-server` or `redis-server` is installed (CI installs +`redis-server`). + +### Bandwidth + +Three changes, measured on New York with the real feeds at the same moment +(2026-10-12, 04:20 local, so ~550 vehicles; daytime is ~3× that), one view +on the old protocol and one on the new: + +| View | Before: first snapshots | Before: deltas | After: first snapshots | After: deltas | +| --- | ---: | ---: | ---: | ---: | +| All of New York, z12 | 1,225 KB | 18.1 KB/s | 25 KB | 0.7 KB/s | +| Midtown, z15 | 1,225 KB | 18.1 KB/s | 24 KB | 0.3 KB/s | + +1. **Per-kind rooms at several levels** (`LIVE_ROOM_ZOOMS`): at a street or + city zoom transit comes in ~15 km rooms and bike share in ~3.7 km ones — a + street view receives its neighbourhood, not the city; zoomed out to a + region, the coarser rooms keep every TRAIN there, and no bus. And the + client asks for bike share only from z15, where it draws it: New York's + 2,458 docks were most of the old snapshot. + + A regional view of New York (~z9.5, four z9 rooms), measured with the real + feeds on 2026-10-12 at 21:15 local: **3.4 KiB** of first snapshots (77 + trains and subway cars) where every vehicle under those rooms would be + **100.2 KiB** (2,344, nearly all buses); deltas 16.3 KiB over 19 s (one + bus poll) against 34.8 KiB. The subscription's answer lists its rooms per + kind, so a client drops exactly the rooms it left — an aircraft room's z7 + tile no longer keeps the transit room of the same tile, whose buses froze + there after zooming in. +2. **Moves**: a vehicle that only moved is a tuple, not an entity. +3. **Packing**: messages over 2 KB as gzip bytes (~9.6× on a transit + snapshot: 257 KB of JSON in 27 KB). + +### Configuration + +Backend (the ECS task definition). Values marked SSM are SecureString +parameters the task definition names as `secrets`. + +| Variable | Value | Where | +| --- | --- | --- | +| `LIVE_SOURCES` | `renfe-cercanias,fgc-geotren,mta-lirr,mta-mnr,mta-subway,amb-bus,aisa-bus,emt-fuenlabrada,digitraffic-ais` to start; add `mta-bus` with its key, `adsb-lol` after the operator's answer, `nyc-bikeshare` after Lyft's (→ Licences) | environment | +| `REDIS_URL` | `/oxy/_shared/REDIS_URL` (the fleet Valkey), as allo, clarity and moovo read it. Read only when `LIVE_SOURCES` lists a source; if it is malformed, GoWay logs a warning (never the value) and live runs without a bus — a variable a dozen services share never stops GoWay booting | SSM, `secrets` | +| `LIVE_SOURCE_MTA_BUS_KEY` | the MTA Bus Time key | SSM `/oxy/goway/LIVE_SOURCE_MTA_BUS_KEY`, `secrets` | + +Nothing else: the user agent (`GoWay-Live/1.0 (+https://goway.to)`, which +names a way to reach GoWay to feeds that ask callers to identify themselves), +the connection cap (2,000 per task) and the rate limits are code +(`LIVE_POLICY` in `config/live.ts`). Frontend: nothing — `live` is on by +default. + +Capacity: a task holding every source's static GTFS measured 592 MB resident +(902 MB peak while loading) against today's 1,024 MB task. Live needs **2,048 +MB** per task (CPU 512 is enough: polling is one task's work, serving is +JSON per room). + +### Owner's checklist + +Infrastructure (`oxy-infra`; nothing here writes SSM or touches AWS): + +1. `aws ssm put-parameter --name /oxy/goway/LIVE_SOURCE_MTA_BUS_KEY --type SecureString --value ` once the key exists (below). +2. `terraform-uswest2/app-goway.tf`: add the environment variable + `LIVE_SOURCES` above, `memory = "2048"`, and to `secrets`: + `{ name = "REDIS_URL", valueFrom = "${local.ssm}/oxy/_shared/REDIS_URL" }` + and `{ name = "LIVE_SOURCE_MTA_BUS_KEY", valueFrom = "${local.ssm}/oxy/goway/LIVE_SOURCE_MTA_BUS_KEY" }` + (replacing the comment that says GoWay reads no `REDIS_URL` — it does now). +3. `terraform-uswest2/iam-goway-deploy.tf`, `goway_execution`: extend its + `ssm:GetParameters` resources with `/oxy/_shared/REDIS_URL` and + `/oxy/goway/LIVE_SOURCE_MTA_BUS_KEY` — the execution role reads only + `DATABASE_URL` today, and a task naming a secret it cannot read does not + start. +4. Register a revision derived from the RUNNING one carrying those, repoint the + service once, and confirm it serves (`curl https://api.goway.to/api/v1/live/sources` + answers 200 with the sources) — the deploy carries the running revision + forward and never adds variables itself (`~/Oxy/docs/infra-and-deploys.md`). +5. The tasks' security group already reaches Valkey (`redis-sg` admits + `tasks`); nothing to change. + +### Licences, per source + +| Source | Ready? | What is left | +| --- | --- | --- | +| `renfe-cercanias` | **yes** | Credit "Origen de los datos: Renfe Operadora" drawn while its trains are; every position's age shown. | +| `fgc-geotren` | **yes** | Credit drawn; 2,880 calls a day against FGC's 5,000/IP — one poller now keeps it there whatever the task count. | +| `mta-lirr`, `mta-mnr`, `mta-subway` | **yes** | "Data: MTA", no MTA marks, no data edits; subway positions labelled as estimates. | +| `mta-bus` | after a key | Register the Bus Time key (below), set it, add `mta-bus` to `LIVE_SOURCES`. | +| `amb-bus` | **yes** | Credit and update time shown; the estimate is labelled as GoWay's. | +| `aisa-bus`, `emt-fuenlabrada` | **yes** | "Powered by MITRAMS" drawn. | +| `digitraffic-ais` | **yes** | Credit drawn; `Digitraffic-User` sent. Baltic only. | +| `adsb-lol` | after the operator's answer | The operator asks production users to get in touch first (email below), and an API key for non-feeders is announced: plan an ADS-B feeder (an RTL-SDR stick, an antenna and `readsb` on a small computer feeding adsb.lol). ODbL: the credit and the licence link are drawn and listed. | +| `nyc-bikeshare` | **no, not yet** | The Bikeshare Data License forbids distributing the data "as a stand-alone dataset". Whether a public map that draws it, over a socket anyone can open, is that is Lyft's (or counsel's) question. Ask before listing it. | + +### MTA Bus Time key + +1. Open https://register.developer.obanyc.com/ (the MTA's OneBusAway NYC + developer registration, linked from https://new.mta.info/developers). +2. Give a name, an email address that will read MTA notices, and a short + description — "GoWay (goway.to), a public map; draws MTA buses with the + credit 'Data: MTA', never MTA marks." +3. Accept the MTA data feed terms. The key arrives by email (usually within + minutes; manual review at the MTA's discretion). +4. Store it: checklist step 1. GoWay sends it as `?key=` on every Bus Time + request and never logs it (`redact` in `hub.ts`). + +### adsb.lol — the email to send + +> **Subject:** GoWay — production use of the adsb.lol API (ODbL, credited) +> +> Hi, +> +> I run GoWay (https://goway.to), an open map. We'd like to show live +> aircraft from the adsb.lol API on the map, and your page asks production +> users to get in touch first — so here we are. +> +> How we'd use it: +> - Server-side only: one process polls `/v2/lat/{lat}/lon/{lon}/dist/{nm}` for the regions people +> are looking at, at most every 15 s and at most 4 regions per poll, and +> stops at the first 429 for as long as `Retry-After` says. Viewers never +> call you; one deployment makes the same requests whatever its traffic. +> - Requests carry `User-Agent: GoWay-Live/1.0 (+https://goway.to)`. +> - Aircraft flagged LADD or PIA in your data are dropped before display. +> - Attribution: "© adsb.lol contributors" and the ODbL 1.0 link are shown +> whenever aircraft are on the map, and listed in the app's credits. The +> positions we re-serve to the map are under ODbL as well. +> +> We'd also like to contribute: we're planning to run a feeder. If you'd +> rather we use an API key, or a different endpoint or rate, tell us and +> we'll do that. +> +> Thanks for running this. +> , GoWay + +The address: the contact the operator lists on https://adsb.lol (its +GitHub, `adsblol`, or the community Discord) — use whichever it names for API +questions at the time. + +### Still open + +- **Native** — `MapCanvas.native.tsx` ignores `live` (the registry offers it + on web only). MapLibre Native can draw the same GeoJSON symbol layer with a + `ShapeSource` and a ~2–4 Hz React update; the 3D pass stays web only. +- **Observability** — per-source poll latency, failures, entity counts and + lease holder as metrics (in `GET /live/sources` and the logs today). +- **Embeds** — whether third-party embeds get `/live` is a licence question + first (the answers above). +- Bicing (Open Data BCN token) and TMB (key) as further sources. ## Limits, honestly @@ -584,7 +828,8 @@ uncompressed. Bandwidth, not CPU, is the first limit. on arrival. The UI states every age; it cannot make them younger. - adsb.lol's dynamic rate limit means aircraft can go stale for a poll or two under load; they are shown stale, not extrapolated. -- One process holds everything in memory; a restart starts from empty and +- Every task holds every source's current positions in memory; a task that + starts is synced by the leaders, and a deployment that restarts entirely refills within one poll interval. - HSL, Bicing, TMB, EMT Madrid, NYC Ferry and Baltic-only ships are documented above as next sources, not built. diff --git a/packages/backend/package.json b/packages/backend/package.json index f6bbf147..a1da5898 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -20,7 +20,8 @@ "captures:cleanup": "bun run src/capture/runCleanup.ts", "street3d:tick": "bun run src/street3d/runTick.ts", "street3d:admin": "bun run src/street3d/runAdmin.ts", - "street3d:serve-assets": "bun run src/street3d/runServeAssets.ts" + "street3d:serve-assets": "bun run src/street3d/runServeAssets.ts", + "live:areas": "bun run src/live/runAreas.ts" }, "dependencies": { "@goway/contracts": "workspace:*", diff --git a/packages/backend/src/config/live.ts b/packages/backend/src/config/live.ts index 7ee04726..2a6ada0d 100644 --- a/packages/backend/src/config/live.ts +++ b/packages/backend/src/config/live.ts @@ -99,6 +99,20 @@ export interface LiveConfig extends LivePolicy { /** Source ids, as listed, de-duplicated. Checked against the catalogue by the caller. */ sources: readonly string[]; settings: Readonly>; + /** + * Valkey shared by the API tasks (`live/cluster.ts`): one task polls each + * feed and the others are told. `REDIS_URL` — the fleet's shared Valkey + * (`/oxy/_shared/REDIS_URL`), owned by oxy-infra and read by a dozen + * services; unset, this process polls alone (right for one task, N× the + * upstream load for N). Read only when a source is listed. + */ + valkeyUrl: string | null; + /** + * Why `REDIS_URL` was not usable, when it was set and malformed — never the + * value. A shared variable another team owns must not stop GoWay booting: + * live then runs without a bus and `createLiveService` logs this. + */ + valkeyUrlProblem: string | null; } const sourceList = z.preprocess( @@ -157,10 +171,27 @@ export function parseLiveConfig( settings[id] = entry; } + // The bus matters only to a deployment that serves live. With no source + // listed it is not read at all; when it is malformed it is reported, not + // fatal: REDIS_URL is shared by a dozen services, and a `valkey://` scheme + // or a bare host:port there must not take maps, search and routing down. + let valkeyUrl: string | null = null; + let valkeyUrlProblem: string | null = null; + const rawValkey = listed.length > 0 ? emptyAsUndefined(source.REDIS_URL) : undefined; + if (rawValkey !== undefined) { + try { + const url = new URL(String(rawValkey).trim()); + if (url.protocol !== 'redis:' && url.protocol !== 'rediss:') throw new Error('protocol'); + valkeyUrl = url.toString(); + } catch { + valkeyUrlProblem = 'REDIS_URL is not a redis:// or rediss:// URL'; + } + } + if (problems.length > 0) { throw new Error(`Invalid live configuration:\n${problems.join('\n')}`); } - return { ...LIVE_POLICY, sources: listed, settings }; + return { ...LIVE_POLICY, sources: listed, settings, valkeyUrl, valkeyUrlProblem }; } /** The one parse for this process, against the catalogue's ids. */ diff --git a/packages/backend/src/live/__tests__/areas.test.ts b/packages/backend/src/live/__tests__/areas.test.ts new file mode 100644 index 00000000..22dbe3b2 --- /dev/null +++ b/packages/backend/src/live/__tests__/areas.test.ts @@ -0,0 +1,93 @@ +/** + * A source is woken by viewers near its operator's own stops, not anywhere in + * one box round its whole network: Renfe's box took in Toulouse. + */ +import { describe, expect, it } from 'bun:test'; +import { liveTileOf } from '@goway/contracts'; +import { areasOf, sourceAreas } from '../areas'; +import { LIVE_SOURCES } from '../catalogue'; +import { LiveHub } from '../hub'; +import type { LiveSourceDefinition } from '../types'; + +const contains = ( + box: { west: number; south: number; east: number; north: number }, + [latitude, longitude]: readonly [number, number], +) => + box.south <= latitude && latitude <= box.north && box.west <= longitude && longitude <= box.east; + +describe('areas from stops', () => { + it('follow a network round its corners, and leave the land between out', () => { + // An L: stations west to east along 42°N, then north along 3°E. + const line: [number, number][] = []; + for (let lon = -1; lon <= 3; lon += 0.1) line.push([42, lon]); + for (let lat = 42; lat <= 44.5; lat += 0.1) line.push([lat, 3]); + const derived = areasOf(line); + if (!derived) throw new Error('no areas'); + const inAny = (point: [number, number]) => derived.areas.some((area) => contains(area, point)); + expect(inAny([42, 0.5])).toBe(true); + expect(inAny([44, 3])).toBe(true); + // The corner's inside, ~150 km from either arm: in the outer box, in no area. + expect(contains(derived.outer, [43.6, 1.44])).toBe(true); + expect(inAny([43.6, 1.44])).toBe(false); + expect(areasOf([])).toBeNull(); + expect(areasOf([[0, 0]])).toBeNull(); + }); + + it('exist for every source whose stops are published, and are its coverage', () => { + for (const source of LIVE_SOURCES) { + const stops = Boolean(source.staticGtfs) || source.kind === 'bikeshare'; + const derived = sourceAreas(source.id); + expect({ id: source.id, derived: Boolean(derived) }).toEqual({ + id: source.id, + derived: stops, + }); + if (!derived) continue; + expect(source.coverage).toEqual(derived.outer); + for (const area of derived.areas) { + expect(area.west).toBeGreaterThanOrEqual(derived.outer.west); + expect(area.east).toBeLessThanOrEqual(derived.outer.east); + expect(area.south).toBeGreaterThanOrEqual(derived.outer.south); + expect(area.north).toBeLessThanOrEqual(derived.outer.north); + } + } + }); +}); + +describe('demand by area', () => { + it('a street view of Toulouse wakes no Spanish feed; one of Barcelona does', async () => { + const renfe = LIVE_SOURCES.find((s) => s.id === 'renfe-cercanias') as LiveSourceDefinition; + const polls: string[] = []; + const statics: string[] = []; + const hub = new LiveHub({ + sources: [ + { + definition: { + ...renfe, + create: () => ({ + poll: async () => { + polls.push('renfe'); + return { entities: [], rejected: 0 }; + }, + }), + }, + settings: { url: renfe.defaultUrl, userAgent: 'test', maxRegionsPerPoll: 1 }, + }, + ], + fetch: async (url) => { + statics.push(url); + throw new Error('not fetched in this test'); + }, + publish: () => {}, + }); + const street = (latitude: number, longitude: number) => [ + { tile: liveTileOf({ latitude, longitude }, 'transit'), kind: 'transit' as const }, + ]; + hub.watch('viewer', street(43.6045, 1.444)); + await hub.tick(); + expect([polls, statics]).toEqual([[], []]); + hub.watch('viewer', street(41.39, 2.17)); + await hub.tick(); + expect(polls).toEqual(['renfe']); + hub.stop(); + }); +}); diff --git a/packages/backend/src/live/__tests__/cluster.test.ts b/packages/backend/src/live/__tests__/cluster.test.ts new file mode 100644 index 00000000..a7890c6c --- /dev/null +++ b/packages/backend/src/live/__tests__/cluster.test.ts @@ -0,0 +1,772 @@ +/** + * GoWay Live across several API tasks (`cluster.ts`): one poller per source, + * every task serving its own sockets, leader failover, a joining task brought + * up to date, and polling locally while the bus is down. + * + * In-memory "Valkey" for the logic; a REAL Valkey (or Redis) when + * `TEST_VALKEY_URL` is set — CI runs one, as it runs Postgres. + */ + +import { afterAll, describe, expect, it } from 'bun:test'; +import { type LiveDelta, type LiveEntity, liveTileOf } from '@goway/contracts'; +import { + decodeMessage, + encodeMessage, + type LiveBus, + MemoryLiveNetwork, + ValkeyLiveBus, +} from '../cluster'; +import { LIVE_SOURCES } from '../catalogue'; +import { demandEntries, LiveHub } from '../hub'; +import type { LivePollResult, LiveSourceDefinition } from '../types'; + +const T0 = Date.parse('2026-10-12T12:00:00Z'); +const BARCELONA = { latitude: 41.39, longitude: 2.17 }; +const ROOM = { tile: liveTileOf(BARCELONA, 'transit'), kind: 'transit' as const }; + +function bus(id: string, observedAt: number, longitude = 2.17): LiveEntity { + return { + kind: 'transit', + id: `test-bus:${id}`, + sourceId: 'test-bus', + position: { latitude: 41.39, longitude }, + headingDegrees: 90, + speedMetersPerSecond: 8, + observedAt: new Date(observedAt).toISOString(), + mode: 'bus', + operator: 'Test', + line: 'V15', + headsign: null, + vehicleLabel: null, + lineColor: '#FF0000', + lineTextColor: '#FFFFFF', + track: null, + trackStatus: 'unknown', + positionBasis: 'reported', + }; +} + +interface Task { + hub: LiveHub; + polls: number; + deltas: LiveDelta[]; +} + +/** A task: a hub on its own bus, with a fake feed that answers `answer()`. */ +function task( + liveBus: LiveBus, + now: () => number, + answer: () => LivePollResult, + cluster?: { leaseMs?: number; demandTtlMs?: number; callTimeoutMs?: number }, +): Task { + const out: Task = { hub: null as unknown as LiveHub, polls: 0, deltas: [] }; + const definition: LiveSourceDefinition = { + id: 'test-bus', + kind: 'transit', + name: 'Test buses', + attribution: { label: 'Test', href: 'https://example.org' }, + licence: { label: 'CC BY 4.0', href: 'https://creativecommons.org/licenses/by/4.0/' }, + coverage: { west: 1.9, south: 41.2, east: 2.4, north: 41.6 }, + completeness: 'network', + defaultUrl: 'https://example.org/feed', + requiresKey: false, + pollIntervalSeconds: 10, + staleAfterSeconds: 60, + dropAfterSeconds: 120, + create: () => ({ + poll: async () => { + out.polls++; + return answer(); + }, + }), + }; + out.hub = new LiveHub({ + sources: [ + { + definition, + settings: { url: definition.defaultUrl, userAgent: 'test', maxRegionsPerPoll: 1 }, + }, + ], + fetch: async () => { + throw new Error('the adapter is faked'); + }, + now, + random: () => 0.5, + bus: liveBus, + publish: (delta) => out.deltas.push(delta), + ...(cluster ? { cluster } : {}), + }); + return out; +} + +describe('the Valkey bus subscription', () => { + /** A connection whose (re)connects the test drives. */ + const fakeConnection = () => { + const connection = { + onconnect: null as (() => void) | null, + onclose: null as (() => void) | null, + connected: true, + subscribed: 0, + closed: false, + async send() { + return null; + }, + async publish() { + return 0; + }, + async subscribe() { + connection.onconnect?.(); + connection.subscribed++; + return 1; + }, + close() { + connection.closed = true; + connection.connected = false; + }, + }; + return connection; + }; + + it('counts as lost after its connection reconnects, and resubscribes on a fresh one', async () => { + const made: ReturnType[] = []; + const liveBus = new ValkeyLiveBus(() => { + const connection = fakeConnection(); + made.push(connection); + return connection; + }); + made.shift(); // the command connection + expect(liveBus.isSubscribed()).toBe(false); + await liveBus.subscribe(() => {}); + expect(liveBus.isSubscribed()).toBe(true); + // Valkey restarted: the client reconnected by itself, without its subscription. + made[0]?.onconnect?.(); + expect(liveBus.isSubscribed()).toBe(false); + await liveBus.subscribe(() => {}); + expect(liveBus.isSubscribed()).toBe(true); + expect(made).toHaveLength(2); + expect(made[0]?.closed).toBe(true); + await liveBus.close(); + expect(made[1]?.closed).toBe(true); + expect(liveBus.isSubscribed()).toBe(false); + }); + + it('a client still connecting is never replaced; one that connected and dropped, or failed, is', async () => { + const made: ReturnType[] = []; + const liveBus = new ValkeyLiveBus(() => { + const connection = fakeConnection(); + connection.connected = false; // mid-handshake + made.push(connection); + return connection; + }); + liveBus.recover(); + expect(made).toHaveLength(1); + expect(made[0]?.closed).toBe(false); + // It connects, then the connection drops: now it is dead. + (made[0] as ReturnType).connected = true; + made[0]?.onconnect?.(); + (made[0] as ReturnType).connected = false; + liveBus.recover(); + expect(made).toHaveLength(2); + // A call that fails on its connection marks the client failed, connected or not. + const failing = made[1] as ReturnType; + failing.send = async () => { + throw Object.assign(new Error('Connection has failed'), { + code: 'ERR_REDIS_CONNECTION_CLOSED', + }); + }; + await expect(liveBus.lease('x', 1000)).rejects.toThrow(); + liveBus.recover(); + expect(made).toHaveLength(3); + }); + + it('a client that gave up is replaced: the subscriber counts as lost, the command connection is rebuilt', async () => { + const made: ReturnType[] = []; + const liveBus = new ValkeyLiveBus(() => { + const connection = fakeConnection(); + made.push(connection); + return connection; + }); + const commands = made[0] as ReturnType; + await liveBus.subscribe(() => {}); + const subscriber = made[1] as ReturnType; + // Bun after a long outage: both connections failed for good, no reconnect. + commands.onconnect?.(); + commands.connected = false; + subscriber.connected = false; + subscriber.onclose?.(); + expect(liveBus.isSubscribed()).toBe(false); + liveBus.recover(); + expect(commands.closed).toBe(true); + expect(made).toHaveLength(3); // a fresh command connection + // A healthy connection is left alone. + liveBus.recover(); + expect(made).toHaveLength(3); + }); +}); + +describe('the bus wire format', () => { + it('round-trips a message, compressed, and refuses junk', () => { + const message = { + type: 'poll' as const, + from: 'a', + sourceId: 'test-bus', + entities: Array.from({ length: 200 }, (_, i) => bus(String(i), T0)), + lastSuccessAt: T0, + epoch: 7, + }; + const text = encodeMessage(message); + expect(decodeMessage(text)).toEqual(message); + // 200 buses of near-identical JSON compress well: the bus carries a fraction. + expect(text.length).toBeLessThan(JSON.stringify(message).length / 5); + expect(decodeMessage('not base64 gzip')).toBeNull(); + expect(decodeMessage(Buffer.from('{}').toString('base64'))).toBeNull(); + }); +}); + +describe('several tasks on one bus (in memory)', () => { + const setup = () => { + const clock = { now: T0 }; + const network = new MemoryLiveNetwork(() => clock.now); + let fleet = [bus('1', T0)]; + const answer = (): LivePollResult => ({ entities: fleet, rejected: 0 }); + const tasks = ['a', 'b', 'c'].map((id) => + task(network.bus(id), () => clock.now, answer, { leaseMs: 15_000, demandTtlMs: 10_000 }), + ); + const everyone = async (step: (t: Task) => Promise) => { + for (const t of tasks) await step(t); + }; + return { + clock, + network, + tasks, + everyone, + setFleet: (next: LiveEntity[]) => { + fleet = next; + }, + }; + }; + + it('polls once for the whole fleet, and every task serves what the leader got', async () => { + const { tasks, everyone } = setup(); + const [a, b, c] = tasks as [Task, Task, Task]; + await everyone((t) => t.hub.connect()); + // Only task c has a viewer. + c.hub.watch('viewer', [ROOM]); + await everyone((t) => t.hub.coordinate()); + await everyone((t) => t.hub.coordinate()); + expect(tasks.filter((t) => t.hub.isLeader('test-bus'))).toHaveLength(1); + await everyone((t) => t.hub.tick()); + // One upstream request, from whichever task holds the lease. + expect(a.polls + b.polls + c.polls).toBe(1); + // Every task holds the answer; c's viewer got the delta. + for (const t of tasks) expect(t.hub.snapshot(ROOM.tile, 'transit').entities).toHaveLength(1); + expect(c.deltas.at(-1)?.upserts.map((e) => e.id)).toEqual(['test-bus:1']); + // Nobody else had a viewer: nothing published to their sockets. + expect(a.deltas).toEqual([]); + expect(b.deltas).toEqual([]); + }); + + it('a new leader takes over when the old one dies, and polls at once', async () => { + const { clock, tasks, everyone } = setup(); + await everyone((t) => t.hub.connect()); + for (const t of tasks) t.hub.watch('viewer', [ROOM]); + await everyone((t) => t.hub.coordinate()); + const leader = tasks.find((t) => t.hub.isLeader('test-bus')) as Task; + await everyone((t) => t.hub.tick()); + expect(leader.polls).toBe(1); + // The leader dies: no release, no renewal. The others keep coordinating. + const survivors = tasks.filter((t) => t !== leader); + clock.now += 5_000; + for (const t of survivors) await t.hub.coordinate(); + expect(survivors.some((t) => t.hub.isLeader('test-bus'))).toBe(false); // lease still held + clock.now += 11_000; // past the 15 s lease + for (const t of survivors) await t.hub.coordinate(); + const next = survivors.filter((t) => t.hub.isLeader('test-bus')); + expect(next).toHaveLength(1); + for (const t of survivors) await t.hub.tick(); + expect((next[0] as Task).polls).toBe(1); + expect(survivors.reduce((sum, t) => sum + t.polls, 0)).toBe(1); + }); + + it('a task that joins later is brought up to date by the leader', async () => { + const { clock, network, tasks, everyone } = setup(); + await everyone((t) => t.hub.connect()); + (tasks[0] as Task).hub.watch('viewer', [ROOM]); + await everyone((t) => t.hub.coordinate()); + await everyone((t) => t.hub.tick()); + const late = task( + network.bus('d'), + () => clock.now, + () => ({ entities: [], rejected: 0 }), + ); + expect(late.hub.snapshot(ROOM.tile, 'transit').entities).toHaveLength(0); + await late.hub.connect(); // subscribes, then asks for a sync + expect(late.hub.snapshot(ROOM.tile, 'transit').entities.map((e) => e.id)).toEqual([ + 'test-bus:1', + ]); + expect(late.polls).toBe(0); + }); + + it('polls locally while the bus is down, and shares again when it is back', async () => { + const { network, tasks, everyone } = setup(); + await everyone((t) => t.hub.connect()); + for (const t of tasks) t.hub.watch('viewer', [ROOM]); + await everyone((t) => t.hub.coordinate()); + network.up = false; + await everyone((t) => t.hub.coordinate()); + await everyone((t) => t.hub.tick()); + // Duplicate requests, but live keeps working everywhere. + expect(tasks.every((t) => t.polls === 1)).toBe(true); + network.up = true; + await everyone((t) => t.hub.coordinate()); + expect(tasks.filter((t) => t.hub.isLeader('test-bus'))).toHaveLength(1); + }); + + it('a follower that could not subscribe hears the leader once the bus is back', async () => { + const { network, tasks, everyone } = setup(); + const [a, b] = tasks as [Task, Task, Task]; + // Valkey is away when every task starts: nobody subscribes. + network.disconnect(); + await everyone((t) => t.hub.connect()); + a.hub.watch('viewer', [ROOM]); + b.hub.watch('viewer', [ROOM]); + network.up = true; + // a first, so a takes the lease; b follows. + await a.hub.coordinate(); + await b.hub.coordinate(); + expect(a.hub.isLeader('test-bus')).toBe(true); + expect(b.hub.isLeader('test-bus')).toBe(false); + await a.hub.tick(); + await b.hub.tick(); + expect(b.polls).toBe(0); + // b stopped polling, so it must be listening: the leader's answer reached it. + expect(b.hub.snapshot(ROOM.tile, 'transit').entities.map((e) => e.id)).toEqual(['test-bus:1']); + }); + + it('a subscription lost with its connection is restored, and the follower catches up', async () => { + const { clock, network, tasks, everyone, setFleet } = setup(); + const [a, b] = tasks as [Task, Task, Task]; + await everyone((t) => t.hub.connect()); + a.hub.watch('viewer', [ROOM]); + b.hub.watch('viewer', [ROOM]); + await a.hub.coordinate(); + await b.hub.coordinate(); + await a.hub.tick(); + // Valkey restarts: connections drop and come back without their subscriptions. + network.disconnect(); + network.up = true; + await a.hub.coordinate(); + await b.hub.coordinate(); + setFleet([bus('1', T0 + 10_000), bus('2', T0 + 10_000, 2.18)]); + clock.now += 10_000; + await a.hub.tick(); + expect( + b.hub + .snapshot(ROOM.tile, 'transit') + .entities.map((e) => e.id) + .sort(), + ).toEqual(['test-bus:1', 'test-bus:2']); + }); + + it('stopping during an outage still closes the bus', async () => { + const { network, tasks, everyone } = setup(); + const a = tasks[0] as Task; + await everyone((t) => t.hub.connect()); + a.hub.watch('viewer', [ROOM]); + network.up = false; + await a.hub.coordinate(); // the bus is down: a polls locally + a.hub.stop(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(network.closed.has('a')).toBe(true); + }); + + it('a bus that never answers counts as down within the deadline: the task polls locally', async () => { + const clock = { now: T0 }; + const network = new MemoryLiveNetwork(() => clock.now); + const a = task( + network.bus('a'), + () => clock.now, + () => ({ entities: [bus('1', T0)], rejected: 0 }), + { + leaseMs: 15_000, + callTimeoutMs: 40, + }, + ); + await a.hub.connect(); + a.hub.watch('viewer', [ROOM]); + await a.hub.coordinate(); + // A silent partition: every call hangs. + network.hang = true; + const started = Date.now(); + const settled = await Promise.race([ + a.hub.coordinate().then(() => 'settled'), + new Promise((resolve) => setTimeout(() => resolve('hung'), 2_000)), + ]); + expect(settled).toBe('settled'); + expect(Date.now() - started).toBeLessThan(1_000); + expect(network.recoveries).toBeGreaterThan(0); + await a.hub.tick(); + expect(a.polls).toBe(1); + }); + + it('shares source ids decided on the exact rooms: a viewer in Philadelphia wants no New York feed', async () => { + // The real catalogue's New York feeds, their upstreams faked and counted. + const ids = ['mta-subway', 'mta-lirr', 'mta-mnr', 'mta-bus']; + const definitions = ids.map( + (id) => LIVE_SOURCES.find((d) => d.id === id) as LiveSourceDefinition, + ); + const network = new MemoryLiveNetwork(() => T0); + const upstream: string[] = []; + const make = (taskId: string) => + new LiveHub({ + sources: definitions.map((definition) => ({ + definition: { + ...definition, + create: () => ({ + poll: async () => { + upstream.push(`poll ${definition.id}`); + return { entities: [], rejected: 0 }; + }, + }), + }, + settings: { url: definition.defaultUrl, userAgent: 'test', maxRegionsPerPoll: 1 }, + })), + fetch: async (url) => { + upstream.push(`static ${url}`); + throw new Error('not fetched in this test'); + }, + now: () => T0, + bus: network.bus(taskId), + publish: () => {}, + }); + const a = make('a'); + const b = make('b'); + await a.connect(); + await b.connect(); + const room = (latitude: number, longitude: number) => ({ + tile: liveTileOf({ latitude, longitude }, 'transit'), + kind: 'transit' as const, + }); + // Philadelphia: the same z6 tile as New York, ~130 km from the MTA's coverage. + a.watch('viewer', [room(39.9526, -75.1652)]); + for (let round = 0; round < 2; round++) { + await a.coordinate(); + await b.coordinate(); + } + await a.tick(); + await b.tick(); + expect(network.demand.get('a')?.rooms).toEqual([]); + expect(ids.some((id) => a.isLeader(id) || b.isLeader(id))).toBe(false); + expect(upstream).toEqual([]); + + // Manhattan: wanted, polled by one task; the other — no viewer of its own — + // reads no static GTFS. + a.watch('viewer', [room(40.758, -73.9855)]); + for (let round = 0; round < 2; round++) { + await a.coordinate(); + await b.coordinate(); + } + expect([...(network.demand.get('a')?.rooms ?? [])].sort()).toEqual([...ids].sort()); + expect(ids.every((id) => a.isLeader(id) !== b.isLeader(id))).toBe(true); + upstream.length = 0; + await b.tick(); + expect(upstream.filter((entry) => entry.startsWith('static'))).toEqual([]); + a.stop(); + b.stop(); + }); + + it('a regional view wakes the rail feeds, not the buses; only a street view reads static GTFS', async () => { + const ids = ['mta-subway', 'mta-lirr', 'mta-mnr', 'mta-bus']; + const definitions = ids.map( + (id) => LIVE_SOURCES.find((d) => d.id === id) as LiveSourceDefinition, + ); + const network = new MemoryLiveNetwork(() => T0); + const statics: Record = { a: [], b: [] }; + const make = (taskId: 'a' | 'b') => + new LiveHub({ + sources: definitions.map((definition) => ({ + definition: { + ...definition, + create: () => ({ poll: async () => ({ entities: [], rejected: 0 }) }), + }, + settings: { url: definition.defaultUrl, userAgent: 'test', maxRegionsPerPoll: 1 }, + })), + fetch: async (url) => { + statics[taskId]?.push(url); + throw new Error('not fetched in this test'); + }, + now: () => T0, + bus: network.bus(taskId), + publish: () => {}, + }); + const a = make('a'); + const b = make('b'); + await a.connect(); + await b.connect(); + const midtown = { latitude: 40.758, longitude: -73.9855 }; + const at = (zoom: number) => ({ + tile: liveTileOf(midtown, 'transit', zoom), + kind: 'transit' as const, + }); + // A regional view (z9 rooms): trains and the subway, no buses. + a.watch('viewer', [at(9)]); + for (let round = 0; round < 2; round++) { + await a.coordinate(); + await b.coordinate(); + } + expect([...(network.demand.get('a')?.rooms ?? [])].sort()).toEqual( + ['mta-lirr', 'mta-mnr', 'mta-subway'].sort(), + ); + expect(a.isLeader('mta-bus') || b.isLeader('mta-bus')).toBe(false); + // Another task's regional viewer: it leads nothing and asks no shapes. + b.watch('viewer', [at(9)]); + await b.tick(); + expect(statics.b).toEqual([]); + // Its viewer zooms into the street: now it may be asked for shapes. + b.watch('viewer', [at(11)]); + await b.tick(); + expect(statics.b?.length).toBeGreaterThan(0); + a.stop(); + b.stop(); + }); + + it('a regional feed shares z6 regions, never a room', () => { + const aircraft = LIVE_SOURCES.find((d) => d.id === 'adsb-lol') as LiveSourceDefinition; + const box = { west: 2.16, south: 41.38, east: 2.18, north: 41.4 }; + expect(demandEntries([aircraft], () => [box])).toEqual(['adsb-lol@6/32/23']); + const renfe = LIVE_SOURCES.find((d) => d.id === 'renfe-cercanias') as LiveSourceDefinition; + expect(demandEntries([renfe, aircraft], (d) => (d === renfe ? [box] : []))).toEqual([ + 'renfe-cercanias', + ]); + }); + + it('drops a message from an older leader delivered after a newer one', async () => { + const { tasks, everyone } = setup(); + const follower = tasks[1] as Task; + await everyone((t) => t.hub.connect()); + follower.hub.watch('viewer', [ROOM]); + const message = (epoch: number, id: string) => ({ + type: 'poll' as const, + from: 'someone', + sourceId: 'test-bus', + entities: [bus(id, T0)], + lastSuccessAt: T0, + epoch, + }); + follower.hub.receive(message(5, 'newer')); + // A publish abandoned at its deadline by the previous holder, arriving late. + follower.hub.receive(message(4, 'stale')); + expect(follower.hub.snapshot(ROOM.tile, 'transit').entities.map((e) => e.id)).toEqual([ + 'test-bus:newer', + ]); + }); + + it('hears a leader whose epoch went backwards once the newer one has been silent a whole lease', async () => { + const { clock, tasks, everyone } = setup(); + const follower = tasks[1] as Task; + await everyone((t) => t.hub.connect()); + follower.hub.watch('viewer', [ROOM]); + const message = (epoch: number, id: string) => ({ + type: 'poll' as const, + from: 'someone', + sourceId: 'test-bus', + entities: [bus(id, clock.now)], + lastSuccessAt: clock.now, + epoch, + }); + follower.hub.receive(message(9, 'before')); + // A failover lost the counter: the only leader now stamps a lower epoch. + clock.now += 5_000; + follower.hub.receive(message(2, 'after')); + const ids = () => follower.hub.snapshot(ROOM.tile, 'transit').entities.map((e) => e.id); + expect(ids()).toEqual(['test-bus:before']); + clock.now += 15_000; + follower.hub.receive(message(2, 'after')); + expect(ids()).toEqual(['test-bus:after']); + }); + + it('sends a vehicle that only moved as a compact move, not the whole entity', async () => { + const { clock, tasks, everyone, setFleet } = setup(); + await everyone((t) => t.hub.connect()); + for (const t of tasks) t.hub.watch('viewer', [ROOM]); + await everyone((t) => t.hub.coordinate()); + await everyone((t) => t.hub.tick()); + setFleet([bus('1', T0 + 10_000, 2.171)]); + clock.now += 10_000; + await everyone((t) => t.hub.tick()); + for (const t of tasks) { + const last = t.deltas.at(-1); + expect(last?.upserts).toEqual([]); + expect(last?.moves).toEqual([ + ['test-bus:1', 41.39, 2.171, 90, 8, new Date(T0 + 10_000).toISOString(), null, null], + ]); + } + }); +}); + +// --------------------------------------------------------------------------- +// A real Valkey +// --------------------------------------------------------------------------- + +const VALKEY = process.env.TEST_VALKEY_URL; +const opened: LiveBus[] = []; +afterAll(async () => { + for (const b of opened) await b.close().catch(() => {}); +}); + +describe.if(Boolean(VALKEY))('several tasks on a real Valkey', () => { + const prefix = `goway:live:test:${crypto.randomUUID().slice(0, 8)}`; + const open = (id: string) => { + const b = ValkeyLiveBus.fromUrl(VALKEY as string, `${prefix}`); + Object.defineProperty(b, 'taskId', { value: id }); + opened.push(b); + return b; + }; + const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + + it('one poller, fan-out to every task, failover after the lease', async () => { + const answer = (): LivePollResult => ({ entities: [bus('1', Date.now())], rejected: 0 }); + const timings = { leaseMs: 800, demandTtlMs: 2_000 }; + const tasks = ['x', 'y', 'z'].map((id) => task(open(id), Date.now, answer, timings)); + for (const t of tasks) await t.hub.connect(); + (tasks[2] as Task).hub.watch('viewer', [ROOM]); + for (let round = 0; round < 2; round++) for (const t of tasks) await t.hub.coordinate(); + const leaders = tasks.filter((t) => t.hub.isLeader('test-bus')); + expect(leaders).toHaveLength(1); + for (const t of tasks) await t.hub.tick(); + await sleep(150); // pub/sub delivery + expect(tasks.reduce((sum, t) => sum + t.polls, 0)).toBe(1); + for (const t of tasks) expect(t.hub.snapshot(ROOM.tile, 'transit').entities).toHaveLength(1); + + // The leader crashes: its connections close without releasing the lease. + const leader = leaders[0] as Task; + const survivors = tasks.filter((t) => t !== leader); + await ( + opened.find((b) => b.taskId === ['x', 'y', 'z'][tasks.indexOf(leader)]) as LiveBus + ).close(); + await sleep(timings.leaseMs + 200); + for (const t of survivors) await t.hub.coordinate(); + const next = survivors.filter((t) => t.hub.isLeader('test-bus')); + expect(next).toHaveLength(1); + for (const t of survivors) await t.hub.tick(); + expect((next[0] as Task).polls).toBe(1); + await sleep(150); + for (const t of survivors) + expect(t.hub.snapshot(ROOM.tile, 'transit').entities).toHaveLength(1); + }); + + it('stamps leases from a per-source counter, never a clock', async () => { + const first = open('first'); + const second = open('second'); + const one = await first.lease('counted', 60_000); + expect(await first.lease('counted', 60_000)).toBe(one); // renewing keeps it + expect(await second.lease('counted', 60_000)).toBe(0); // held + await first.release('counted'); + expect(await second.lease('counted', 60_000)).toBe(one + 1); + // Small integers, not milliseconds since 1970 — and the counter never + // expires, so the fleet Valkey's volatile-lru cannot evict it. + expect(one).toBeLessThan(1_000_000); + const { RedisClient } = await import('bun'); + const raw = new RedisClient(VALKEY as string); + expect(Number(await raw.send('PTTL', [`${prefix}:epoch:counted`]))).toBe(-1); + raw.close(); + }); +}); + +// --------------------------------------------------------------------------- +// A real outage: a Valkey (or Redis) server this test starts, stops and starts +// again. Needs the server binary (`valkey-server` or `redis-server`; CI +// installs one) — skipped without it. +// --------------------------------------------------------------------------- + +const SERVER_BIN = Bun.which('valkey-server') ?? Bun.which('redis-server'); + +describe.if(Boolean(SERVER_BIN))('a Valkey outage of 10 s', () => { + it('leaves one leader per source and followers hearing again once Valkey is back', async () => { + const port = 50_000 + Math.floor(Math.random() * 10_000); + const url = `redis://127.0.0.1:${port}`; + const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + let server: ReturnType | null = null; + const startServer = async () => { + server = Bun.spawn( + [SERVER_BIN as string, '--port', String(port), '--save', '', '--appendonly', 'no'], + { + stdout: 'ignore', + stderr: 'ignore', + }, + ); + for (let i = 0; i < 50; i++) { + const probe = new (await import('bun')).RedisClient(url); + try { + await probe.send('PING', []); + probe.close(); + return; + } catch { + probe.close(); + await sleep(100); + } + } + throw new Error('server did not start'); + }; + const stopServer = async () => { + server?.kill('SIGKILL'); + await server?.exited; + }; + await startServer(); + try { + let fleet = [bus('before', Date.now())]; + const answer = (): LivePollResult => ({ entities: fleet, rejected: 0 }); + const timings = { leaseMs: 3_000, demandTtlMs: 4_000, callTimeoutMs: 1_000 }; + const prefix = `goway:live:outage:${crypto.randomUUID().slice(0, 8)}`; + const tasks = ['p', 'q'].map((id) => { + const b = ValkeyLiveBus.fromUrl(url, prefix); + Object.defineProperty(b, 'taskId', { value: id }); + opened.push(b); + return task(b, Date.now, answer, timings); + }); + const everyone = async (step: (t: Task) => Promise) => { + await Promise.all(tasks.map(step)); + }; + await everyone((t) => t.hub.connect()); + for (const t of tasks) t.hub.watch('viewer', [ROOM]); + await everyone((t) => t.hub.coordinate()); + await everyone((t) => t.hub.coordinate()); + expect(tasks.filter((t) => t.hub.isLeader('test-bus'))).toHaveLength(1); + + // Valkey goes away for 10 s; the tasks keep coordinating (and fail). + await stopServer(); + const outageEnds = Date.now() + 10_000; + while (Date.now() < outageEnds) { + await everyone((t) => t.hub.coordinate()); + await sleep(500); + } + // Down: every task polls on its own rather than nobody. + expect(tasks.every((t) => t.hub.isLeader('test-bus'))).toBe(true); + + await startServer(); + // Back: within a few rounds, ONE poller again. + let leaders = tasks.length; + for (let round = 0; round < 30 && leaders !== 1; round++) { + await everyone((t) => t.hub.coordinate()); + leaders = tasks.filter((t) => t.hub.isLeader('test-bus')).length; + if (leaders !== 1) await sleep(300); + } + expect(leaders).toBe(1); + // And the follower hears the leader again. + const leader = tasks.find((t) => t.hub.isLeader('test-bus')) as Task; + const follower = tasks.find((t) => t !== leader) as Task; + fleet = [bus('after', Date.now())]; + const pollsBefore = follower.polls; + await sleep(11_000); // the leader's next poll is due (10 s interval) + await everyone((t) => t.hub.tick()); + await sleep(300); + expect(follower.polls).toBe(pollsBefore); + expect(follower.hub.snapshot(ROOM.tile, 'transit').entities.map((e) => e.id)).toEqual([ + 'test-bus:after', + ]); + for (const t of tasks) t.hub.stop(); + } finally { + await stopServer(); + } + }, 60_000); +}); diff --git a/packages/backend/src/live/__tests__/hub.test.ts b/packages/backend/src/live/__tests__/hub.test.ts index 579b981d..e79dd1c4 100644 --- a/packages/backend/src/live/__tests__/hub.test.ts +++ b/packages/backend/src/live/__tests__/hub.test.ts @@ -16,6 +16,7 @@ import { liveTileBounds, liveTileOf, liveTilesFor, + liveTilesOf, } from '@goway/contracts'; import { LiveHub, MAX_BACKOFF_MS } from '../hub'; import { contains, LiveTileIndex } from '../tileIndex'; @@ -56,23 +57,52 @@ const STATIC_FILES: Record = { 'https://gtfs.example/shapes.txt': 'shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence\nline,41.39,2.16,1\nline,41.39,2.18,2\n', }; -const BARCELONA_TILE = liveTileOf(BARCELONA); -const NEW_YORK_TILE = liveTileOf({ latitude: 40.75, longitude: -73.98 }); +const BARCELONA_TILE = liveTileOf(BARCELONA, 'transit'); +const BARCELONA_AIR_TILE = liveTileOf(BARCELONA, 'aircraft'); +const NEW_YORK_TILE = liveTileOf({ latitude: 40.75, longitude: -73.98 }, 'transit'); // --------------------------------------------------------------------------- // Room tiles (contracts) and the index // --------------------------------------------------------------------------- describe('room tiles', () => { - it('puts a city in one or two z7 rooms', () => { - expect(BARCELONA_TILE).toBe('7/64/47'); - const city = liveTilesFor({ west: 2.05, south: 41.32, east: 2.25, north: 41.47 }); - expect(city).toEqual(['7/64/47']); + it('keys each kind at its own zoom: a city is one aircraft room, a few transit ones', () => { + expect(BARCELONA_AIR_TILE).toBe('7/64/47'); + const city = { west: 2.05, south: 41.32, east: 2.25, north: 41.47 }; + expect(liveTilesFor(city, 'aircraft')).toEqual(['7/64/47']); + expect(liveTilesFor(city, 'vessel')).toEqual(['7/64/47']); + const transit = liveTilesFor(city, 'transit') ?? []; + expect(transit.length).toBeGreaterThan(1); + expect(transit.every((tile) => tile.startsWith('11/'))).toBe(true); + expect(BARCELONA_TILE.startsWith('11/')).toBe(true); + // Bike share: a neighbourhood at street zoom; a whole city is too wide. + expect(liveTilesFor(city, 'bikeshare')).toBeNull(); + const street = { west: 2.16, south: 41.385, east: 2.175, north: 41.395 }; + expect((liveTilesFor(street, 'bikeshare') ?? []).every((t) => t.startsWith('13/'))).toBe(true); + }); + + it('serves a whole region from coarser rooms: all of Catalonia, every train', () => { + const catalonia = { west: 0.2, south: 40.6, east: 3.3, north: 42.8 }; + const tiles = liveTilesFor(catalonia, 'transit') ?? []; + expect(tiles.length).toBeGreaterThan(0); + expect(tiles.length).toBeLessThanOrEqual(LIVE_MAX_ROOM_TILES); + // Catalonia is 25 z9 rooms: it takes the z7 ones (4). + expect(tiles).toEqual(['7/64/47', '7/64/48', '7/65/47', '7/65/48']); + // The Barcelona region (~80 km) takes z9 rooms; a city view z11. + const metro = liveTilesFor({ west: 1.6, south: 41.1, east: 2.6, north: 41.8 }, 'transit') ?? []; + expect(metro.every((tile) => tile.startsWith('9/'))).toBe(true); + // Girona and Tarragona, far from Barcelona, are inside the region's rooms. + for (const [latitude, longitude] of [ + [41.98, 2.82], + [41.12, 1.25], + ] as const) { + expect(tiles).toContain(liveTileOf({ latitude, longitude }, 'transit', 7)); + } }); it('refuses a box wider than the room budget, and crosses the antimeridian', () => { - expect(liveTilesFor({ west: -30, south: 20, east: 30, north: 60 })).toBeNull(); - const pacific = liveTilesFor({ west: 179, south: -1, east: -179, north: 1 }); + expect(liveTilesFor({ west: -30, south: 20, east: 30, north: 60 }, 'aircraft')).toBeNull(); + const pacific = liveTilesFor({ west: 179, south: -1, east: -179, north: 1 }, 'aircraft'); expect(pacific).toEqual(['7/127/63', '7/127/64', '7/0/63', '7/0/64']); expect((pacific ?? []).length).toBeLessThanOrEqual(LIVE_MAX_ROOM_TILES); }); @@ -81,32 +111,62 @@ describe('room tiles', () => { const box = liveTileBounds(BARCELONA_TILE); expect(contains(box, BARCELONA)).toBe(true); const centre = { latitude: (box.north + box.south) / 2, longitude: (box.east + box.west) / 2 }; - expect(liveTileOf(centre)).toBe(BARCELONA_TILE); + expect(liveTileOf(centre, 'transit')).toBe(BARCELONA_TILE); }); }); describe('the tile index', () => { - it('reports changes, ignores repeats, and moves between rooms', () => { + it('reports changes, ignores repeats, and moves between rooms — at every level', () => { const index = new LiveTileIndex(); - const first = bus('1', 41.39, 2.17, T0); - expect(index.upsert(first)).toEqual([ - { type: 'upsert', room: `${BARCELONA_TILE}|transit`, entity: first }, - ]); + const train = (latitude: number, longitude: number, at: number) => + ({ ...bus('1', latitude, longitude, at), mode: 'rail' }) as LiveEntity; + const first = train(41.39, 2.17, T0); + const rooms = (latitude: number, longitude: number) => + liveTilesOf({ latitude, longitude }, 'transit', 'rail').map((tile) => `${tile}|transit`); + const barcelona = rooms(41.39, 2.17); + const newYork = rooms(40.75, -73.98); + // A train is in one room per transit level: z7, z9, z11. + expect(barcelona.map((room) => room.split('/')[0])).toEqual(['7', '9', '11']); + expect(index.upsert(first)).toEqual( + barcelona.map((room) => ({ type: 'upsert', room, entity: first })), + ); expect(index.upsert({ ...first })).toEqual([]); - const moved = bus('1', 40.75, -73.98, T0 + 1000); + const moved = train(40.75, -73.98, T0 + 1000); expect(index.upsert(moved)).toEqual([ - { type: 'remove', room: `${BARCELONA_TILE}|transit`, id: 'test-bus:1' }, - { type: 'upsert', room: `${NEW_YORK_TILE}|transit`, entity: moved }, + ...barcelona.map((room) => ({ type: 'remove' as const, room, id: 'test-bus:1' })), + ...newYork.map((room) => ({ type: 'upsert' as const, room, entity: moved })), ]); expect(index.inRoom(BARCELONA_TILE, 'transit')).toEqual([]); expect(index.inRoom(NEW_YORK_TILE, 'transit')).toEqual([moved]); + expect(index.inRoom((newYork[0] as string).split('|')[0] as string, 'transit')).toEqual([ + moved, + ]); expect(index.inRoom(NEW_YORK_TILE, 'aircraft')).toEqual([]); expect(index.countOfSource('test-bus')).toBe(1); - expect(index.remove('test-bus:1')).toHaveLength(1); + expect(index.remove('test-bus:1')).toHaveLength(3); expect(index.size).toBe(0); expect(index.countOfSource('test-bus')).toBe(0); }); + it('puts a bus in its city-zoom room only, and moves a vehicle between levels when its mode is learnt', () => { + const index = new LiveTileIndex(); + const [z7, z9, z11] = liveTilesOf({ latitude: 41.39, longitude: 2.17 }, 'transit', 'rail'); + const street = bus('1', 41.39, 2.17, T0); + expect(index.upsert(street).map((change) => change.room)).toEqual([`${z11}|transit`]); + expect(index.inRoom(z7 as string, 'transit')).toEqual([]); + expect(index.inRoom(z9 as string, 'transit')).toEqual([]); + // Its operator's routes.txt says it is a train: now regional too. + const train = { ...street, mode: 'rail' } as LiveEntity; + expect( + index.upsert(train).map((change) => `${change.type} ${change.room.split('/')[0]}`), + ).toEqual(['upsert 7', 'upsert 9', 'upsert 11']); + // And back: it leaves the regional rooms. + expect( + index.upsert(street).map((change) => `${change.type} ${change.room.split('/')[0]}`), + ).toEqual(['remove 7', 'remove 9', 'upsert 11']); + expect(index.inRoom(z9 as string, 'transit')).toEqual([]); + }); + it('answers a box query exactly, by kind', () => { const index = new LiveTileIndex(); index.upsert(bus('in', 41.39, 2.17, T0)); @@ -199,7 +259,7 @@ describe('the hub', () => { expect(h.polls).toHaveLength(0); // Aircraft over Barcelona, and buses over New York: neither is this source's demand. h.hub.watch('a', [ - { tile: BARCELONA_TILE, kind: 'aircraft' }, + { tile: BARCELONA_AIR_TILE, kind: 'aircraft' }, { tile: NEW_YORK_TILE, kind: 'transit' }, ]); await h.hub.tick(); diff --git a/packages/backend/src/live/__tests__/socket.test.ts b/packages/backend/src/live/__tests__/socket.test.ts index cbf416c4..d0b1008e 100644 --- a/packages/backend/src/live/__tests__/socket.test.ts +++ b/packages/backend/src/live/__tests__/socket.test.ts @@ -11,6 +11,7 @@ import '../../__tests__/testEnv'; import { afterEach, describe, expect, it } from 'bun:test'; +import { gunzipSync } from 'node:zlib'; import { createServer, type Server as HttpServer } from 'node:http'; import type { AddressInfo } from 'node:net'; import { @@ -163,12 +164,17 @@ describe('the /live namespace', () => { zoom: 12, kinds: ['transit', 'aircraft'], }); - expect(liveSubscriptionStateSchema.parse(await answer)).toEqual({ - state: 'ok', - tiles: [liveTileOf({ latitude: 41.39, longitude: 2.17 })], - kinds: ['transit', 'aircraft'], - retryAfterSeconds: null, - }); + const state = liveSubscriptionStateSchema.parse(await answer); + expect(state).toMatchObject({ state: 'ok', retryAfterSeconds: null }); + // Each kind's rooms at its own zoom, listed per kind: transit z11, + // aircraft z7 — the aircraft tile is not a transit room. + expect(state.tiles.transit).toContain( + liveTileOf({ latitude: 41.39, longitude: 2.17 }, 'transit'), + ); + expect(state.tiles.aircraft).toEqual([ + liveTileOf({ latitude: 41.39, longitude: 2.17 }, 'aircraft'), + ]); + expect(state.tiles.transit).not.toContain(state.tiles.aircraft?.[0]); // Nothing polled yet: an empty, authoritative snapshot. expect(liveSnapshotSchema.parse(await snapshot).entities).toEqual([]); @@ -176,7 +182,7 @@ describe('the /live namespace', () => { await server.live.hub.tick(); const received = liveDeltaSchema.parse(await delta); expect(received.upserts.map((entity) => entity.id)).toEqual(['renfe-cercanias:1']); - expect(received.tile).toBe('7/64/47'); + expect(received.tile).toBe(liveTileOf({ latitude: 41.39, longitude: 2.17 }, 'transit')); }); it('sends a room only to its members', async () => { @@ -214,7 +220,7 @@ describe('the /live namespace', () => { zoom: 3, kinds: ['transit'], }); - expect(await answer).toMatchObject({ state: 'too_wide', tiles: [], kinds: [] }); + expect(await answer).toMatchObject({ state: 'too_wide', tiles: {} }); for (const junk of [ null, @@ -226,10 +232,30 @@ describe('the /live namespace', () => { ]) { answer = next(visitor, LIVE_SOCKET_EVENTS.subscription); visitor.emit(LIVE_SOCKET_EVENTS.subscribe, junk); - expect(await answer).toMatchObject({ state: 'invalid', tiles: [] }); + expect(await answer).toMatchObject({ state: 'invalid', tiles: {} }); } }); + it('sends a large message as gzip bytes that inflate to the same JSON', async () => { + const server = await start(); + server.feed.result = { + entities: Array.from({ length: 60 }, (_, i) => vehicle(String(i), 41.39, 2.17 + i * 1e-4)), + rejected: 0, + }; + const visitor = client(server); + await next(visitor, 'connect'); + visitor.emit(LIVE_SOCKET_EVENTS.subscribe, { ...BARCELONA_BOX, zoom: 12, kinds: ['transit'] }); + await next(visitor, LIVE_SOCKET_EVENTS.subscription); + const delta = next(visitor, LIVE_SOCKET_EVENTS.delta); + await server.live.hub.tick(); + const packed = await delta; + expect(packed).toBeInstanceOf(Buffer); + const json = gunzipSync(packed as Buffer).toString('utf8'); + expect(liveDeltaSchema.parse(JSON.parse(json)).upserts).toHaveLength(60); + // Several times smaller on the wire. + expect((packed as Buffer).length).toBeLessThan(json.length / 5); + }); + it('rate-limits subscriptions and keeps the rooms it had', async () => { const server = await start({ subscribeBurst: 2, subscribePerMinute: 6 }); const visitor = client(server); @@ -244,7 +270,8 @@ describe('the /live namespace', () => { } await new Promise((resolve) => setTimeout(resolve, 200)); expect(answers.map((answer) => answer.state)).toEqual(['ok', 'ok', 'rate_limited']); - expect(answers[2]).toMatchObject({ tiles: ['7/64/47'], retryAfterSeconds: 10 }); + expect(answers[2]).toMatchObject({ tiles: answers[1]?.tiles, retryAfterSeconds: 10 }); + expect(answers[2]?.tiles.transit?.length).toBeGreaterThan(0); }); it('refuses the handshake beyond the connection cap', async () => { @@ -292,6 +319,51 @@ describe('live configuration', () => { }); }); + it('shares polling through Valkey when REDIS_URL is set', () => { + const listed = { LIVE_SOURCES: 'adsb-lol' }; + expect(parseLiveConfig(listed, ['adsb-lol']).valkeyUrl).toBeNull(); + expect( + parseLiveConfig({ ...listed, REDIS_URL: 'redis://cache.internal:6379' }, ['adsb-lol']) + .valkeyUrl, + ).toBe('redis://cache.internal:6379'); + // One name for it: no second variable kept in case. + expect( + parseLiveConfig({ ...listed, LIVE_VALKEY_URL: 'redis://other:6379' }, ['adsb-lol']).valkeyUrl, + ).toBeNull(); + }); + + it('a malformed shared REDIS_URL never stops the backend booting: live runs without a bus', () => { + const warnings: Record[] = []; + for (const REDIS_URL of [ + 'valkey://cache:6379', + 'cache.internal:6379', + 'https://user:secret@x', + ]) { + const config = parseLiveConfig({ LIVE_SOURCES: 'renfe-cercanias', REDIS_URL }, [ + 'renfe-cercanias', + ]); + expect(config).toMatchObject({ sources: ['renfe-cercanias'], valkeyUrl: null }); + expect(config.valkeyUrlProblem).toContain('REDIS_URL'); + const service = createLiveService({ + config, + logger: { info: () => {}, warn: (object) => warnings.push(object) }, + }); + expect(service).not.toBeNull(); + service?.stop(); + } + // Reported, never echoed: it may carry a password. + expect(warnings).toHaveLength(3); + expect(JSON.stringify(warnings)).not.toContain('secret'); + expect(JSON.stringify(warnings)).not.toContain('cache.internal'); + }); + + it('with no source listed (live off), the bus setting cannot stop the backend booting', () => { + for (const LIVE_SOURCES of [undefined, '', ' ']) { + const config = parseLiveConfig({ LIVE_SOURCES, REDIS_URL: 'not-a-url' }, ['adsb-lol']); + expect(config).toMatchObject({ sources: [], valkeyUrl: null }); + } + }); + it('is on exactly when a source is listed: no switch, and the retired one is ignored', () => { const logger = { info: () => {}, warn: () => {} }; expect(createLiveService({ config: parseLiveConfig({}, []), logger })).toBeNull(); diff --git a/packages/backend/src/live/areas.generated.ts b/packages/backend/src/live/areas.generated.ts new file mode 100644 index 00000000..8fef232f --- /dev/null +++ b/packages/backend/src/live/areas.generated.ts @@ -0,0 +1,359 @@ +// Generated by `bun run --cwd packages/backend live:areas` (`runAreas.ts`): do not edit. +import type { SourceAreas } from './areas'; + +export const LIVE_SOURCE_AREAS: Readonly> = { + "renfe-cercanias": { + "outer": { + "west": -8.28, + "south": 36.36, + "east": 3.21, + "north": 43.79 + }, + "areas": [ + { + "west": -8.28, + "south": 43.44, + "east": -8.15, + "north": 43.55 + }, + { + "west": -8.25, + "south": 43.45, + "east": -5.63, + "north": 43.79 + }, + { + "west": -7.11, + "south": 43.43, + "east": -6.99, + "north": 43.54 + }, + { + "west": -6.46, + "south": 38.04, + "east": -5.75, + "north": 38.5 + }, + { + "west": -6.45, + "south": 38.51, + "east": -6.29, + "north": 39.51 + }, + { + "west": -6.43, + "south": 39.74, + "east": -4.23, + "north": 40.03 + }, + { + "west": -6.34, + "south": 36.36, + "east": -6.02, + "north": 36.96 + }, + { + "west": -6.26, + "south": 36.97, + "east": -5.74, + "north": 37.54 + }, + { + "west": -6.16, + "south": 42.95, + "east": -1.75, + "north": 43.54 + }, + { + "west": -6.15, + "south": 39.97, + "east": -6.05, + "north": 40.07 + }, + { + "west": -5.92, + "south": 37.5, + "east": -5.48, + "north": 37.98 + }, + { + "west": -5.76, + "south": 42.49, + "east": -2.97, + "north": 43.05 + }, + { + "west": -5.55, + "south": 41.97, + "east": -3.62, + "north": 42.51 + }, + { + "west": -4.96, + "south": 41, + "east": -4.65, + "north": 41.37 + }, + { + "west": -4.81, + "south": 36.49, + "east": -4.38, + "north": 36.96 + }, + { + "west": -4.78, + "south": 41.59, + "east": -4.38, + "north": 42.03 + }, + { + "west": -4.73, + "south": 40.45, + "east": -2.79, + "north": 41 + }, + { + "west": -3.93, + "south": 39.99, + "east": -3.32, + "north": 40.55 + }, + { + "west": -2.84, + "south": 40.95, + "east": -1.59, + "north": 41.4 + }, + { + "west": -2.49, + "south": 42.01, + "east": -1.55, + "north": 42.97 + }, + { + "west": -1.83, + "south": 37.34, + "east": -1.52, + "north": 37.52 + }, + { + "west": -1.78, + "south": 37.53, + "east": -0.74, + "north": 38.02 + }, + { + "west": -1.57, + "south": 41.47, + "east": -0.49, + "north": 42.05 + }, + { + "west": -1.25, + "south": 38.98, + "east": 0, + "north": 40.04 + }, + { + "west": -1.05, + "south": 37.99, + "east": -0.11, + "north": 39.04 + }, + { + "west": -0.78, + "south": 42.08, + "east": -0.32, + "north": 42.8 + }, + { + "west": -0.54, + "south": 41.03, + "east": 2.4, + "north": 41.55 + }, + { + "west": 0.01, + "south": 40.01, + "east": 0.51, + "north": 40.52 + }, + { + "west": 0.4, + "south": 40.55, + "east": 0.96, + "north": 41.05 + }, + { + "west": 0.58, + "south": 41.45, + "east": 2.87, + "north": 42.05 + }, + { + "west": 1.85, + "south": 41.98, + "east": 3.21, + "north": 42.51 + } + ] + }, + "fgc-geotren": { + "outer": { + "west": 0.58, + "south": 41.3, + "east": 2.22, + "north": 42.45 + }, + "areas": [ + { + "west": 0.58, + "south": 41.57, + "east": 1.02, + "north": 42.29 + }, + { + "west": 1.58, + "south": 41.3, + "east": 2.22, + "north": 41.78 + }, + { + "west": 2.1, + "south": 42.25, + "east": 2.22, + "north": 42.45 + } + ] + }, + "mta-bus": { + "outer": { + "west": -74.3, + "south": 40.45, + "east": -73.65, + "north": 40.98 + }, + "areas": [ + { + "west": -74.3, + "south": 40.45, + "east": -73.65, + "north": 40.98 + } + ] + }, + "mta-lirr": { + "outer": { + "west": -74.04, + "south": 40.54, + "east": -71.9, + "north": 41.15 + }, + "areas": [ + { + "west": -74.04, + "south": 40.54, + "east": -72.08, + "north": 41.04 + }, + { + "west": -72.48, + "south": 41, + "east": -71.9, + "north": 41.15 + } + ] + }, + "mta-mnr": { + "outer": { + "west": -74.03, + "south": 40.7, + "east": -72.87, + "north": 41.86 + }, + "areas": [ + { + "west": -74.03, + "south": 40.7, + "east": -73.63, + "north": 41.04 + }, + { + "west": -74.03, + "south": 40.95, + "east": -72.87, + "north": 41.54 + }, + { + "west": -74.03, + "south": 41.45, + "east": -73, + "north": 41.86 + } + ] + }, + "mta-subway": { + "outer": { + "west": -74.3, + "south": 40.46, + "east": -73.71, + "north": 40.95 + }, + "areas": [ + { + "west": -74.3, + "south": 40.46, + "east": -73.71, + "north": 40.95 + } + ] + }, + "amb-bus": { + "outer": { + "west": 1.8, + "south": 41.21, + "east": 2.35, + "north": 41.58 + }, + "areas": [ + { + "west": 1.8, + "south": 41.21, + "east": 2.35, + "north": 41.58 + } + ] + }, + "emt-fuenlabrada": { + "outer": { + "west": -3.89, + "south": 40.23, + "east": -3.71, + "north": 40.36 + }, + "areas": [ + { + "west": -3.89, + "south": 40.23, + "east": -3.71, + "north": 40.36 + } + ] + }, + "nyc-bikeshare": { + "outer": { + "west": -74.15, + "south": 40.56, + "east": -73.78, + "north": 40.96 + }, + "areas": [ + { + "west": -74.15, + "south": 40.56, + "east": -73.78, + "north": 40.96 + } + ] + } +}; diff --git a/packages/backend/src/live/areas.ts b/packages/backend/src/live/areas.ts new file mode 100644 index 00000000..163e0e95 --- /dev/null +++ b/packages/backend/src/live/areas.ts @@ -0,0 +1,157 @@ +/** + * Where a source's vehicles can be: its operator's own stops, as a handful of + * tight boxes. A source's `coverage` is one box round all of it — Renfe's + * reaches from Galicia to Catalonia and so takes in Toulouse — and demand + * matched against that one box would wake a feed (and read its static GTFS) + * for a viewer hundreds of kilometres from its nearest station. Demand is + * matched against these areas instead (`hub.ts`). + * + * Derived, never drawn by hand: `bun run --cwd packages/backend live:areas` + * reads each source's static GTFS (`stops.txt`, or the shapes where a + * portal publishes no stops) or its GBFS stations and rewrites + * `areas.generated.ts`. The method: the points fall in 0.5° cells; each + * row's run of neighbouring cells, merged with the identical runs above and + * below it, is one box, shrunk to the points inside it and padded by + * {@link AREA_PAD_DEGREES}. A line of stations stays one chain of boxes; the + * empty land between two networks is in none. + */ + +import type { GeoBoundingBox } from '@goway/contracts'; +import { LIVE_SOURCE_AREAS } from './areas.generated'; + +/** The grid the points are bucketed on, in degrees (~55 km of latitude). */ +export const AREA_CELL_DEGREES = 0.5; +/** The margin round the points of each box (~5 km): a depot, a terminus loop. */ +export const AREA_PAD_DEGREES = 0.05; + +export interface SourceAreas { + /** One box round every area: the source's `coverage`. */ + outer: GeoBoundingBox; + areas: readonly GeoBoundingBox[]; +} + +const round = (value: number) => Math.round(value * 100) / 100; + +/** The areas of a set of points (`[latitude, longitude]`); `null` for none. */ +export function areasOf(points: readonly (readonly [number, number])[]): SourceAreas | null { + const cells = new Map(); + const rows = new Map>(); + for (const point of points) { + const [latitude, longitude] = point; + if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) continue; + if (Math.abs(latitude) > 85 || Math.abs(longitude) > 180) continue; + if (latitude === 0 && longitude === 0) continue; + const row = Math.floor((latitude + 90) / AREA_CELL_DEGREES); + const column = Math.floor((longitude + 180) / AREA_CELL_DEGREES); + const key = `${row}/${column}`; + let list = cells.get(key); + if (!list) { + list = []; + cells.set(key, list); + } + list.push(point); + let columns = rows.get(row); + if (!columns) { + columns = new Set(); + rows.set(row, columns); + } + columns.add(column); + } + if (cells.size === 0) return null; + + // Each row's runs of neighbouring columns. + const runs: { row: number; from: number; to: number }[] = []; + for (const [row, columns] of rows) { + const sorted = [...columns].sort((a, b) => a - b); + let from = sorted[0] as number; + let to = from; + for (const column of sorted.slice(1)) { + if (column === to + 1) { + to = column; + continue; + } + runs.push({ row, from, to }); + from = column; + to = column; + } + runs.push({ row, from, to }); + } + // The same run in consecutive rows is one block. + runs.sort((a, b) => a.from - b.from || a.to - b.to || a.row - b.row); + const blocks: { first: number; last: number; from: number; to: number }[] = []; + for (const run of runs) { + const previous = blocks[blocks.length - 1]; + if ( + previous && + previous.from === run.from && + previous.to === run.to && + previous.last === run.row - 1 + ) { + previous.last = run.row; + continue; + } + blocks.push({ first: run.row, last: run.row, from: run.from, to: run.to }); + } + + const areas: GeoBoundingBox[] = []; + for (const block of blocks) { + let west = Number.POSITIVE_INFINITY; + let south = Number.POSITIVE_INFINITY; + let east = Number.NEGATIVE_INFINITY; + let north = Number.NEGATIVE_INFINITY; + for (let row = block.first; row <= block.last; row++) { + for (let column = block.from; column <= block.to; column++) { + for (const [latitude, longitude] of cells.get(`${row}/${column}`) ?? []) { + west = Math.min(west, longitude); + east = Math.max(east, longitude); + south = Math.min(south, latitude); + north = Math.max(north, latitude); + } + } + } + areas.push({ + west: round(west - AREA_PAD_DEGREES), + south: round(south - AREA_PAD_DEGREES), + east: round(east + AREA_PAD_DEGREES), + north: round(north + AREA_PAD_DEGREES), + }); + } + areas.sort((a, b) => a.west - b.west || a.south - b.south); + return { + outer: { + west: Math.min(...areas.map((area) => area.west)), + south: Math.min(...areas.map((area) => area.south)), + east: Math.max(...areas.map((area) => area.east)), + north: Math.max(...areas.map((area) => area.north)), + }, + areas, + }; +} + +/** A source's derived areas, when its stops are known (`areas.generated.ts`). */ +export function sourceAreas(sourceId: string): SourceAreas | null { + return LIVE_SOURCE_AREAS[sourceId] ?? null; +} + +/** + * A source's `coverage`: its derived outer box, or `fallback` until + * `live:areas` has derived one (a new source; `areas.test.ts` fails the build + * for a source with stops and no areas). + */ +export function derivedCoverage(sourceId: string, fallback: GeoBoundingBox): GeoBoundingBox { + return sourceAreas(sourceId)?.outer ?? fallback; +} + +/** Whether a box (a watched room) meets a source's areas, or its coverage when it has none. */ +export function meetsSource( + source: { id: string; coverage: GeoBoundingBox | null }, + box: GeoBoundingBox, +): boolean { + const derived = sourceAreas(source.id); + if (derived) return derived.areas.some((area) => intersects(area, box)); + return source.coverage === null || intersects(source.coverage, box); +} + +function intersects(a: GeoBoundingBox, b: GeoBoundingBox): boolean { + return a.west <= b.east && a.east >= b.west && a.south <= b.north && a.north >= b.south; +} diff --git a/packages/backend/src/live/catalogue.ts b/packages/backend/src/live/catalogue.ts index 0d2c37af..92be2672 100644 --- a/packages/backend/src/live/catalogue.ts +++ b/packages/backend/src/live/catalogue.ts @@ -18,6 +18,7 @@ * are generic. */ +import { derivedCoverage } from './areas'; import type { GeoBoundingBox } from '@goway/contracts'; import { createArrivalEstimateAdapter } from './adapters/arrivalEstimates'; import { createDigitrafficAdapter } from './adapters/digitraffic'; @@ -53,6 +54,20 @@ const NAP_LICENCE: LiveSourceDefinition['licence'] = { /** The Baltic Sea and the Gulf of Finland, where Finland's AIS network hears ships. */ const BALTIC: GeoBoundingBox = { west: 9.0, south: 53.0, east: 31.0, north: 66.5 }; +/** + * Each source's coverage, derived from its operator's stops (`areas.ts`); + * the hand-drawn boxes above stand until `live:areas` has derived one. + */ +const RENFE_CERCANIAS_COVERAGE = derivedCoverage('renfe-cercanias', SPAIN); +const FGC_GEOTREN_COVERAGE = derivedCoverage('fgc-geotren', CATALONIA); +const MTA_BUS_COVERAGE = derivedCoverage('mta-bus', NEW_YORK_CITY); +const MTA_LIRR_COVERAGE = derivedCoverage('mta-lirr', NEW_YORK_REGION); +const MTA_MNR_COVERAGE = derivedCoverage('mta-mnr', NEW_YORK_REGION); +const MTA_SUBWAY_COVERAGE = derivedCoverage('mta-subway', NEW_YORK_CITY); +const AMB_BUS_COVERAGE = derivedCoverage('amb-bus', AMB_AREA); +const EMT_FUENLABRADA_COVERAGE = derivedCoverage('emt-fuenlabrada', MADRID_REGION); +const NYC_BIKESHARE_COVERAGE = derivedCoverage('nyc-bikeshare', NEW_YORK_CITY); + const CC_BY_4: LiveSourceDefinition['licence'] = { label: 'CC BY 4.0', href: 'https://creativecommons.org/licenses/by/4.0/', @@ -104,10 +119,11 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ { id: 'renfe-cercanias', kind: 'transit', + transitModes: ['rail'], name: 'Trains — Renfe Cercanías and Rodalies', attribution: { label: 'Origen de los datos: Renfe Operadora', href: 'https://data.renfe.com' }, licence: CC_BY_4, - coverage: SPAIN, + coverage: RENFE_CERCANIAS_COVERAGE, completeness: 'network', defaultUrl: 'https://gtfsrt.renfe.com/vehicle_positions.pb', requiresKey: false, @@ -124,7 +140,7 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ { sourceId: 'renfe-cercanias', operator: 'Renfe', - coverage: SPAIN, + coverage: RENFE_CERCANIAS_COVERAGE, defaultMode: 'rail', lineOf: (vehicle) => renfeLine(vehicle.vehicleLabel), labelOf: (vehicle) => renfeTrain(vehicle.vehicleLabel), @@ -135,13 +151,14 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ { id: 'fgc-geotren', kind: 'transit', + transitModes: ['rail'], name: 'Trains — FGC (Catalonia)', attribution: { label: 'Ferrocarrils de la Generalitat de Catalunya', href: 'https://dadesobertes.fgc.cat', }, licence: CC_BY_4, - coverage: CATALONIA, + coverage: FGC_GEOTREN_COVERAGE, completeness: 'network', defaultUrl: 'https://dadesobertes.fgc.cat', requiresKey: false, @@ -167,10 +184,11 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ { id: 'mta-bus', kind: 'transit', + transitModes: ['bus'], name: 'Buses — MTA Bus Time (New York)', attribution: { label: 'Data: MTA', href: 'https://bustime.mta.info' }, licence: MTA_TERMS, - coverage: NEW_YORK_CITY, + coverage: MTA_BUS_COVERAGE, completeness: 'network', defaultUrl: 'https://gtfsrt.prod.obanyc.com/vehiclePositions', // Documented as required, answered without one when this was written: a @@ -191,7 +209,7 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ { sourceId: 'mta-bus', operator: 'MTA', - coverage: NEW_YORK_CITY, + coverage: MTA_BUS_COVERAGE, defaultMode: 'bus', labelOf: (vehicle) => mtaBusNumber(vehicle.vehicleId), keyQueryParameter: 'key', @@ -202,10 +220,11 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ { id: 'mta-lirr', kind: 'transit', + transitModes: ['rail'], name: 'Trains — Long Island Rail Road', attribution: { label: 'Data: MTA', href: 'https://www.mta.info/developers' }, licence: MTA_TERMS, - coverage: NEW_YORK_REGION, + coverage: MTA_LIRR_COVERAGE, completeness: 'network', defaultUrl: 'https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/lirr%2Fgtfs-lirr', requiresKey: false, @@ -215,17 +234,23 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ staticGtfs: { zip: 'https://rrgtfsfeeds.s3.amazonaws.com/gtfslirr.zip' }, create: (settings) => createGtfsRealtimeAdapter( - { sourceId: 'mta-lirr', operator: 'LIRR', coverage: NEW_YORK_REGION, defaultMode: 'rail' }, + { + sourceId: 'mta-lirr', + operator: 'LIRR', + coverage: MTA_LIRR_COVERAGE, + defaultMode: 'rail', + }, settings, ), }, { id: 'mta-mnr', kind: 'transit', + transitModes: ['rail'], name: 'Trains — Metro-North Railroad', attribution: { label: 'Data: MTA', href: 'https://www.mta.info/developers' }, licence: MTA_TERMS, - coverage: NEW_YORK_REGION, + coverage: MTA_MNR_COVERAGE, completeness: 'network', defaultUrl: 'https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/mnr%2Fgtfs-mnr', requiresKey: false, @@ -238,7 +263,7 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ { sourceId: 'mta-mnr', operator: 'Metro-North', - coverage: NEW_YORK_REGION, + coverage: MTA_MNR_COVERAGE, defaultMode: 'rail', }, settings, @@ -250,10 +275,11 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ // predictions, and drawn as estimates (`adapters/arrivalEstimates.ts`). id: 'mta-subway', kind: 'transit', + transitModes: ['subway'], name: 'Subway — MTA (New York), estimated', attribution: { label: 'Data: MTA', href: 'https://www.mta.info/developers' }, licence: MTA_TERMS, - coverage: NEW_YORK_CITY, + coverage: MTA_SUBWAY_COVERAGE, completeness: 'network', defaultUrl: 'https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs', requiresKey: false, @@ -288,13 +314,14 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ // buses, which publish nothing keyless (`docs/MAP_LIVE.md`). id: 'amb-bus', kind: 'transit', + transitModes: ['bus'], name: 'Buses — AMB (Barcelona area), estimated', attribution: { label: 'Font: AMB', href: 'https://www.ambmobilitat.cat' }, licence: { label: 'Reuse under Llei 37/2007', href: 'https://www.amb.cat/web/area-metropolitana/dades-obertes/condicions-d-us', }, - coverage: AMB_AREA, + coverage: AMB_BUS_COVERAGE, completeness: 'partial', defaultUrl: 'https://www.ambmobilitat.cat/transit/trips-updates/trips.bin', requiresKey: false, @@ -315,6 +342,7 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ // reported, as approximate (no track). id: 'aisa-bus', kind: 'transit', + transitModes: ['bus'], name: 'Buses — AISA (Madrid region)', attribution: MITRAMS_CREDIT, licence: NAP_LICENCE, @@ -335,10 +363,11 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ // EMT Fuenlabrada: real GPS positions and a keyless static GTFS with shapes. id: 'emt-fuenlabrada', kind: 'transit', + transitModes: ['bus'], name: 'Buses — EMT Fuenlabrada (Madrid region)', attribution: MITRAMS_CREDIT, licence: NAP_LICENCE, - coverage: MADRID_REGION, + coverage: EMT_FUENLABRADA_COVERAGE, completeness: 'partial', defaultUrl: 'https://api.control.optibus.co/opendata/v1/gtfs-rt/vehicle-positions?uid=c-5cfcd2d1', @@ -352,7 +381,7 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ { sourceId: 'emt-fuenlabrada', operator: 'EMT Fuenlabrada', - coverage: MADRID_REGION, + coverage: EMT_FUENLABRADA_COVERAGE, defaultMode: 'bus', }, settings, @@ -371,7 +400,7 @@ export const LIVE_SOURCES: readonly LiveSourceDefinition[] = [ label: 'Bikeshare Data License Agreement', href: 'https://citibikenyc.com/data-sharing-policy', }, - coverage: NEW_YORK_CITY, + coverage: NYC_BIKESHARE_COVERAGE, completeness: 'network', defaultUrl: 'https://gbfs.citibikenyc.com/gbfs/2.3/gbfs.json', requiresKey: false, diff --git a/packages/backend/src/live/cluster.ts b/packages/backend/src/live/cluster.ts new file mode 100644 index 00000000..7afdbe38 --- /dev/null +++ b/packages/backend/src/live/cluster.ts @@ -0,0 +1,452 @@ +/** + * GoWay Live across several API tasks: ONE task polls each feed, every task + * serves its own sockets. + * + * Without this, each backend task polled every feed on its own — N tasks, N× + * the upstream requests and N× the rate limits (`docs/MAP_LIVE.md` → What + * production needs). With it, the tasks share a {@link LiveBus} (Valkey, in + * production `REDIS_URL`): + * + * - **Demand** — each task reports the rooms its sockets watch (room tiles × + * kinds: coarse public tiles, never a viewport, no viewer), kept for + * {@link DEMAND_TTL_MS}; every task reads the union. A feed is wanted while + * ANY task has a viewer in its coverage. + * - **Leadership** — per source, a lease (`SET NX PX`, renewed by its holder) + * picks the one task that polls it. A leader that dies stops renewing; the + * lease expires after {@link LEASE_MS} and the next task to look takes it + * and polls at once. Nothing is lost but a poll or two. + * - **Fan-out** — the leader publishes each poll's answer (the enriched + * entities, gzip-compressed) and every other task applies it to its own + * index exactly as if it had polled, then tells its own sockets. A task that + * starts asks the leaders to republish what they hold (`sync`). + * + * Positions pass through Valkey's pub/sub and are never stored there: the + * only keys are the leases and the per-task demand, both with a TTL. + * + * When the bus fails (Valkey unreachable), every task falls back to polling + * on its own — duplicate upstream requests for the duration, but live keeps + * working; it is logged once per outage. + */ + +import { gunzipSync, gzipSync } from 'node:zlib'; +import type { LiveEntity } from '@goway/contracts'; +import { RedisClient } from 'bun'; + +/** A lease outlives its holder by this long at most. */ +export const LEASE_MS = 15_000; +/** How often a task renews its leases, reports its demand and reads everyone's. */ +export const COORDINATE_MS = 2_000; +/** A task's reported demand expires after this (it renews every `COORDINATE_MS`). */ +export const DEMAND_TTL_MS = 10_000; + +export type LiveClusterMessage = + /** A leader's poll answered: the source's entities, as the leader applied them. */ + | { + type: 'poll'; + from: string; + sourceId: string; + /** A full answer replaces what a follower holds (unless the source's polls are partial). */ + entities: LiveEntity[]; + lastSuccessAt: number; + /** The sender's lease epoch: a follower drops anything older than the newest it heard. */ + epoch: number; + } + /** A leader's poll failed: its status, so every task reports the same. */ + | { + type: 'status'; + from: string; + sourceId: string; + failures: number; + lastSuccessAt: number | null; + epoch: number; + } + /** A task started: leaders, republish what you hold. */ + | { type: 'sync'; from: string } + /** A leader's answer to `sync`: what it holds now. */ + | { + type: 'state'; + from: string; + sourceId: string; + entities: LiveEntity[]; + lastSuccessAt: number | null; + failures: number; + epoch: number; + }; + +/** The shared medium. `ValkeyLiveBus` in production, `MemoryLiveBus` in tests. */ +export interface LiveBus { + /** This task's id: unique per process. */ + readonly taskId: string; + publish(message: LiveClusterMessage): Promise; + /** + * Every message from OTHER tasks. Resolves once subscribed. Calling it again + * replaces the subscription (after one was lost). + */ + subscribe(handler: (message: LiveClusterMessage) => void): Promise; + /** + * Whether the subscription is in place NOW. False before `subscribe`, after + * it failed, and once its connection dropped — a reconnected Valkey client + * does not restore its subscriptions, so the hub subscribes again. + */ + isSubscribed(): boolean; + /** + * After a failed or timed-out call: replace connections that are no longer + * connected. Bun's client gives up after a long outage and stays failed + * even once Valkey is back (measured: a 10 s stop of redis-server left both + * clients answering `ERR_REDIS_CONNECTION_CLOSED` for good, while a fresh + * client answered at once). + */ + recover(): void; + /** + * Take or renew a source's lease. While this task holds it, its EPOCH — a + * per-source counter drawn when this holder took it, so every later + * holder's is larger — else 0. Messages carry it, and a message from an older holder (a + * publish abandoned at its deadline and delivered late) is dropped. + */ + lease(sourceId: string, ttlMs: number): Promise; + release(sourceId: string): Promise; + /** + * Report this task's demand entries (`demandEntries` in `hub.ts`: source + * ids, and z6 regions for a regional feed); returns the union over every + * live task. + */ + exchangeDemand(entries: readonly string[], ttlMs: number): Promise>; + close(): Promise; +} + +export function encodeMessage(message: LiveClusterMessage): string { + return gzipSync(Buffer.from(JSON.stringify(message))).toString('base64'); +} + +export function decodeMessage(text: string): LiveClusterMessage | null { + try { + const value = JSON.parse(gunzipSync(Buffer.from(text, 'base64')).toString('utf8')) as unknown; + if (typeof value !== 'object' || value === null) return null; + const message = value as { type?: unknown; from?: unknown }; + if (typeof message.from !== 'string') return null; + if (!['poll', 'status', 'sync', 'state'].includes(String(message.type))) return null; + return value as LiveClusterMessage; + } catch { + return null; + } +} + +/** A Bun Redis error about the connection itself, not about a command. */ +export function isConnectionFailure(error: unknown): boolean { + const code = (error as { code?: unknown } | null)?.code; + return typeof code === 'string' && code.startsWith('ERR_REDIS_CONNECTION'); +} + +export function newTaskId(): string { + return `${process.pid.toString(36)}-${crypto.randomUUID().slice(0, 8)}`; +} + +// --------------------------------------------------------------------------- +// Valkey +// --------------------------------------------------------------------------- + +/** The Valkey commands used, as Bun's built-in client speaks them. */ +interface RedisLike { + send(command: string, args: string[]): Promise; + publish(channel: string, message: string): Promise; + subscribe( + channel: string, + listener: (message: string, channel: string) => void, + ): Promise; + close(): void; + /** Called on every (re)connect. */ + onconnect?: (() => void) | null; + /** Called when the connection closes (Bun: once it has given up reconnecting). */ + onclose?: ((error?: unknown) => void) | null; + /** Whether the connection is up now. */ + readonly connected?: boolean; +} + +/** + * The lease is `|`. Renewing keeps the epoch; taking it draws + * the next value of the source's counter (`INCR`, KEYS[2]) — never a clock, + * which a failover to a replica whose clock lags would turn backwards. The + * counter has no TTL, so the fleet Valkey's `volatile-lru` never evicts it. + */ +const LEASE_SCRIPT = ` +local held = redis.call('GET', KEYS[1]) +if held then + local owner, epoch = string.match(held, '^(.*)|(%d+)$') + if owner == ARGV[1] then + redis.call('PEXPIRE', KEYS[1], ARGV[2]) + return tonumber(epoch) + end + return 0 +end +local epoch = redis.call('INCR', KEYS[2]) +redis.call('SET', KEYS[1], ARGV[1] .. '|' .. epoch, 'PX', ARGV[2]) +return epoch`; + +const RELEASE_SCRIPT = ` +local held = redis.call('GET', KEYS[1]) +if held and string.match(held, '^(.*)|%d+$') == ARGV[1] then return redis.call('DEL', KEYS[1]) end +return 0`; + +export class ValkeyLiveBus implements LiveBus { + readonly taskId: string; + private readonly channel: string; + private commands: RedisLike; + private subscriber: RedisLike | null = null; + private subscribed = false; + /** Clients whose connection failed under a call, or dropped after connecting. */ + private readonly failed = new WeakSet(); + /** Clients that have connected at least once. */ + private readonly established = new WeakSet(); + + constructor( + /** A new connection: the command one, and one per subscription. */ + private readonly connect: () => RedisLike, + private readonly prefix = 'goway:live', + taskId = newTaskId(), + ) { + this.taskId = taskId; + this.channel = `${prefix}:bus`; + this.commands = this.open(); + } + + private open(): RedisLike { + const client = this.connect(); + const previous = client.onconnect; + client.onconnect = () => { + this.established.add(client); + previous?.(); + }; + return client; + } + + /** Given up on: its connection failed under a call, or dropped after it had connected. */ + private dead(client: RedisLike): boolean { + return this.failed.has(client) || (this.established.has(client) && client.connected === false); + } + + /** Connections to `url`, made lazily (a subscriber cannot send commands, so it has its own). */ + static fromUrl(url: string, prefix?: string): ValkeyLiveBus { + return new ValkeyLiveBus(() => new RedisClient(url) as unknown as RedisLike, prefix); + } + + isSubscribed(): boolean { + return this.subscribed && this.subscriber?.connected !== false; + } + + recover(): void { + if (this.dead(this.commands)) { + this.commands.close(); + this.commands = this.open(); + } + // A dead subscriber is not listening; `subscribe` replaces it. + if (this.subscriber && this.dead(this.subscriber)) this.subscribed = false; + } + + async publish(message: LiveClusterMessage): Promise { + await this.watch(this.commands, (client) => + client.publish(this.channel, encodeMessage(message)), + ); + } + + private send(command: string, args: string[]): Promise { + return this.watch(this.commands, (client) => client.send(command, args)); + } + + /** + * Run a call, noting a client that failed on its CONNECTION (as opposed to + * a command error): that one is replaced by `recover`. A client that is + * merely not connected YET — just made, mid-handshake — is left alone. + */ + private async watch(client: RedisLike, call: (client: RedisLike) => Promise): Promise { + try { + return await call(client); + } catch (error) { + if (isConnectionFailure(error)) this.failed.add(client); + throw error; + } + } + + async subscribe(handler: (message: LiveClusterMessage) => void): Promise { + // A fresh connection every time: Bun's client reconnects by itself but + // does NOT restore subscriptions (measured: a Redis restart left the old + // subscriber connected and deaf), and re-subscribing on it would double + // the listener. + this.subscribed = false; + this.subscriber?.close(); + const subscriber = this.connect(); + this.subscriber = subscriber; + let connects = 0; + subscriber.onconnect = () => { + this.established.add(subscriber); + connects++; + // Any connect after the first is a reconnect: the subscription is gone. + if (connects > 1 && this.subscriber === subscriber) this.subscribed = false; + }; + subscriber.onclose = () => { + if (this.subscriber === subscriber) this.subscribed = false; + }; + await this.watch(subscriber, (client) => + client.subscribe(this.channel, (text) => { + const message = decodeMessage(text); + if (message && message.from !== this.taskId) handler(message); + }), + ); + if (this.subscriber === subscriber) this.subscribed = true; + } + + async lease(sourceId: string, ttlMs: number): Promise { + const epoch = await this.send('EVAL', [ + LEASE_SCRIPT, + '2', + `${this.prefix}:lease:${sourceId}`, + `${this.prefix}:epoch:${sourceId}`, + this.taskId, + String(ttlMs), + ]); + const value = Number(epoch); + return Number.isFinite(value) && value > 0 ? value : 0; + } + + async release(sourceId: string): Promise { + await this.send('EVAL', [RELEASE_SCRIPT, '1', `${this.prefix}:lease:${sourceId}`, this.taskId]); + } + + async exchangeDemand(rooms: readonly string[], ttlMs: number): Promise> { + const now = Date.now(); + const tasks = `${this.prefix}:tasks`; + await this.send('SET', [ + `${this.prefix}:demand:${this.taskId}`, + JSON.stringify(rooms), + 'PX', + String(ttlMs), + ]); + await this.send('ZADD', [tasks, String(now), this.taskId]); + await this.send('ZREMRANGEBYSCORE', [tasks, '-inf', String(now - ttlMs)]); + // The set expires too, should every task stop. + await this.send('PEXPIRE', [tasks, String(ttlMs * 3)]); + const ids = (await this.send('ZRANGEBYSCORE', [ + tasks, + String(now - ttlMs), + '+inf', + ])) as string[]; + const out = new Set(rooms); + if (!Array.isArray(ids) || ids.length === 0) return out; + const values = (await this.send( + 'MGET', + ids.map((id) => `${this.prefix}:demand:${id}`), + )) as (string | null)[]; + for (const value of values ?? []) { + if (typeof value !== 'string') continue; + try { + for (const room of JSON.parse(value) as unknown[]) + if (typeof room === 'string') out.add(room); + } catch { + // A malformed entry is somebody else's bug; it adds no demand. + } + } + return out; + } + + async close(): Promise { + this.subscribed = false; + const subscriber = this.subscriber; + // Cleared first: its `onclose` then finds it is no longer ours. + this.subscriber = null; + this.commands.close(); + subscriber?.close(); + } +} + +// --------------------------------------------------------------------------- +// In memory (tests): one "Valkey" shared by several buses in one process +// --------------------------------------------------------------------------- + +export class MemoryLiveNetwork { + readonly leases = new Map(); + private epochs = 0; + /** A new, larger epoch (Valkey's per-source `INCR` counter). */ + nextEpoch(): number { + this.epochs++; + return this.epochs; + } + readonly demand = new Map(); + readonly handlers = new Map void>(); + /** Buses closed, by task id. */ + readonly closed = new Set(); + /** Set false to simulate Valkey being unreachable (calls fail at once). */ + up = true; + /** Set true to simulate a silent partition: calls never answer. */ + hang = false; + /** Times a bus asked to recover its connections. */ + recoveries = 0; + + /** Valkey goes away: every connection drops, and with it every subscription. */ + disconnect(): void { + this.up = false; + this.handlers.clear(); + } + + constructor(readonly now: () => number = Date.now) {} + + bus(taskId: string): LiveBus { + const network = this; + const check = async () => { + if (network.hang) await new Promise(() => {}); + if (!network.up) throw new Error('bus down'); + }; + return { + taskId, + async publish(message) { + await check(); + // Through the wire format, as Valkey would carry it. + const text = encodeMessage(message); + for (const [id, handler] of network.handlers) { + if (id === taskId) continue; + const decoded = decodeMessage(text); + if (decoded) handler(decoded); + } + }, + async subscribe(handler) { + await check(); + network.handlers.set(taskId, handler); + }, + isSubscribed() { + return network.up && network.handlers.has(taskId); + }, + recover() { + network.recoveries++; + }, + async lease(sourceId, ttlMs) { + await check(); + const now = network.now(); + const current = network.leases.get(sourceId); + if (current && current.until > now) { + if (current.owner !== taskId) return 0; + current.until = now + ttlMs; + return current.epoch; + } + const epoch = network.nextEpoch(); + network.leases.set(sourceId, { owner: taskId, until: now + ttlMs, epoch }); + return epoch; + }, + async release(sourceId) { + await check(); + if (network.leases.get(sourceId)?.owner === taskId) network.leases.delete(sourceId); + }, + async exchangeDemand(rooms, ttlMs) { + await check(); + const now = network.now(); + network.demand.set(taskId, { rooms: [...rooms], until: now + ttlMs }); + const out = new Set(); + for (const entry of network.demand.values()) + if (entry.until > now) for (const room of entry.rooms) out.add(room); + return out; + }, + async close() { + network.handlers.delete(taskId); + network.demand.delete(taskId); + network.closed.add(taskId); + }, + }; + } +} diff --git a/packages/backend/src/live/hub.ts b/packages/backend/src/live/hub.ts index b46f16f7..b065b8a9 100644 --- a/packages/backend/src/live/hub.ts +++ b/packages/backend/src/live/hub.ts @@ -28,10 +28,20 @@ * {@link MAX_BACKOFF_MS}, ±20% jitter), or the upstream's `Retry-After` when * it gave one and it is longer. One poll per source is in flight at a time, * bounded by {@link POLL_TIMEOUT_MS}. + * + * ## Several tasks + * + * With a {@link LiveBus} (`cluster.ts`), demand is the union of every task's + * watchers, one task (the lease holder) polls each source and publishes what + * it got, and every other task applies that as if it had polled. Without + * one — or while it is unreachable — this process polls on its own. */ +import { meetsSource } from './areas'; import { type GeoBoundingBox, + isRegionalTransitMode, + LIVE_ROOM_ZOOMS, type LiveDelta, type LiveEntity, type LiveKind, @@ -41,6 +51,13 @@ import { type LiveSourceStatus, liveTileBounds, } from '@goway/contracts'; +import { + COORDINATE_MS, + DEMAND_TTL_MS, + LEASE_MS, + type LiveBus, + type LiveClusterMessage, +} from './cluster'; import { getBytes } from './normalise'; import { loadStaticGtfs, @@ -49,7 +66,7 @@ import { type PreviousMatch, type StaticGtfs, } from './staticGtfs'; -import { contains, type IndexChange, LiveTileIndex, roomKey } from './tileIndex'; +import { contains, type IndexChange, LiveTileIndex, moveOf, roomKey } from './tileIndex'; import { type LiveAdapter, type LiveAdapterSettings, @@ -86,6 +103,16 @@ export interface LiveHubOptions { logger?: LiveHubLogger; /** Jitter in `[0, 1)`; injected so a test's backoff is exact. */ random?: () => number; + /** Shared with the other API tasks (`cluster.ts`); absent, this process is alone. */ + bus?: LiveBus | null; + /** Lease and demand timings (`cluster.ts`); tests shorten them. */ + cluster?: { + leaseMs?: number; + demandTtlMs?: number; + coordinateMs?: number; + /** A bus call unanswered for this long counts as the bus being down (default lease ÷ 3). */ + callTimeoutMs?: number; + }; } interface SourceState { @@ -109,6 +136,14 @@ interface SourceState { /** Each vehicle's last match, to keep it on its direction's shape. */ matches: Map; settings: LiveAdapterSettings; + /** Whether this task holds the source's lease (always, without a bus). */ + leader: boolean; + /** This task's lease epoch while it leads (`LiveBus.lease`); 0 otherwise. */ + epoch: number; + /** The newest leader epoch heard for this source: older messages are dropped. */ + heardEpoch: number; + /** When a message at {@link heardEpoch} last arrived. */ + heardAt: number; } /** Static GTFS is re-read this often while a source is watched. */ @@ -154,7 +189,186 @@ export class LiveHub { gtfsFailures: 0, matches: new Map(), settings, + leader: !options.bus, + epoch: 0, + heardEpoch: 0, + heardAt: 0, })); + this.bus = options.bus ?? null; + } + + private readonly bus: LiveBus | null; + /** Every task's watched room keys, as of the last {@link coordinate}. */ + /** Every task's demand entries (`demandEntries`), as of the last {@link coordinate}. */ + private clusterDemand = new Set(); + /** The bus failed: this process polls on its own until it answers again. */ + private busDown = false; + private coordinateTimer: ReturnType | null = null; + private coordinating: Promise | null = null; + + /** Whether this process polls a source: it holds the lease, or there is nobody to share with. */ + private leads(state: SourceState): boolean { + return !this.bus || this.busDown || state.leader; + } + + /** Whether this task currently polls a source (tests, `GET /live/sources` diagnostics). */ + isLeader(sourceId: string): boolean { + const state = this.states.find((candidate) => candidate.definition.id === sourceId); + return state ? this.leads(state) : false; + } + + /** + * A bus call, bounded. Unanswered within a third of the lease it counts as + * a failure — the task polls on its own — rather than leaving a follower + * neither polling nor hearing for as long as the client takes to give up + * (measured: ~10.7 s per command with Valkey stopped; minutes on a silent + * partition, long enough for the lease to pass to a second poller). + */ + private leaseMs(): number { + return this.options.cluster?.leaseMs ?? LEASE_MS; + } + + private bounded(work: Promise): Promise { + const ms = this.options.cluster?.callTimeoutMs ?? Math.round(this.leaseMs() / 3); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`live bus: no answer within ${ms} ms`)), ms); + work.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error: unknown) => { + clearTimeout(timer); + reject(error); + }, + ); + }); + } + + private busFailed(error: unknown): void { + // Replace connections the client has given up on (Bun's never come back). + this.bus?.recover(); + if (!this.busDown) { + this.options.logger?.warn( + { reason: describe(error) }, + 'Live bus unreachable; polling locally until it answers', + ); + } + this.busDown = true; + } + + /** + * Share demand, take or renew the leases of the sources somebody watches, + * give up the others. Runs every {@link COORDINATE_MS} from `start()`; a test + * awaits it. + */ + coordinate(): Promise { + if (!this.bus || this.stopped) return Promise.resolve(); + if (this.coordinating) return this.coordinating; + const bus = this.bus; + const run = (async () => { + try { + // Not listening (the first connect failed, or the subscriber's + // connection dropped and came back deaf): listen again and catch up + // BEFORE giving up local polling — a follower that cannot hear the + // leader must not stop polling. + if (!bus.isSubscribed()) await this.join(bus); + this.clusterDemand = await this.bounded( + bus.exchangeDemand( + demandEntries( + this.states.map((state) => state.definition), + (definition) => this.localDemandFor(definition), + ), + this.options.cluster?.demandTtlMs ?? DEMAND_TTL_MS, + ), + ); + for (const state of this.states) { + const wanted = this.demandFor(state.definition).length > 0; + if (wanted) { + state.epoch = await this.bounded(bus.lease(state.definition.id, this.leaseMs())); + state.leader = state.epoch > 0; + if (state.leader) state.heardEpoch = Math.max(state.heardEpoch, state.epoch); + } else if (state.leader) { + await this.bounded(bus.release(state.definition.id)); + state.leader = false; + state.epoch = 0; + } + } + if (this.busDown) { + this.options.logger?.info({}, 'Live bus answering again; sharing polls'); + this.busDown = false; + // While down, every task polled; now the leases decide again. + } + } catch (error) { + this.busFailed(error); + } + })().finally(() => { + this.coordinating = null; + }); + this.coordinating = run; + return run; + } + + /** Join the other tasks: listen, then ask the leaders for what they hold. */ + async connect(): Promise { + if (!this.bus) return; + try { + await this.join(this.bus); + } catch (error) { + this.busFailed(error); + } + } + + private async join(bus: LiveBus): Promise { + await this.bounded(bus.subscribe((message) => this.receive(message))); + await this.bounded(bus.publish({ type: 'sync', from: bus.taskId })); + } + + /** Another task's message. */ + receive(message: LiveClusterMessage): void { + if (this.stopped || !this.bus) return; + if (message.type === 'sync') { + for (const state of this.states) { + if (!state.leader || this.busDown) continue; + const entities = this.index + .idsOfSource(state.definition.id) + .map((id) => this.index.get(id)) + .filter((entity): entity is LiveEntity => entity !== undefined); + void this.bounded( + this.bus.publish({ + type: 'state', + from: this.bus.taskId, + sourceId: state.definition.id, + entities, + lastSuccessAt: state.lastSuccessAt, + failures: state.failures, + epoch: state.epoch, + }), + ).catch((error: unknown) => this.busFailed(error)); + } + return; + } + const state = this.states.find((candidate) => candidate.definition.id === message.sourceId); + // Not configured here, or this task polls it itself. + if (!state || this.leads(state)) return; + // From a holder older than one already heard: a publish abandoned at its + // deadline and delivered late, after the lease moved on. Never applied — + // unless the newer holder has been silent for a whole lease: then the + // counter went backwards (a Valkey failover lost it) and the mark goes + // with it, rather than leave this task deaf to the only leader there is. + const silentFor = this.now() - state.heardAt; + if (message.epoch < state.heardEpoch && silentFor < this.leaseMs()) return; + state.heardEpoch = message.epoch; + state.heardAt = this.now(); + if (message.type === 'status') { + state.failures = message.failures; + state.lastSuccessAt = message.lastSuccessAt; + } else { + state.failures = message.type === 'state' ? message.failures : 0; + state.lastSuccessAt = message.lastSuccessAt; + this.apply(state, message.entities); + } + this.notifySources(); } // ── Watchers ────────────────────────────────────────────────────────────── @@ -242,37 +456,87 @@ export class LiveHub { return now - state.lastSuccessAt > 3 * interval + POLL_TIMEOUT_MS ? 'stale' : 'live'; } - /** The watched room boxes inside a source's coverage, of its kind. */ - private demandFor(definition: LiveSourceDefinition): GeoBoundingBox[] { + /** + * The rooms THIS task's sockets watch inside a source's coverage, of its + * kind: exact, never shared. A regional transit room counts only for a + * source that may carry rail (the only vehicles in it, `liveRoomZoomsOf`); + * for `shapes`, only the rooms a client asks shapes in + * (`LIVE_SHAPES_MIN_ZOOM`: the finest transit level). + */ + private localDemandFor( + definition: LiveSourceDefinition, + purpose: 'poll' | 'shapes' = 'poll', + ): GeoBoundingBox[] { const boxes: GeoBoundingBox[] = []; + const levels = LIVE_ROOM_ZOOMS[definition.kind]; + const finest = levels[levels.length - 1]; + const regional = + !definition.transitModes || definition.transitModes.some(isRegionalTransitMode); for (const key of this.roomWatchers.keys()) { const room = parseRoom(key); if (room.kind !== definition.kind) continue; + if (room.kind === 'transit') { + const zoom = Number(room.tile.slice(0, room.tile.indexOf('/'))); + if (zoom !== finest && (purpose === 'shapes' || !regional)) continue; + } const box = liveTileBounds(room.tile); - if (definition.coverage && !intersects(box, definition.coverage)) continue; + // Its operator's stops, not one box round them all (`areas.ts`). + if (!meetsSource(definition, box)) continue; boxes.push(box); } return boxes; } + /** + * Where a source is wanted: this task's rooms, and — with a bus — what the + * other tasks said they want (`demandEntries`): the source by id, or for a + * regional feed its z6 regions. + */ + private demandFor(definition: LiveSourceDefinition): GeoBoundingBox[] { + const boxes = this.localDemandFor(definition); + for (const entry of this.clusterDemand) { + const [id, region] = entry.split('@') as [string, string | undefined]; + if (id !== definition.id) continue; + // A whole-feed source needs no region; its coverage stands for "wanted". + boxes.push(region ? liveTileBounds(region) : (definition.coverage ?? WORLD)); + } + return boxes; + } + // ── The clock ───────────────────────────────────────────────────────────── start(): void { if (this.timer || this.stopped) return; - this.timer = setInterval(() => void this.tick(), TICK_MS); - // Never what keeps the process alive: shutdown closes the server and exits. - const handle: unknown = this.timer; - if (typeof handle === 'object' && handle !== null && 'unref' in handle) { - const unref = (handle as { unref?: unknown }).unref; - if (typeof unref === 'function') unref.call(handle); + this.timer = unref(setInterval(() => void this.tick(), TICK_MS)); + if (this.bus) { + void this.connect().then(() => this.coordinate()); + this.coordinateTimer = unref( + setInterval( + () => void this.coordinate(), + this.options.cluster?.coordinateMs ?? COORDINATE_MS, + ), + ); } } stop(): void { this.stopped = true; if (this.timer) clearInterval(this.timer); + if (this.coordinateTimer) clearInterval(this.coordinateTimer); this.timer = null; + this.coordinateTimer = null; for (const state of this.states) state.abort?.abort(); + // Hand the leases over now rather than after they expire (when the bus + // answers), and close its connections whatever its state. + const bus = this.bus; + if (bus) { + const releases = this.busDown + ? [] + : this.states + .filter((state) => state.leader) + .map((state) => bus.release(state.definition.id).catch(() => {})); + void Promise.all(releases).finally(() => void bus.close().catch(() => {})); + } } /** @@ -295,9 +559,22 @@ export class LiveHub { } state.idleSince = null; state.everActive = true; - if (state.definition.staticGtfs && !state.gtfsLoading && now >= state.gtfsDueAt) { + // The static GTFS is read by the leader (to put vehicles on their + // tracks) and by a task whose own viewers are close enough to ask it + // for shapes — not by every task the source is wanted somewhere for, + // nor for a regional view, which never asks. + const needsStatic = + this.leads(state) || this.localDemandFor(state.definition, 'shapes').length > 0; + if ( + needsStatic && + state.definition.staticGtfs && + !state.gtfsLoading && + now >= state.gtfsDueAt + ) { started.push(this.loadStatic(state)); } + // Another task polls it and publishes what it gets (`receive`). + if (!this.leads(state)) continue; if (state.inFlight || now < state.nextDueAt) continue; started.push(this.poll(state, demand)); } @@ -327,7 +604,15 @@ export class LiveHub { state.nextDueAt = startedAt + Math.max(state.definition.pollIntervalSeconds, result.backoffSeconds ?? 0) * 1000; - this.apply(state, this.enrich(state, result.entities, result.hints)); + const entities = this.enrich(state, result.entities, result.hints); + this.apply(state, entities); + this.share({ + type: 'poll', + sourceId: state.definition.id, + entities, + lastSuccessAt: state.lastSuccessAt, + epoch: state.epoch, + }); }, (error: unknown) => { if (this.stopped) return; @@ -350,6 +635,13 @@ export class LiveHub { }, 'Live source poll failed', ); + this.share({ + type: 'status', + sourceId: state.definition.id, + failures: state.failures, + lastSuccessAt: state.lastSuccessAt, + epoch: state.epoch, + }); }, ) .finally(() => { @@ -362,6 +654,18 @@ export class LiveHub { return run; } + /** Publish a leader's news to the other tasks; a failure falls back to polling locally. */ + private share( + message: + | Omit, 'from'> + | Omit, 'from'>, + ): void { + if (!this.bus || this.busDown) return; + void this.bounded( + this.bus.publish({ ...message, from: this.bus.taskId } as LiveClusterMessage), + ).catch((error: unknown) => this.busFailed(error)); + } + /** Fetch (or refresh) a source's static GTFS; a failure retries with backoff and never stops the feed. */ private loadStatic(state: SourceState): Promise { const location = state.definition.staticGtfs; @@ -516,30 +820,41 @@ export class LiveHub { private flush(changes: readonly IndexChange[]): void { if (changes.length === 0) return; const at = new Date(this.now()).toISOString(); - const perRoom = new Map; removes: Set }>(); + const perRoom = new Map< + string, + { upserts: Map; moves: Map; removes: Set } + >(); for (const change of changes) { if (!this.roomWatchers.has(change.room)) continue; let room = perRoom.get(change.room); if (!room) { - room = { upserts: new Map(), removes: new Set() }; + room = { upserts: new Map(), moves: new Map(), removes: new Set() }; perRoom.set(change.room, room); } if (change.type === 'upsert') { room.removes.delete(change.entity.id); + room.moves.delete(change.entity.id); room.upserts.set(change.entity.id, change.entity); + } else if (change.type === 'move') { + // A move after an upsert in the same batch is still news the client + // has not had: it rides in the upsert. + if (room.upserts.has(change.entity.id)) room.upserts.set(change.entity.id, change.entity); + else room.moves.set(change.entity.id, change.entity); } else { room.upserts.delete(change.id); + room.moves.delete(change.id); room.removes.add(change.id); } } for (const [key, room] of perRoom) { - if (room.upserts.size === 0 && room.removes.size === 0) continue; + if (room.upserts.size === 0 && room.moves.size === 0 && room.removes.size === 0) continue; const { tile, kind } = parseRoom(key); this.options.publish({ tile, kind, at, upserts: [...room.upserts.values()], + moves: [...room.moves.values()].map(moveOf), removes: [...room.removes], }); } @@ -569,6 +884,70 @@ export class LiveHub { } } +const WORLD: GeoBoundingBox = { west: -180, south: -85, east: 180, north: 85 }; + +/** + * The zoom a REGIONAL feed's demand is shared at (the aircraft feed, polled + * per z6 region by `readsb.ts → planRegions`): ~5.6° of longitude. + */ +export const SHARED_REGION_ZOOM = 6; + +/** + * What a task tells the others it wants — computed from its exact rooms, + * which never leave the process: + * + * - `` for every source one of its rooms falls in (a source id + * says no more than the source's coverage, ~a city or a country); + * - `@` for a regional feed (`partialPolls`), whose leader + * polls one region per z6 tile anyway. + * + * So a viewer in Philadelphia asks for no New York feed: the decision is made + * on the exact rooms, here, not on a coarse tile somewhere else. + */ +export function demandEntries( + definitions: readonly LiveSourceDefinition[], + localDemandFor: (definition: LiveSourceDefinition) => readonly GeoBoundingBox[], +): string[] { + const out = new Set(); + for (const definition of definitions) { + const boxes = localDemandFor(definition); + if (boxes.length === 0) continue; + if (!definition.partialPolls) { + out.add(definition.id); + continue; + } + for (const box of boxes) { + const lat = (box.south + box.north) / 2; + const lon = (box.west + box.east) / 2; + out.add(`${definition.id}@${regionTileOf(lat, lon)}`); + } + } + return [...out]; +} + +function regionTileOf(latitude: number, longitude: number): string { + const n = 2 ** SHARED_REGION_ZOOM; + const x = Math.min(n - 1, Math.max(0, Math.floor(((longitude + 180) / 360) * n))); + const radians = (Math.max(-85.05, Math.min(85.05, latitude)) * Math.PI) / 180; + const y = Math.min( + n - 1, + Math.max( + 0, + Math.floor(((1 - Math.log(Math.tan(radians) + 1 / Math.cos(radians)) / Math.PI) / 2) * n), + ), + ); + return `${SHARED_REGION_ZOOM}/${x}/${y}`; +} + +/** A timer that never keeps the process alive: shutdown closes the server and exits. */ +function unref(handle: T): T { + if (typeof handle === 'object' && handle !== null && 'unref' in handle) { + const fn = (handle as { unref?: unknown }).unref; + if (typeof fn === 'function') fn.call(handle); + } + return handle; +} + function round6(value: number): number { return Math.round(value * 1e6) / 1e6; } @@ -578,12 +957,6 @@ function parseRoom(key: string): Room { return { tile, kind }; } -function intersects(a: GeoBoundingBox, b: GeoBoundingBox): boolean { - if (a.north < b.south || a.south > b.north) return false; - // Neither of GoWay's coverages crosses the antimeridian; a room tile never does. - return a.west <= b.east && a.east >= b.west; -} - function describe(error: unknown): string { if (error instanceof Error) return `${error.name}: ${error.message}`.slice(0, 200); return String(error).slice(0, 200); diff --git a/packages/backend/src/live/index.ts b/packages/backend/src/live/index.ts index ae79e4fb..705a2890 100644 --- a/packages/backend/src/live/index.ts +++ b/packages/backend/src/live/index.ts @@ -10,6 +10,8 @@ import type { Namespace, Server as SocketIOServer } from 'socket.io'; import type { LiveConfig } from '../config/live'; import { liveSourceEnvName } from '../config/live'; import { LIVE_SOURCES } from './catalogue'; +import { type LiveBus, ValkeyLiveBus } from './cluster'; +import { packLivePayload } from './pack'; import { LiveHub, type LiveHubLogger, type LiveHubSource } from './hub'; import { attachLiveNamespace, liveRoomName } from './socket'; import type { LiveFetch } from './types'; @@ -27,6 +29,11 @@ export interface CreateLiveServiceOptions { logger: LiveHubLogger; fetch?: LiveFetch; now?: () => number; + /** + * The bus shared with the other API tasks. Default: Valkey at + * `config.valkeyUrl` when set, else none (this process polls alone). + */ + bus?: LiveBus | null; } /** The hub's sources from configuration: listed, known, and keyed when they must be. */ @@ -65,20 +72,48 @@ export function createLiveService(options: CreateLiveServiceOptions): LiveServic logger.warn({}, 'No listed live source is usable; live stays off'); return null; } + if (config.valkeyUrlProblem && options.bus === undefined) { + // Never the value: it may carry a password. + logger.warn( + { problem: config.valkeyUrlProblem }, + 'Live runs without a shared bus: every task polls on its own', + ); + } let namespace: Namespace | null = null; + const bus = + options.bus !== undefined + ? options.bus + : config.valkeyUrl + ? ValkeyLiveBus.fromUrl(config.valkeyUrl) + : null; const hub = new LiveHub({ + bus, sources, fetch: options.fetch ?? (globalThis.fetch as unknown as LiveFetch), ...(options.now ? { now: options.now } : {}), logger, publish: (delta: LiveDelta) => { - namespace?.to(liveRoomName(delta.tile, delta.kind)).emit(LIVE_SOCKET_EVENTS.delta, delta); + // Packed once per room, whoever is in it. + namespace + ?.to(liveRoomName(delta.tile, delta.kind)) + .emit(LIVE_SOCKET_EVENTS.delta, packLivePayload(delta)); }, onSourcesChanged: () => { - namespace?.emit(LIVE_SOCKET_EVENTS.sources, { items: hub.sources(), nextCursor: null }); + namespace?.emit( + LIVE_SOCKET_EVENTS.sources, + packLivePayload({ items: hub.sources(), nextCursor: null }), + ); }, }); - logger.info({ sources: sources.map((source) => source.definition.id) }, 'Live layer configured'); + logger.info( + { + sources: sources.map((source) => source.definition.id), + // Never the URL: it may carry credentials. + shared: bus !== null, + ...(bus ? { task: bus.taskId } : {}), + }, + 'Live layer configured', + ); return { hub, attach(io) { diff --git a/packages/backend/src/live/pack.ts b/packages/backend/src/live/pack.ts new file mode 100644 index 00000000..020b0085 --- /dev/null +++ b/packages/backend/src/live/pack.ts @@ -0,0 +1,15 @@ +/** + * Large server → client payloads travel as gzip bytes (`LIVE_PACKED_MIN_BYTES` + * in the contract): a dense room's snapshot is mostly repeated keys. Done + * here, in the application, because Bun's WebSocket server does not negotiate + * permessage-deflate. The client inflates with `unpackLivePayload`. + */ +import { gzipSync } from 'node:zlib'; +import { LIVE_PACKED_MIN_BYTES } from '@goway/contracts'; + +/** The payload as JSON when small, else its JSON gzipped (a Socket.IO binary attachment). */ +export function packLivePayload(value: unknown): unknown { + const text = JSON.stringify(value); + if (Buffer.byteLength(text) < LIVE_PACKED_MIN_BYTES) return value; + return gzipSync(Buffer.from(text), { level: 6 }); +} diff --git a/packages/backend/src/live/runAreas.ts b/packages/backend/src/live/runAreas.ts new file mode 100644 index 00000000..1894240f --- /dev/null +++ b/packages/backend/src/live/runAreas.ts @@ -0,0 +1,107 @@ +/** + * `bun run --cwd packages/backend live:areas` — derive every live source's + * areas from its operator's own stops (`areas.ts`) and rewrite + * `areas.generated.ts`. Run it when a source is added or its network changes; + * `areas.test.ts` fails for a source with stops and no areas. + * + * Reads, per source: the static GTFS's `stops.txt` (the shapes where a + * portal publishes no stops), or a bike share's GBFS stations. Nothing else + * is fetched, and nothing about anyone is. + */ + +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { LIVE_POLICY } from '../config/live'; +import { areasOf, type SourceAreas } from './areas'; +import { LIVE_SOURCES } from './catalogue'; +import { parseCsv } from './staticGtfs'; +import type { LiveSourceDefinition } from './types'; +import { readZipEntries } from './zip'; + +type Point = readonly [number, number]; + +const headers = { 'user-agent': LIVE_POLICY.userAgent }; + +async function bytes(url: string): Promise { + const response = await fetch(url, { headers, signal: AbortSignal.timeout(300_000) }); + if (!response.ok) throw new Error(`${url} answered ${response.status}`); + return new Uint8Array(await response.arrayBuffer()); +} + +function pointsOfCsv(text: string, latitude: string, longitude: string): Point[] { + const { header, rows } = parseCsv(text); + const la = header.indexOf(latitude); + const lo = header.indexOf(longitude); + if (la < 0 || lo < 0) return []; + return rows.map((row) => [Number(row[la]), Number(row[lo])] as const); +} + +async function gtfsPoints(definition: LiveSourceDefinition): Promise { + const location = definition.staticGtfs; + if (!location) return []; + const decoder = new TextDecoder(); + if ('routes' in location) { + return pointsOfCsv( + decoder.decode(await bytes(location.shapes)), + 'shape_pt_lat', + 'shape_pt_lon', + ); + } + const zips = 'zip' in location ? [location.zip] : location.zips; + const points: Point[] = []; + for (const url of zips) { + const entries = readZipEntries(await bytes(url), ['stops.txt', 'shapes.txt']); + const stops = entries.get('stops.txt'); + const shapes = entries.get('shapes.txt'); + if (stops) points.push(...pointsOfCsv(decoder.decode(stops), 'stop_lat', 'stop_lon')); + else if (shapes) + points.push(...pointsOfCsv(decoder.decode(shapes), 'shape_pt_lat', 'shape_pt_lon')); + } + return points; +} + +async function gbfsPoints(definition: LiveSourceDefinition): Promise { + const json = async (url: string) => + JSON.parse(new TextDecoder().decode(await bytes(url))) as Record; + const discovery = (await json(definition.defaultUrl)).data as Record< + string, + { feeds?: { name: string; url: string }[] } + >; + const feeds = Object.values(discovery)[0]?.feeds ?? []; + const information = feeds.find((feed) => feed.name === 'station_information'); + if (!information) return []; + const stations = ((await json(information.url)).data as { stations?: unknown[] }).stations ?? []; + return stations.map((station) => { + const { lat, lon } = station as { lat: number; lon: number }; + return [lat, lon] as const; + }); +} + +async function main(): Promise { + const out: Record = {}; + for (const definition of LIVE_SOURCES) { + const points = + definition.kind === 'bikeshare' ? await gbfsPoints(definition) : await gtfsPoints(definition); + const areas = areasOf(points); + if (!areas) { + console.log(`${definition.id}: no stops known — keeps its coverage box`); + continue; + } + out[definition.id] = areas; + console.log(`${definition.id}: ${points.length} points → ${areas.areas.length} areas`); + } + // `__dirname`: this package compiles to CommonJS (`db/migrationsFolder.ts`). + const target = join(__dirname, 'areas.generated.ts'); + writeFileSync( + target, + `// Generated by \`bun run --cwd packages/backend live:areas\` (\`runAreas.ts\`): do not edit.\n` + + `import type { SourceAreas } from './areas';\n\n` + + `export const LIVE_SOURCE_AREAS: Readonly> = ${JSON.stringify(out, null, 2)};\n`, + ); + console.log(`wrote ${target}`); +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +}); diff --git a/packages/backend/src/live/socket.ts b/packages/backend/src/live/socket.ts index b33f2d9a..0df88ae8 100644 --- a/packages/backend/src/live/socket.ts +++ b/packages/backend/src/live/socket.ts @@ -40,6 +40,7 @@ import { import type { Namespace, Server as SocketIOServer, Socket } from 'socket.io'; import type { LiveConfig } from '../config/live'; import type { LiveHub } from './hub'; +import { packLivePayload } from './pack'; export function liveRoomName(tile: string, kind: LiveKind): string { return `live:${tile}:${kind}`; @@ -101,45 +102,54 @@ export function attachLiveNamespace(io: SocketIOServer, options: AttachLiveOptio options.now, ); // What this socket is in, as the server decided it — never the box. - let tiles: string[] = []; - let kinds: LiveKind[] = []; + let tiles: LiveSubscriptionState['tiles'] = {}; const answer = (state: LiveSubscriptionState) => socket.emit(LIVE_SOCKET_EVENTS.subscription, state); - socket.emit(LIVE_SOCKET_EVENTS.sources, { items: hub.sources(), nextCursor: null }); + socket.emit( + LIVE_SOCKET_EVENTS.sources, + packLivePayload({ items: hub.sources(), nextCursor: null }), + ); socket.on(LIVE_SOCKET_EVENTS.subscribe, (payload: unknown) => { const wait = bucket.take(); if (wait > 0) { - answer({ state: 'rate_limited', tiles, kinds, retryAfterSeconds: wait }); + answer({ state: 'rate_limited', tiles, retryAfterSeconds: wait }); return; } const parsed = liveSubscriptionSchema.safeParse(payload); if (!parsed.success) { - answer({ state: 'invalid', tiles, kinds, retryAfterSeconds: null }); + answer({ state: 'invalid', tiles, retryAfterSeconds: null }); return; } const subscription = parsed.data; - const nextKinds = [...new Set(subscription.kinds)]; - // No kinds: leave everything, whatever the box. - const covered = nextKinds.length === 0 ? [] : liveTilesFor(subscription); - const nextTiles = covered ?? []; - const rooms = nextTiles.flatMap((tile) => nextKinds.map((kind) => ({ tile, kind }))); + const asked = [...new Set(subscription.kinds)]; + // Each kind's rooms at its own zoom; a kind covering too many is left out. + const rooms: { tile: string; kind: LiveKind }[] = []; + const nextTiles: LiveSubscriptionState['tiles'] = {}; + for (const kind of asked) { + const covered = liveTilesFor(subscription, kind); + if (!covered) continue; + nextTiles[kind] = covered; + for (const tile of covered) rooms.push({ tile, kind }); + } const { joined, left } = hub.watch(socket.id, rooms); for (const room of left) void socket.leave(liveRoomName(room.tile, room.kind)); for (const room of joined) void socket.join(liveRoomName(room.tile, room.kind)); tiles = nextTiles; - kinds = covered ? nextKinds : []; answer({ - state: covered ? 'ok' : 'too_wide', + // No kinds asked: leave everything, whatever the box — that is `ok`. + state: asked.length > 0 && rooms.length === 0 ? 'too_wide' : 'ok', tiles, - kinds, retryAfterSeconds: null, }); // After the answer, so a client knows which rooms the snapshots belong to. for (const room of joined) - socket.emit(LIVE_SOCKET_EVENTS.snapshot, hub.snapshot(room.tile, room.kind)); + socket.emit( + LIVE_SOCKET_EVENTS.snapshot, + packLivePayload(hub.snapshot(room.tile, room.kind)), + ); }); socket.on('disconnect', () => hub.unwatch(socket.id)); diff --git a/packages/backend/src/live/tileIndex.ts b/packages/backend/src/live/tileIndex.ts index b12e3f80..9971c41f 100644 --- a/packages/backend/src/live/tileIndex.ts +++ b/packages/backend/src/live/tileIndex.ts @@ -1,6 +1,7 @@ /** * The in-memory spatial index of live entities: the latest fix per entity, - * bucketed by room — a `LIVE_ROOM_ZOOM` web-mercator tile and a kind. + * bucketed by room — a web-mercator tile at each of its levels + * (`liveRoomZoomsOf`: its kind's, but a bus only at city zoom) and the kind. * * Rooms ARE the spatial index. A subscription is answered from the buckets of * the tiles it covers (`liveTilesFor`), a delta is addressed to the bucket an @@ -10,15 +11,19 @@ * rule in code. * * Every mutation reports the rooms it touched, so the hub can turn a poll into - * per-room deltas without diffing whole buckets. + * per-room deltas without diffing whole buckets — and says when an entity only + * MOVED within its room (`move`: nothing but its position, heading, speed, + * time and place along its track changed), which travels as a compact + * `LiveMove` instead of the whole entity. */ import { type GeoBoundingBox, type LiveEntity, type LiveKind, - liveTileOf, + type LiveMove, liveTilesFor, + liveTilesOf, } from '@goway/contracts'; export function roomKey(tile: string, kind: LiveKind): string { @@ -27,14 +32,50 @@ export function roomKey(tile: string, kind: LiveKind): string { interface Entry { entity: LiveEntity; - room: string; + /** Its room at each of its levels (`liveRoomZoomsOf`), coarsest first. */ + rooms: string[]; /** What a client would receive, as text: the change test. */ signature: string; + /** The same without what a move carries: equal ones differ only in motion. */ + fixed: string; +} + +/** Everything about an entity but its motion, as text. */ +function fixedSignature(entity: LiveEntity): string { + const { + position: _p, + headingDegrees: _h, + speedMetersPerSecond: _s, + observedAt: _o, + ...rest + } = entity; + if (rest.kind === 'transit' && rest.track) { + const { alongMeters: _a, offsetMeters: _f, ...track } = rest.track; + return JSON.stringify({ ...rest, track }); + } + return JSON.stringify(rest); +} + +/** An entity's motion, as a {@link LiveMove}. */ +export function moveOf(entity: LiveEntity): LiveMove { + const track = entity.kind === 'transit' ? entity.track : null; + return [ + entity.id, + entity.position.latitude, + entity.position.longitude, + entity.headingDegrees, + entity.speedMetersPerSecond, + entity.observedAt, + track ? track.alongMeters : null, + track ? track.offsetMeters : null, + ]; } export type IndexChange = /** New, or changed in place. */ | { type: 'upsert'; room: string; entity: LiveEntity } + /** Only moved, within the same room: its motion changed and nothing else. */ + | { type: 'move'; room: string; entity: LiveEntity } /** Gone from this room — deleted, or moved out (a matching `upsert` names the new room). */ | { type: 'remove'; room: string; id: string }; @@ -51,32 +92,50 @@ export class LiveTileIndex { return this.entries.get(id)?.entity; } - /** Insert or replace. Returns the changes — none when nothing a client sees changed. */ + /** + * Insert or replace. Returns the changes — none when nothing a client sees + * changed — per room: an entity is in one room at each of its kind's + * levels (`LIVE_ROOM_ZOOMS`), and a subscriber is in one level's rooms. + */ upsert(entity: LiveEntity): IndexChange[] { - const room = roomKey(liveTileOf(entity.position), entity.kind); + const mode = entity.kind === 'transit' ? entity.mode : undefined; + const rooms = liveTilesOf(entity.position, entity.kind, mode).map((tile) => + roomKey(tile, entity.kind), + ); const signature = JSON.stringify(entity); const existing = this.entries.get(entity.id); if (existing && existing.signature === signature) return []; + const fixed = fixedSignature(entity); + const onlyMoved = existing !== undefined && existing.fixed === fixed; const changes: IndexChange[] = []; - if (existing && existing.room !== room) { - this.rooms.get(existing.room)?.delete(entity.id); - this.dropEmptyRoom(existing.room); - changes.push({ type: 'remove', room: existing.room, id: entity.id }); + // Rooms it left — moved out, or no longer at that level (a bus whose + // static file said "train" a moment ago). + for (const before of existing?.rooms ?? []) { + if (rooms.includes(before)) continue; + this.rooms.get(before)?.delete(entity.id); + this.dropEmptyRoom(before); + changes.push({ type: 'remove', room: before, id: entity.id }); } - this.entries.set(entity.id, { entity, room, signature }); - let members = this.rooms.get(room); - if (!members) { - members = new Set(); - this.rooms.set(room, members); + for (const room of rooms) { + if (onlyMoved && existing?.rooms.includes(room)) { + changes.push({ type: 'move', room, entity }); + continue; + } + let members = this.rooms.get(room); + if (!members) { + members = new Set(); + this.rooms.set(room, members); + } + members.add(entity.id); + changes.push({ type: 'upsert', room, entity }); } - members.add(entity.id); + this.entries.set(entity.id, { entity, rooms, signature, fixed }); let owned = this.sources.get(entity.sourceId); if (!owned) { owned = new Set(); this.sources.set(entity.sourceId, owned); } owned.add(entity.id); - changes.push({ type: 'upsert', room, entity }); return changes; } @@ -84,12 +143,16 @@ export class LiveTileIndex { const existing = this.entries.get(id); if (!existing) return []; this.entries.delete(id); - this.rooms.get(existing.room)?.delete(id); - this.dropEmptyRoom(existing.room); + const changes: IndexChange[] = []; + for (const room of existing.rooms) { + this.rooms.get(room)?.delete(id); + this.dropEmptyRoom(room); + changes.push({ type: 'remove', room, id }); + } const owned = this.sources.get(existing.entity.sourceId); owned?.delete(id); if (owned?.size === 0) this.sources.delete(existing.entity.sourceId); - return [{ type: 'remove', room: existing.room, id }]; + return changes; } /** The ids a source currently has in the index. A copy: safe to mutate while iterating. */ @@ -115,14 +178,14 @@ export class LiveTileIndex { /** * Entities inside a box, of the given kinds. `null` when the box covers more - * rooms than a subscription may (`LIVE_MAX_ROOM_TILES`). + * of a kind's rooms than a subscription may (`LIVE_MAX_ROOM_TILES`). */ query(box: GeoBoundingBox, kinds: readonly LiveKind[]): LiveEntity[] | null { - const tiles = liveTilesFor(box); - if (!tiles) return null; const out: LiveEntity[] = []; - for (const tile of tiles) { - for (const kind of kinds) { + for (const kind of kinds) { + const tiles = liveTilesFor(box, kind); + if (!tiles) return null; + for (const tile of tiles) { for (const entity of this.inRoom(tile, kind)) { if (contains(box, entity.position)) out.push(entity); } diff --git a/packages/backend/src/live/types.ts b/packages/backend/src/live/types.ts index 65c999bf..bb78052a 100644 --- a/packages/backend/src/live/types.ts +++ b/packages/backend/src/live/types.ts @@ -18,7 +18,13 @@ * every viewer's socket. */ -import type { GeoBoundingBox, LiveEntity, LiveKind, LiveLink } from '@goway/contracts'; +import type { + GeoBoundingBox, + LiveEntity, + LiveKind, + LiveLink, + LiveTransitMode, +} from '@goway/contracts'; import type { MatchHint, StaticGtfs, StaticGtfsLocation } from './staticGtfs'; /** What a poll may use. Injected, so every adapter is tested without a network. */ @@ -85,6 +91,12 @@ export interface LiveSourceDefinition { licence: LiveLink; /** `null` for worldwide. */ coverage: GeoBoundingBox | null; + /** + * The transit modes it can carry, when it is sure (MTA buses: `['bus']`). + * A source of street-level modes only is not woken by a regional view: its + * vehicles are in city-zoom rooms alone (`liveRoomZoomsOf`). Absent: any. + */ + transitModes?: readonly LiveTransitMode[]; /** Whether it carries a whole network in its coverage (`LiveSource.completeness`). */ completeness: 'network' | 'partial'; /** The feed's public endpoint; `LIVE_SOURCE__URL` overrides it. */ diff --git a/packages/contracts/openapi.json b/packages/contracts/openapi.json index 365e4171..d91a3729 100644 --- a/packages/contracts/openapi.json +++ b/packages/contracts/openapi.json @@ -13376,25 +13376,42 @@ ] }, "tiles": { - "maxItems": 20, - "type": "array", - "items": { - "type": "string", - "pattern": "^7\\/\\d{1,3}\\/\\d{1,3}$" - } - }, - "kinds": { - "maxItems": 4, - "type": "array", - "items": { - "type": "string", - "enum": [ - "aircraft", - "transit", - "bikeshare", - "vessel" - ] - } + "type": "object", + "properties": { + "aircraft": { + "maxItems": 20, + "type": "array", + "items": { + "type": "string", + "pattern": "^(7|9|11|13)\\/\\d{1,5}\\/\\d{1,5}$" + } + }, + "transit": { + "maxItems": 20, + "type": "array", + "items": { + "type": "string", + "pattern": "^(7|9|11|13)\\/\\d{1,5}\\/\\d{1,5}$" + } + }, + "bikeshare": { + "maxItems": 20, + "type": "array", + "items": { + "type": "string", + "pattern": "^(7|9|11|13)\\/\\d{1,5}\\/\\d{1,5}$" + } + }, + "vessel": { + "maxItems": 20, + "type": "array", + "items": { + "type": "string", + "pattern": "^(7|9|11|13)\\/\\d{1,5}\\/\\d{1,5}$" + } + } + }, + "additionalProperties": false }, "retryAfterSeconds": { "anyOf": [ @@ -13412,7 +13429,6 @@ "required": [ "state", "tiles", - "kinds", "retryAfterSeconds" ] }, @@ -13421,7 +13437,7 @@ "properties": { "tile": { "type": "string", - "pattern": "^7\\/\\d{1,3}\\/\\d{1,3}$" + "pattern": "^(7|9|11|13)\\/\\d{1,5}\\/\\d{1,5}$" }, "kind": { "type": "string", @@ -13456,7 +13472,7 @@ "properties": { "tile": { "type": "string", - "pattern": "^7\\/\\d{1,3}\\/\\d{1,3}$" + "pattern": "^(7|9|11|13)\\/\\d{1,5}\\/\\d{1,5}$" }, "kind": { "type": "string", @@ -13478,6 +13494,83 @@ "$ref": "#/components/schemas/LiveEntity" } }, + "moves": { + "default": [], + "type": "array", + "items": { + "type": "array", + "prefixItems": [ + { + "type": "string", + "minLength": 3, + "maxLength": 160 + }, + { + "type": "number", + "minimum": -90, + "maximum": 90 + }, + { + "type": "number", + "minimum": -180, + "maximum": 180 + }, + { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "exclusiveMaximum": 360 + }, + { + "type": "null" + } + ] + }, + { + "anyOf": [ + { + "type": "number", + "minimum": 0 + }, + { + "type": "null" + } + ] + }, + { + "type": "string", + "minLength": 1, + "format": "date-time" + }, + { + "anyOf": [ + { + "type": "number", + "minimum": 0 + }, + { + "type": "null" + } + ] + }, + { + "anyOf": [ + { + "type": "number", + "minimum": 0 + }, + { + "type": "null" + } + ] + } + ], + "items": false, + "minItems": 8, + "maxItems": 8 + } + }, "removes": { "type": "array", "items": { @@ -13492,6 +13585,7 @@ "kind", "at", "upserts", + "moves", "removes" ] }, diff --git a/packages/contracts/src/live.ts b/packages/contracts/src/live.ts index 05c56466..996ee6f6 100644 --- a/packages/contracts/src/live.ts +++ b/packages/contracts/src/live.ts @@ -27,14 +27,21 @@ * * `GET /api/v1/live/sources` (HTTP, in `API_OPERATIONS`) lists the sources and * their freshness. Positions travel over Socket.IO, namespace - * {@link LIVE_SOCKET_NAMESPACE}, in rooms keyed by a {@link LIVE_ROOM_ZOOM} - * web-mercator tile and a kind — so a client receives only the kinds it asked - * for, in the tiles it is looking at. The messages are the schemas below. + * {@link LIVE_SOCKET_NAMESPACE}, in rooms keyed by a web-mercator tile — at + * a zoom per kind ({@link LIVE_ROOM_ZOOMS}) — and a kind, so a client + * receives only the kinds it asked for, in the tiles it is looking at. The + * messages are the schemas below. */ import { z } from 'zod'; import { type GeoBoundingBox, geoBoundingBoxSchema, geoCoordinateSchema } from './geo'; -import { LIVE_KINDS, liveKindSchema, liveShapeIdSchema, liveSourceIdSchema } from './live-source'; +import { + LIVE_KINDS, + type LiveKind, + liveKindSchema, + liveShapeIdSchema, + liveSourceIdSchema, +} from './live-source'; import { instantSchema } from './time'; // --------------------------------------------------------------------------- @@ -205,27 +212,82 @@ export const liveEntitySchema = z.discriminatedUnion('kind', [ export type LiveEntity = z.infer; // --------------------------------------------------------------------------- -// Rooms: web-mercator tiles at one zoom +// Rooms: web-mercator tiles at a few zooms per kind // --------------------------------------------------------------------------- /** - * The zoom of the tile grid rooms are keyed on. A z7 tile is 2.8° of - * longitude — about 235 × 310 km at Barcelona — so a city view is one or two - * rooms, and an aircraft crossing a room boundary changes room every few - * minutes rather than every few seconds. + * The zooms of the tile grids each kind's rooms are keyed on, coarsest + * first. Every entity is in one room at EACH of its kind's levels; a + * subscription takes, per kind, the FINEST level whose rooms cover the + * viewport in at most {@link LIVE_MAX_ROOM_TILES}. So a street view of New + * York receives Midtown's buses, and a view of all Catalonia still receives + * every Rodalies train — from coarser rooms. + * + * - aircraft, vessels: z7 — 2.8° of longitude, ~235 × 310 km at Barcelona. + * - transit: z7 (a region or a country), z9 (~60 km: a metropolitan area) + * and z11 (~15 km: a borough) — but only RAIL is in the regional levels + * ({@link LIVE_REGIONAL_TRANSIT_MODES}); buses and every other street-level + * mode are in z11 rooms alone ({@link liveRoomZoomsOf}). + * - bike share: z13 — ~3.7 km. Its docks matter only at street zooms (the map + * draws, and asks for, them from z15). */ -export const LIVE_ROOM_ZOOM = 7; +export const LIVE_ROOM_ZOOMS: Readonly> = { + aircraft: [7], + vessel: [7], + transit: [7, 9, 11], + bikeshare: [13], +}; /** - * The most rooms one subscription may cover. A wider viewport is refused with - * `too_wide` and the client says "zoom in": 20 z7 tiles is roughly a z8 - * desktop viewport, and below that the map is a continent, where tens of - * thousands of moving dots are noise rather than information. + * The transit modes a REGIONAL view carries: trains and metros — Rodalies + * across Catalonia, the LIRR across Long Island. A bus, tram, ferry, cable car + * or vehicle of unknown mode is a street-level thing, drawn at city zoom only: + * a regional view of New York would otherwise carry every one of the MTA's + * ~6,000 buses (~205 KiB, then ~132 KiB every 15 s). + */ +export const LIVE_REGIONAL_TRANSIT_MODES: readonly LiveTransitMode[] = ['rail', 'subway']; + +/** Whether a transit mode is in the regional room levels ({@link LIVE_REGIONAL_TRANSIT_MODES}). */ +export function isRegionalTransitMode(mode: LiveTransitMode): boolean { + return LIVE_REGIONAL_TRANSIT_MODES.includes(mode); +} + +/** + * The room levels an entity is in, coarsest first: its kind's + * ({@link LIVE_ROOM_ZOOMS}), except that a street-level transit mode is only + * in the finest. + */ +export function liveRoomZoomsOf(entity: { + kind: LiveKind; + mode?: LiveTransitMode; +}): readonly number[] { + const levels = LIVE_ROOM_ZOOMS[entity.kind]; + if (entity.kind === 'transit' && entity.mode && !isRegionalTransitMode(entity.mode)) + return levels.slice(-1); + return levels; +} + +/** + * The map zoom from which a client asks for an operator's track shapes + * (`GET /live/sources/:id/shapes/:shapeId`): below it a train is a few pixels + * and its curve invisible. A view at this zoom takes the finest transit + * rooms, so only those rooms can use shapes. + */ +export const LIVE_SHAPES_MIN_ZOOM = 12; + +/** + * The most rooms one subscription may cover, PER KIND, at the level it + * takes. A kind whose COARSEST level still needs more is left out of the + * subscription (`kinds` says which were kept); if every kind is, the answer is + * `too_wide` and the client says "zoom in". 20 z7 tiles is roughly a z8 + * desktop viewport — below that the map is a continent. */ export const LIVE_MAX_ROOM_TILES = 20; -/** A room tile, `7/x/y`. */ -export const liveTileKeySchema = z.string().regex(/^7\/\d{1,3}\/\d{1,3}$/, 'must be a live tile'); +/** A room tile, `z/x/y`, at one of the {@link LIVE_ROOM_ZOOMS}. */ +export const liveTileKeySchema = z + .string() + .regex(/^(7|9|11|13)\/\d{1,5}\/\d{1,5}$/, 'must be a live tile'); const MAX_MERCATOR_LATITUDE = 85.051129; @@ -243,19 +305,37 @@ function tileY(latitude: number, n: number): number { return Math.min(n - 1, Math.max(0, y)); } -/** The room tile a position falls in. */ -export function liveTileOf(position: { latitude: number; longitude: number }): string { - const n = 2 ** LIVE_ROOM_ZOOM; - return `${LIVE_ROOM_ZOOM}/${tileX(position.longitude, n)}/${tileY(position.latitude, n)}`; +function finest(kind: LiveKind): number { + const levels = LIVE_ROOM_ZOOMS[kind]; + return levels[levels.length - 1] as number; +} + +/** The room tile a position of a kind falls in, at one of its levels (default: the finest). */ +export function liveTileOf( + position: { latitude: number; longitude: number }, + kind: LiveKind, + zoom: number = finest(kind), +): string { + const n = 2 ** zoom; + return `${zoom}/${tileX(position.longitude, n)}/${tileY(position.latitude, n)}`; } /** - * The room tiles a box covers, or `null` when that is more than - * {@link LIVE_MAX_ROOM_TILES}. A box across the antimeridian (`west > east`) - * covers both edges of the world. + * The room tiles a position of a kind falls in: one per level, coarsest + * first. With a transit `mode`, only that mode's levels ({@link liveRoomZoomsOf}). */ -export function liveTilesFor(box: GeoBoundingBox): string[] | null { - const n = 2 ** LIVE_ROOM_ZOOM; +export function liveTilesOf( + position: { latitude: number; longitude: number }, + kind: LiveKind, + mode?: LiveTransitMode, +): string[] { + return liveRoomZoomsOf(mode ? { kind, mode } : { kind }).map((zoom) => + liveTileOf(position, kind, zoom), + ); +} + +function tilesAt(box: GeoBoundingBox, zoom: number): string[] | null { + const n = 2 ** zoom; const yMin = tileY(box.north, n); const yMax = tileY(box.south, n); const columns: number[] = []; @@ -271,11 +351,25 @@ export function liveTilesFor(box: GeoBoundingBox): string[] | null { const rows = yMax - yMin + 1; if (rows * columns.length > LIVE_MAX_ROOM_TILES) return null; const tiles: string[] = []; - for (const x of columns) - for (let y = yMin; y <= yMax; y++) tiles.push(`${LIVE_ROOM_ZOOM}/${x}/${y}`); + for (const x of columns) for (let y = yMin; y <= yMax; y++) tiles.push(`${zoom}/${x}/${y}`); return tiles; } +/** + * A kind's room tiles for a box: at the finest of its levels that covers the + * box in at most {@link LIVE_MAX_ROOM_TILES}, or `null` when even the + * coarsest needs more. A box across the antimeridian (`west > east`) covers + * both edges of the world. + */ +export function liveTilesFor(box: GeoBoundingBox, kind: LiveKind): string[] | null { + const levels = LIVE_ROOM_ZOOMS[kind]; + for (let i = levels.length - 1; i >= 0; i--) { + const tiles = tilesAt(box, levels[i] as number); + if (tiles) return tiles; + } + return null; +} + /** The box a room tile covers, in degrees. */ export function liveTileBounds(tile: string): GeoBoundingBox { const [z, x, y] = tile.split('/').map(Number) as [number, number, number]; @@ -297,6 +391,16 @@ export function liveTileBounds(tile: string): GeoBoundingBox { /** The Socket.IO namespace. Open to signed-out visitors: the map is. */ export const LIVE_SOCKET_NAMESPACE = '/live'; +/** + * A server → client payload larger than this, serialised, travels as gzip + * BYTES (a Socket.IO binary attachment) instead of JSON: a dense room's + * snapshot is mostly repeated keys and shrinks ~8×. The client inflates it + * (`@goway.to/sdk/live` → `unpackLivePayload`) and parses it as usual. Done in + * the application because the backend's WebSocket server (Bun) does not + * negotiate permessage-deflate. + */ +export const LIVE_PACKED_MIN_BYTES = 2048; + /** Event names, in both directions. */ export const LIVE_SOCKET_EVENTS = { /** client → server: {@link LiveSubscription}. Replaces the previous one. */ @@ -325,12 +429,18 @@ export const liveSubscriptionSchema = geoBoundingBoxSchema }); export type LiveSubscription = z.input; +/** One kind's room tiles in a subscription's answer. */ +const liveRoomTilesSchema = z.array(liveTileKeySchema).max(LIVE_MAX_ROOM_TILES); + /** * The answer to a subscription. * - * - `ok` — in `tiles` × `kinds` rooms now; a snapshot follows for each new one. - * - `too_wide` — the box covers more than {@link LIVE_MAX_ROOM_TILES} rooms; - * nothing is sent until a narrower one arrives. + * - `ok` — in exactly the rooms `tiles` lists, kind by kind (each kind's + * tiles at its own zoom); a snapshot follows for each new room. A kind asked + * for but missing from `tiles` covered more than {@link LIVE_MAX_ROOM_TILES} + * rooms and was left out. + * - `too_wide` — every kind asked for covered more than + * {@link LIVE_MAX_ROOM_TILES} rooms; nothing is sent until a narrower box. * - `rate_limited` — subscriptions are arriving too fast; the previous rooms * (listed) are kept, retry after `retryAfterSeconds`. * - `invalid` — the message did not parse; the previous rooms are kept. @@ -338,11 +448,20 @@ export type LiveSubscription = z.input; export const liveSubscriptionStateSchema = z.object({ state: z.enum(['ok', 'too_wide', 'rate_limited', 'invalid']), /** - * The room tiles this connection is in NOW, and the kinds. A client drops - * whatever it holds outside them: those rooms send nothing more. + * The rooms this connection is in NOW: per kind, its tiles. A client drops + * whatever it holds in any other (tile, kind) — those rooms send nothing + * more. Per kind because the kinds' levels differ: an aircraft room's z7 + * tile is not a transit room this connection is in. */ - tiles: z.array(liveTileKeySchema).max(LIVE_MAX_ROOM_TILES), - kinds: z.array(liveKindSchema).max(LIVE_KINDS.length), + tiles: z + .object({ + aircraft: liveRoomTilesSchema, + transit: liveRoomTilesSchema, + bikeshare: liveRoomTilesSchema, + vessel: liveRoomTilesSchema, + } satisfies Record) + .partial() + .strict(), retryAfterSeconds: z.number().int().min(1).nullable(), }); export type LiveSubscriptionState = z.infer; @@ -357,16 +476,40 @@ export const liveSnapshotSchema = z.object({ }); export type LiveSnapshot = z.infer; +/** + * A vehicle that only MOVED, compactly: `[id, latitude, longitude, heading, + * speed, observedAt, alongMeters, offsetMeters]`. Everything else about it — + * kind, line, colours, labels, its track's shape — is what the client already + * holds; `headingDegrees`, `speedMetersPerSecond`, `observedAt` and, for a + * vehicle on a track, `track.alongMeters`/`offsetMeters` are replaced (`null` + * where it has no track). Most of a busy room's traffic is this: a bus is + * ~450 bytes as an entity and ~80 as a move. + */ +export const liveMoveSchema = z.tuple([ + liveEntityIdSchema, + z.number().min(-90).max(90), + z.number().min(-180).max(180), + z.number().min(0).lt(360).nullable(), + z.number().min(0).nullable(), + instantSchema, + z.number().min(0).nullable(), + z.number().min(0).nullable(), +]); +export type LiveMove = z.infer; + /** * What changed in one room since the last message: entities that arrived or - * moved (`upserts`) and ids that left (`removes` — gone from the feed, too old, - * or moved to another room, whose own delta carries the upsert). + * changed (`upserts`), ones that only moved (`moves`), and ids that left + * (`removes` — gone from the feed, too old, or moved to another room, whose + * own delta carries the upsert). A move names an entity the room's previous + * messages delivered; a client that does not hold it ignores it. */ export const liveDeltaSchema = z.object({ tile: liveTileKeySchema, kind: liveKindSchema, at: instantSchema, upserts: z.array(liveEntitySchema), + moves: z.array(liveMoveSchema).default([]), removes: z.array(liveEntityIdSchema), }); export type LiveDelta = z.infer; diff --git a/packages/frontend/components/__tests__/liveChunk.test.ts b/packages/frontend/components/__tests__/liveChunk.test.ts index e61e6621..ccc7be2c 100644 --- a/packages/frontend/components/__tests__/liveChunk.test.ts +++ b/packages/frontend/components/__tests__/liveChunk.test.ts @@ -130,9 +130,9 @@ describe('the live chunk', () => { }); test('the live HTTP client is only ever import()ed — two lazy chunks share it', () => { - // The shape cache (live chunk) and the settings dialog (its own chunk) both - // use it; a static import from either would hoist it into `__common` - // (measured: +272 B there before both were import()s). + // The live chunk (its availability check and shape cache) and the settings + // dialog (its own chunk) all use it; a static import from either would + // hoist it into `__common` (measured: +272 B there before both were import()s). const statics = files .filter((f) => staticImports(f.text).some((spec) => /lib\/live\/client$/.test(spec))) .map((f) => f.path); @@ -143,6 +143,7 @@ describe('the live chunk', () => { .sort(); expect(dynamics).toEqual([ 'features/explore/MapSettingsModal.tsx', + 'features/live/LiveHost.tsx', 'features/live/shapeCache.ts', ]); }); diff --git a/packages/frontend/features/explore/ExploreScreen.tsx b/packages/frontend/features/explore/ExploreScreen.tsx index 479e49fa..50655658 100644 --- a/packages/frontend/features/explore/ExploreScreen.tsx +++ b/packages/frontend/features/explore/ExploreScreen.tsx @@ -706,7 +706,7 @@ export default function ExploreScreen({ ) : null} {live && !mapError ? ( - + ) : null} diff --git a/packages/frontend/features/live/LiveHost.tsx b/packages/frontend/features/live/LiveHost.tsx index 7d5bf98e..7f29b941 100644 --- a/packages/frontend/features/live/LiveHost.tsx +++ b/packages/frontend/features/live/LiveHost.tsx @@ -12,14 +12,21 @@ * the viewer: see `liveClient.ts`. */ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import type { LiveKind } from '@goway.to/sdk/live'; +import { GoWayUnavailableError } from '@goway.to/sdk'; +import { LIVE_SHAPES_MIN_ZOOM, type LiveKind } from '@goway.to/sdk/live'; import type { MapCreditLink, MapLiveLayer, MapLivePoint } from '@/components/map'; import { API_URL } from '@/lib/config'; +import { watchLiveAvailability } from './availability'; import { FLEETS } from './fleets'; import { LIVE_GLYPHS, LIVE_KIND_ORDER } from './kinds'; -import { createLiveClient, type LiveClient, type LiveClientState } from './liveClient'; +import { + BIKESHARE_MIN_ZOOM, + createLiveClient, + type LiveClient, + type LiveClientState, +} from './liveClient'; import { LivePanel } from './LivePanel'; import { LiveStore } from './liveStore'; import { pointOf } from './points'; @@ -48,8 +55,6 @@ function inView(box: Box, point: { latitude: number; longitude: number }): boole export interface LiveHostProps { onLayer: (layer: MapLiveLayer | null) => void; - /** Start collapsed to a pill (a phone, where the panel would cover the map). */ - compact?: boolean; } /** How often freshness text and fading are re-evaluated with nothing else happening. */ @@ -68,13 +73,8 @@ function meetsView( ); } -/** Bike-share stations are drawn from this zoom: below it a city's docks are a carpet. */ -const BIKESHARE_MIN_ZOOM = 15; -/** Lines are fetched from this zoom: below it a train is a few pixels and its curve invisible. */ -const SHAPES_MIN_ZOOM = 12; - /** Rendered in the map's top-right stack (`ExploreScreen`), which places it. */ -export default function LiveHost({ onLayer, compact = false }: LiveHostProps) { +export default function LiveHost({ onLayer }: LiveHostProps) { const listeners = useRef(new Set<() => void>()); // A shape arriving moves its vehicles onto the track: wake the canvas. const store = useMemo( @@ -175,7 +175,7 @@ export default function LiveHost({ onLayer, compact = false }: LiveHostProps) { setViewport(view) { zoomRef.current = view.zoom; viewRef.current = padded(view.bounds, 0.25); - store.setShapeArea(view.zoom >= SHAPES_MIN_ZOOM ? viewRef.current : null); + store.setShapeArea(view.zoom >= LIVE_SHAPES_MIN_ZOOM ? viewRef.current : null); clientRef.current?.setView(view); }, select(id) { @@ -189,8 +189,30 @@ export default function LiveHost({ onLayer, compact = false }: LiveHostProps) { [store], ); + // Whether this backend serves live at all: a deployment without it + // (no `LIVE_SOURCES`) answers 503 and the layer stays out of sight — the + // switch can be on by default before the backend is. + const [available, setAvailable] = useState(false); + useEffect( + () => + watchLiveAvailability({ + check: () => + import('@/lib/live/client') + .then(({ liveApiClient }) => liveApiClient.live.sources()) + .then( + () => 'yes' as const, + // Anything but "no live here" (offline, a blip) is the socket's + // to report and retry. + (error: unknown) => (error instanceof GoWayUnavailableError ? 'no' : 'yes'), + ), + onAvailable: () => setAvailable(true), + }), + [], + ); + // The connection: opened once per mount, closed on unmount. useEffect(() => { + if (!available) return; const client = createLiveClient({ apiUrl: API_URL, store }); clientRef.current = client; client.setKinds([...kindsRef.current]); @@ -205,7 +227,7 @@ export default function LiveHost({ onLayer, compact = false }: LiveHostProps) { clientRef.current = null; onLayer(null); }; - }, [store, layer, onLayer, notify]); + }, [available, store, layer, onLayer, notify]); useEffect(() => { clientRef.current?.setKinds(LIVE_KIND_ORDER.filter((kind) => kinds.has(kind))); @@ -248,9 +270,11 @@ export default function LiveHost({ onLayer, compact = false }: LiveHostProps) { if (selected && (!selectedSample || !kinds.has(selectedSample.entity.kind))) setSelected(null); }, [selected, selectedSample, kinds]); + if (!available) return null; return ( void; ms: number; cancelled: boolean }[] = []; + let available = 0; + let asked = 0; + const stop = watchLiveAvailability({ + check: async () => answers[Math.min(asked++, answers.length - 1)] as LiveAvailability, + onAvailable: () => { + available++; + }, + schedule: (run, ms) => { + const timer = { run, ms, cancelled: false }; + timers.push(timer); + return () => { + timer.cancelled = true; + }; + }, + }); + const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + const fire = async () => { + const timer = timers.at(-1); + if (timer && !timer.cancelled) timer.run(); + await flush(); + }; + return { timers, stop, flush, fire, available: () => available, asked: () => asked }; +} + +describe('watchLiveAvailability', () => { + test('a backend that enables live later is found without a reload', async () => { + const h = harness(['no', 'no', 'yes']); + await h.flush(); + expect(h.available()).toBe(0); + expect(h.timers.map((t) => t.ms)).toEqual([AVAILABILITY_RETRY_MS[0]]); + await h.fire(); + expect(h.timers.map((t) => t.ms)).toEqual([AVAILABILITY_RETRY_MS[0], AVAILABILITY_RETRY_MS[1]]); + await h.fire(); + expect(h.available()).toBe(1); + expect(h.asked()).toBe(3); + }); + + test('available at once: no retry', async () => { + const h = harness(['yes']); + await h.flush(); + expect(h.available()).toBe(1); + expect(h.timers).toHaveLength(0); + }); + + test('stops asking when the map goes away', async () => { + const h = harness(['no', 'yes']); + await h.flush(); + h.stop(); + expect(h.timers[0]?.cancelled).toBe(true); + await h.fire(); + expect(h.available()).toBe(0); + }); +}); diff --git a/packages/frontend/features/live/__tests__/liveStore.test.ts b/packages/frontend/features/live/__tests__/liveStore.test.ts index 67e2ee96..6106c6a9 100644 --- a/packages/frontend/features/live/__tests__/liveStore.test.ts +++ b/packages/frontend/features/live/__tests__/liveStore.test.ts @@ -16,7 +16,8 @@ import { positionAt, velocityOf, } from '../motion'; -import { BLEND_MS, LiveStore, type ShapeSource } from '../liveStore'; +import { kindsAtZoom } from '../liveClient'; +import { applyMove, BLEND_MS, LiveStore, type ShapeSource } from '../liveStore'; import { consistAlong, consistStraight, pointAlong, type TrackShape } from '../track'; const T0 = Date.parse('2026-10-11T12:00:00Z'); @@ -221,10 +222,21 @@ describe('LiveStore', () => { }); expect(store.get('renfe-cercanias:a')?.position.latitude).toBe(40.9); // Leaving a room drops what it held. - store.retain(['7/64/47'], ['transit']); + store.retain({ transit: ['7/64/47'] }); expect(store.size).toBe(0); }); + test('keeps exactly the rooms answered: an aircraft room never keeps a transit room of the same tile', () => { + const store = new LiveStore(() => T0); + store.applySnapshot(message([train('bus', 41.3, 2.1, T0)])); + store.applySnapshot(message([aircraft()])); + expect(store.size).toBe(2); + // Zoomed in: aircraft still in their z7 room; transit now in z11 rooms. + store.retain({ aircraft: ['7/64/47'], transit: ['11/1036/764'] }); + expect(store.get('adsb-lol:abc123')).toBeDefined(); + expect(store.get('renfe-cercanias:bus')).toBeUndefined(); + }); + test('ignores an older fix arriving after a newer one', () => { const store = new LiveStore(() => T0); store.applySnapshot(message([train('a', 41.31, 2.1, T0)])); @@ -417,3 +429,63 @@ describe('the track', () => { expect(store.sample(ALL)[0]?.onTrack?.along).toBeCloseTo(1280, 3); }); }); + +describe('compact moves', () => { + test('a move replaces only the motion, and a move for something not held is ignored', () => { + const now = T0; + const store = new LiveStore(() => now); + store.applySnapshot( + message([train('1', 41.4, 2.105, T0, { shapeId: 'ell', alongMeters: 417 })]), + ); + const tile = message([]).tile; + store.applyDelta({ + tile, + kind: 'transit', + at: new Date(T0 + 20_000).toISOString(), + upserts: [], + moves: [ + ['renfe-cercanias:1', 41.401, 2.106, 45, 12, new Date(T0 + 20_000).toISOString(), 600, 3], + ['renfe-cercanias:unknown', 41, 2, null, null, new Date(T0).toISOString(), null, null], + ], + removes: [], + }); + const entity = store.get('renfe-cercanias:1'); + expect(entity?.position).toEqual({ latitude: 41.401, longitude: 2.106 }); + expect(entity?.headingDegrees).toBe(45); + expect(entity?.kind === 'transit' && entity.track).toEqual({ + shapeId: 'ell', + alongMeters: 600, + offsetMeters: 3, + }); + // Everything else as it was. + expect(entity?.kind === 'transit' && entity.line).toBe('R2N'); + expect(store.get('renfe-cercanias:unknown')).toBeUndefined(); + }); + + test('applyMove keeps a vehicle off any track off it', () => { + const off = train('2', 41.4, 2.1, T0); + const moved = applyMove(off, [ + 'renfe-cercanias:2', + 41.5, + 2.2, + null, + null, + new Date(T0).toISOString(), + null, + null, + ]); + expect(moved.kind === 'transit' && moved.track).toBeNull(); + expect(moved.position.latitude).toBe(41.5); + }); +}); + +describe('asking only for what is drawn', () => { + test('bike-share docks are asked for from z15 only', () => { + expect(kindsAtZoom(['aircraft', 'transit', 'bikeshare'], 13)).toEqual(['aircraft', 'transit']); + expect(kindsAtZoom(['aircraft', 'transit', 'bikeshare'], 15)).toEqual([ + 'aircraft', + 'transit', + 'bikeshare', + ]); + }); +}); diff --git a/packages/frontend/features/live/availability.ts b/packages/frontend/features/live/availability.ts new file mode 100644 index 00000000..7681f9ac --- /dev/null +++ b/packages/frontend/features/live/availability.ts @@ -0,0 +1,59 @@ +/** + * Whether this backend serves GoWay Live — asked before any socket opens, and + * asked AGAIN while the answer is no: a backend switches live on + * (`LIVE_SOURCES`) independently of the maps already open, and those must + * find out without a reload. Pure: the check and the timer are injected. + */ + +/** Waits between asks while the answer is "not served": then every 5 min. */ +export const AVAILABILITY_RETRY_MS: readonly number[] = [30_000, 60_000, 120_000, 300_000]; + +export type LiveAvailability = 'yes' | 'no'; + +export interface WatchAvailabilityOptions { + /** `'no'` only for "this deployment has no live" (503); anything else is `'yes'`. */ + check: () => Promise; + onAvailable: () => void; + /** `setTimeout`, injectable; returns a cancel. */ + schedule?: (run: () => void, ms: number) => () => void; +} + +/** Start asking; resolves `onAvailable` once. Returns the stop. */ +export function watchLiveAvailability(options: WatchAvailabilityOptions): () => void { + const schedule = + options.schedule ?? + ((run: () => void, ms: number) => { + const timer = setTimeout(run, ms); + return () => clearTimeout(timer); + }); + let stopped = false; + let cancel: (() => void) | null = null; + let attempt = 0; + const ask = () => { + cancel = null; + if (stopped) return; + void options.check().then( + (answer) => { + if (stopped) return; + if (answer === 'yes') { + options.onAvailable(); + return; + } + const wait = + AVAILABILITY_RETRY_MS[Math.min(attempt, AVAILABILITY_RETRY_MS.length - 1)] ?? 300_000; + attempt++; + cancel = schedule(ask, wait); + }, + () => { + // `check` reports failures as answers; a throw is a bug — say yes and + // let the socket report what is wrong. + if (!stopped) options.onAvailable(); + }, + ); + }; + ask(); + return () => { + stopped = true; + cancel?.(); + }; +} diff --git a/packages/frontend/features/live/liveClient.ts b/packages/frontend/features/live/liveClient.ts index ace1244a..685b0f01 100644 --- a/packages/frontend/features/live/liveClient.ts +++ b/packages/frontend/features/live/liveClient.ts @@ -17,6 +17,14 @@ * * Nothing about the viewer is sent: the subscription is the visible box, the * zoom and the kinds — no location, no session, no identifier. + * + * ## Asking only for what is drawn + * + * Bike-share docks are drawn from z{@link BIKESHARE_MIN_ZOOM}: below it they + * are not asked for at all (New York's 2,500 docks were most of a city view's + * traffic). Large messages arrive as gzip bytes and are inflated + * (`unpackLivePayload`); because that is asynchronous, every message goes + * through one queue, so they apply in the order they arrived. */ import type { Socket } from 'socket.io-client'; @@ -27,6 +35,7 @@ import { liveSubscription, liveTilesFor, parseLiveDelta, + unpackLivePayload, parseLiveSnapshot, parseLiveSources, parseLiveSubscriptionState, @@ -35,6 +44,13 @@ import { import type { LiveStore } from './liveStore'; export const SUBSCRIBE_DEBOUNCE_MS = 350; +/** Bike-share docks are drawn, and so asked for, from this zoom. */ +export const BIKESHARE_MIN_ZOOM = 15; + +/** The kinds worth asking for at a zoom. Exported for the tests. */ +export function kindsAtZoom(kinds: readonly LiveKind[], zoom: number): LiveKind[] { + return kinds.filter((kind) => kind !== 'bikeshare' || zoom >= BIKESHARE_MIN_ZOOM); +} export type LiveConnectionState = 'connecting' | 'connected' | 'offline'; export type LiveCoverageState = 'ok' | 'too_wide' | 'waiting'; @@ -100,15 +116,20 @@ export function createLiveClient(options: { apiUrl: string; store: LiveStore }): return; } const bounds = normaliseBounds(view.bounds); + const wanted = kindsAtZoom(kinds, view.zoom); const subscription = liveSubscription({ ...bounds, zoom: Math.max(0, Math.min(24, view.zoom)), - kinds, + kinds: wanted, }); const key = JSON.stringify(subscription); if (key === sent) return; sent = key; - coverage = kinds.length === 0 || liveTilesFor(bounds) ? 'waiting' : 'too_wide'; + // Too wide only when no kind asked for fits (the server keeps the ones that do). + coverage = + wanted.length === 0 || wanted.some((kind) => liveTilesFor(bounds, kind)) + ? 'waiting' + : 'too_wide'; notify(); socket?.emit(LIVE_SOCKET_EVENTS.subscribe, subscription); }; @@ -126,9 +147,10 @@ export function createLiveClient(options: { apiUrl: string; store: LiveStore }): .then(({ io }) => { if (closed) return; const live: Socket = io(`${options.apiUrl}${LIVE_SOCKET_NAMESPACE}`, { - // WebSocket first: no long-polling requests through the API's CORS lane - // unless the WebSocket cannot be opened at all. - transports: ['websocket', 'polling'], + // WebSocket only. Long-polling would need every request of a session + // to reach the same API task (ALB stickiness); a WebSocket is one + // connection to one task, whichever it is (`docs/MAP_LIVE.md`). + transports: ['websocket'], withCredentials: false, reconnectionDelay: 2000, reconnectionDelayMax: 30_000, @@ -149,13 +171,25 @@ export function createLiveClient(options: { apiUrl: string; store: LiveStore }): notify(); }); - live.on(LIVE_SOCKET_EVENTS.sources, (payload: unknown) => { + // One queue for every message: inflating is asynchronous, and a delta + // applied before the snapshot it follows would be lost. + let inbox: Promise = Promise.resolve(); + const receive = (event: string, handle: (payload: unknown) => void) => + live.on(event, (raw: unknown) => { + inbox = inbox + .then(() => unpackLivePayload(raw)) + .then((payload) => { + if (!closed) handle(payload); + }) + .catch(() => {}); + }); + receive(LIVE_SOCKET_EVENTS.sources, (payload: unknown) => { const page = parseLiveSources(payload); if (!page) return; store.setSources(page.items); notify(); }); - live.on(LIVE_SOCKET_EVENTS.subscription, (payload: unknown) => { + receive(LIVE_SOCKET_EVENTS.subscription, (payload: unknown) => { const state = parseLiveSubscriptionState(payload); if (!state) return; if (state.state === 'rate_limited') { @@ -165,17 +199,17 @@ export function createLiveClient(options: { apiUrl: string; store: LiveStore }): return; } if (state.state === 'invalid') return; - store.retain(state.tiles, state.kinds); + store.retain(state.tiles); coverage = state.state === 'too_wide' ? 'too_wide' : 'ok'; notify(); }); - live.on(LIVE_SOCKET_EVENTS.snapshot, (payload: unknown) => { + receive(LIVE_SOCKET_EVENTS.snapshot, (payload: unknown) => { const snapshot = parseLiveSnapshot(payload); if (!snapshot) return; store.applySnapshot(snapshot); notify(); }); - live.on(LIVE_SOCKET_EVENTS.delta, (payload: unknown) => { + receive(LIVE_SOCKET_EVENTS.delta, (payload: unknown) => { const delta = parseLiveDelta(payload); if (!delta) return; store.applyDelta(delta); diff --git a/packages/frontend/features/live/liveStore.ts b/packages/frontend/features/live/liveStore.ts index d10be8f2..004b3db4 100644 --- a/packages/frontend/features/live/liveStore.ts +++ b/packages/frontend/features/live/liveStore.ts @@ -18,7 +18,7 @@ * may arrive in its new room's delta before the old room's remove does. */ -import type { LiveEntity, LiveKind, LiveSource } from '@goway.to/sdk/live'; +import type { LiveEntity, LiveKind, LiveMove, LiveSource } from '@goway.to/sdk/live'; import { blend, @@ -115,6 +115,23 @@ export interface LiveSample { estimated: boolean; } +/** An entity with a {@link LiveMove}'s motion: everything else as held. Exported for the tests. */ +export function applyMove(entity: LiveEntity, move: LiveMove): LiveEntity { + const [, latitude, longitude, headingDegrees, speedMetersPerSecond, observedAt, along, offset] = + move; + const moved = { + ...entity, + position: { latitude, longitude }, + headingDegrees, + speedMetersPerSecond, + observedAt, + } as LiveEntity; + if (moved.kind === 'transit' && moved.track && along !== null) { + return { ...moved, track: { ...moved.track, alongMeters: along, offsetMeters: offset ?? 0 } }; + } + return moved; +} + function roomOf(tile: string, kind: LiveKind): string { return `${tile}|${kind}`; } @@ -179,6 +196,7 @@ export class LiveStore { kind: LiveKind; at: string; upserts: readonly LiveEntity[]; + moves?: readonly LiveMove[]; removes: readonly string[]; }): void { this.observeServerTime(delta.at); @@ -187,11 +205,26 @@ export class LiveStore { if (this.tracked.get(id)?.room === room) this.tracked.delete(id); } for (const entity of delta.upserts) this.upsert(entity, room); + // A move patches what this room already delivered; one we do not hold is + // ignored (its room's next snapshot or upsert will bring it). + for (const move of delta.moves ?? []) { + const item = this.tracked.get(move[0]); + if (!item || item.room !== room) continue; + this.upsert(applyMove(item.entity, move), room); + } } - /** Keep only the rooms the server says this connection is in. */ - retain(tiles: readonly string[], kinds: readonly LiveKind[]): void { - const rooms = new Set(tiles.flatMap((tile) => kinds.map((kind) => roomOf(tile, kind)))); + /** + * Keep only the rooms the server says this connection is in — exactly those + * (tile, kind) pairs: an aircraft room's z7 tile does not keep the transit + * room of the same tile, whose buses would otherwise freeze there. + */ + retain(tiles: Readonly>>): void { + const rooms = new Set( + (Object.entries(tiles) as [LiveKind, readonly string[]][]).flatMap(([kind, list]) => + list.map((tile) => roomOf(tile, kind)), + ), + ); for (const [id, item] of this.tracked) if (!rooms.has(item.room)) this.tracked.delete(id); } diff --git a/packages/frontend/lib/map/__tests__/mapFeatures.test.ts b/packages/frontend/lib/map/__tests__/mapFeatures.test.ts index 438e2c2f..5242e966 100644 --- a/packages/frontend/lib/map/__tests__/mapFeatures.test.ts +++ b/packages/frontend/lib/map/__tests__/mapFeatures.test.ts @@ -49,7 +49,7 @@ describe('the registry', () => { expect(new Set(queries).size).toBe(queries.length); }); - test('defaults: everything on goway.to but the woods fill and live data', () => { + test('defaults: everything on goway.to but the woods fill', () => { const features = resolve(); expect(features.list.map((state) => [state.id, state.on])).toEqual([ ['globe', true], @@ -64,7 +64,7 @@ describe('the registry', () => { ['cityLife', true], ['cityStreets', true], ['landmarks', true], - ['live', false], + ['live', true], ['flyover', true], ]); // Every stable feature is on and offered in production; every Labs one starts off. @@ -134,8 +134,19 @@ describe('the person decides', () => { test('a choice also reaches a feature the panel does not offer in this build', () => { // How the benchmark switches production-unserved features on: it stores - // the choices the panel would (`scripts/bench-map.ts`). - const features = resolve({ choices: { live: true } }); + // the choices the panel would (`scripts/bench-map.ts`). No feature is + // development-only today, so one is made so here. + const features = resolveMapFeatures({ + features: FEATURES.map((feature) => + feature.id === 'live' + ? { ...feature, defaultOn: false, offer: 'development' as const } + : feature, + ), + choices: { live: true }, + platform: 'web', + development: false, + search: '', + }); expect(features.get('live').available).toBe(false); expect(features.on('live')).toBe(true); }); @@ -268,8 +279,10 @@ describe('what the panel offers', () => { } }); - test('what production cannot serve yet only in a development build: live data', () => { - expect(resolve().get('live').available).toBe(false); - expect(resolve({ development: true }).get('live').available).toBe(true); + test('live data is offered on the web everywhere, and on by default; not on native', () => { + for (const development of [false, true]) { + expect(resolve({ development }).get('live')).toMatchObject({ available: true, on: true }); + } + expect(resolve({ platform: 'ios' }).get('live').available).toBe(false); }); }); diff --git a/packages/frontend/lib/map/__tests__/mapSettingsPages.test.ts b/packages/frontend/lib/map/__tests__/mapSettingsPages.test.ts index 00c1382d..8ad5cdd3 100644 --- a/packages/frontend/lib/map/__tests__/mapSettingsPages.test.ts +++ b/packages/frontend/lib/map/__tests__/mapSettingsPages.test.ts @@ -49,6 +49,7 @@ describe('the settings pages', () => { 'city3dTrees', 'cityLife', 'landmarks', + 'live', 'flyover', ]); }); @@ -56,7 +57,6 @@ describe('the settings pages', () => { test('Labs revealed (production): the woods fill joins, and a Labs page', () => { const { pages, features } = layout({ labs: true }); expect(pages).toEqual(['map', 'layers', 'labs', 'general', 'about']); - // goway.to's backend serves no live data yet. expect(ids(features.layers)).toEqual([ 'street3d', 'city3d', @@ -65,11 +65,12 @@ describe('the settings pages', () => { 'city3dTreeFill', 'cityLife', 'landmarks', + 'live', 'flyover', ]); }); - test('Labs revealed (development): live data joins', () => { + test('Labs revealed (development): the same layers as production', () => { const { features } = layout({ labs: true, development: true }); expect(ids(features.layers)).toEqual([ 'street3d', diff --git a/packages/frontend/lib/map/mapFeatures.ts b/packages/frontend/lib/map/mapFeatures.ts index 9c1381f5..30156fc0 100644 --- a/packages/frontend/lib/map/mapFeatures.ts +++ b/packages/frontend/lib/map/mapFeatures.ts @@ -231,15 +231,16 @@ export const MAP_FEATURES = [ { id: 'live', section: 'layers', - tier: 'labs', - defaultOn: false, + tier: 'stable', + // Real vehicles, for everyone. Safe on by default because the BACKEND is + // the gate: a deployment that lists no `LIVE_SOURCES` answers 503 and the + // layer draws nothing (`LiveHost`, which asks again while the map is + // open); which feeds are served — and so which licences are in play — is + // `LIVE_SOURCES`, set by the owner (`docs/MAP_LIVE.md` → Production). The + // panel starts folded to its pill, and `live.maxEntities` caps drawing. + defaultOn: true, requires: [], platforms: WEB, - // GoWay Live needs a backend that lists live sources (`LIVE_SOURCES`) and - // its feeds' licence follow-ups done; goway.to has neither yet - // (`docs/MAP_LIVE.md` → What production needs), so the switch is offered - // where a backend can serve it. - offer: 'development', query: 'live', }, { diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index bca9f921..d1e97b05 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -33,9 +33,19 @@ taxonomy. `liveTileOf`, `liveTileBounds`), `liveSubscription()` and a parser per message (`parseLiveSnapshot`, `parseLiveDelta`, `parseLiveSources`, `parseLiveSubscriptionState`, each returning `null` for a payload that does - not match). Bring your own `socket.io-client`; the package still has one - dependency. Every entity states its source and the instant the source - observed it — show both. + not match). Rooms are tiles at a few levels per kind (`LIVE_ROOM_ZOOMS`; + `liveTileOf(position, kind, zoom?)`, `liveTilesOf(position, kind, mode?)`, + `liveTilesFor(box, kind)` — the finest level that covers the box); only + rail (`LIVE_REGIONAL_TRANSIT_MODES`, `isRegionalTransitMode`) is in the + regional transit levels, a bus or other street-level vehicle only in the + finest (`liveRoomZoomsOf`); a subscription's answer lists its rooms per + kind (`tiles: { transit: [...], aircraft: [...] }`); a delta carries + `moves` — `LiveMove` tuples for vehicles that only moved — beside its + upserts and removes; and a message over `LIVE_PACKED_MIN_BYTES` arrives as + gzip bytes: pass every payload through `unpackLivePayload` (async; keep + arrival order) before parsing. Bring your own `socket.io-client`; the + package still has one dependency. Every entity states its source and the + instant the source observed it — show both. - `places.update(id, { openingHoursDays })` replaces only explicitly named opening weekdays atomically; exported `OpeningHoursDayPatch` describes each replacement. Empty day intervals close that day. Unknown schedules require diff --git a/packages/sdk/scripts/smoke.mjs b/packages/sdk/scripts/smoke.mjs index 0e76871c..8d208829 100644 --- a/packages/sdk/scripts/smoke.mjs +++ b/packages/sdk/scripts/smoke.mjs @@ -536,7 +536,9 @@ const snapshot = { if (live.LIVE_SOCKET_NAMESPACE !== '/live') throw new Error('namespace'); if (live.parseLiveSnapshot(snapshot)?.entities.length !== 1) throw new Error('snapshot did not parse'); if (live.parseLiveSnapshot({ nonsense: true }) !== null) throw new Error('junk parsed'); -if (live.liveTileOf({ latitude: 41.39, longitude: 2.17 }) !== '7/64/47') throw new Error('tile'); +if (live.liveTileOf({ latitude: 41.39, longitude: 2.17 }, 'aircraft') !== '7/64/47') throw new Error('tile'); +if (live.liveTileOf({ latitude: 41.39, longitude: 2.17 }, 'transit') !== '11/1036/764') throw new Error('transit tile'); +if (typeof live.unpackLivePayload !== 'function') throw new Error('unpackLivePayload'); `; const CONSUMER_LIVE_ESM = `import * as live from '@goway.to/sdk/live'; diff --git a/packages/sdk/src/contract.ts b/packages/sdk/src/contract.ts index 7243dfec..ae13ccce 100644 --- a/packages/sdk/src/contract.ts +++ b/packages/sdk/src/contract.ts @@ -141,12 +141,18 @@ export { LIVE_KINDS, LIVE_TRANSIT_MODES, LIVE_SOURCE_STATUSES, - LIVE_ROOM_ZOOM, + LIVE_REGIONAL_TRANSIT_MODES, + LIVE_SHAPES_MIN_ZOOM, + LIVE_ROOM_ZOOMS, LIVE_MAX_ROOM_TILES, + LIVE_PACKED_MIN_BYTES, LIVE_SOCKET_NAMESPACE, LIVE_SOCKET_EVENTS, liveTileOf, + isRegionalTransitMode, + liveRoomZoomsOf, liveTilesFor, + liveTilesOf, liveTileBounds, // Search and directions SEARCH_RESULT_KINDS, @@ -396,6 +402,7 @@ export type { LiveKind, LiveLink, LiveSnapshot, + LiveMove, LiveSource, LiveShape, LiveSourcePage, diff --git a/packages/sdk/src/live.ts b/packages/sdk/src/live.ts index 9da9fed0..87a09eae 100644 --- a/packages/sdk/src/live.ts +++ b/packages/sdk/src/live.ts @@ -39,7 +39,10 @@ import { LIVE_KINDS, LIVE_MAX_ROOM_TILES, - LIVE_ROOM_ZOOM, + LIVE_PACKED_MIN_BYTES, + LIVE_REGIONAL_TRANSIT_MODES, + LIVE_SHAPES_MIN_ZOOM, + LIVE_ROOM_ZOOMS, LIVE_SOCKET_EVENTS, LIVE_SOCKET_NAMESPACE, LIVE_SOURCE_STATUSES, @@ -49,9 +52,12 @@ import { liveSourcePageSchema, liveSubscriptionSchema, liveSubscriptionStateSchema, + isRegionalTransitMode, + liveRoomZoomsOf, liveTileBounds, liveTileOf, liveTilesFor, + liveTilesOf, } from './contract'; import type { LiveDelta, @@ -64,14 +70,20 @@ import type { export { LIVE_KINDS, LIVE_MAX_ROOM_TILES, - LIVE_ROOM_ZOOM, + LIVE_PACKED_MIN_BYTES, + LIVE_REGIONAL_TRANSIT_MODES, + LIVE_SHAPES_MIN_ZOOM, + LIVE_ROOM_ZOOMS, LIVE_SOCKET_EVENTS, LIVE_SOCKET_NAMESPACE, LIVE_SOURCE_STATUSES, LIVE_TRANSIT_MODES, + isRegionalTransitMode, + liveRoomZoomsOf, liveTileBounds, liveTileOf, liveTilesFor, + liveTilesOf, }; export type { @@ -83,6 +95,7 @@ export type { LiveEntity, LiveKind, LiveLink, + LiveMove, LiveShape, LiveSnapshot, LiveSource, @@ -103,6 +116,37 @@ function parsed( return result.success ? (result.data as T) : null; } +/** + * A server → client payload as sent: JSON as is, or gzip bytes (a payload over + * {@link LIVE_PACKED_MIN_BYTES}) inflated and parsed. Resolves `null` for bytes + * that are not gzip JSON. Feed the result to the `parseLive*` functions — and + * keep messages in arrival order: inflating is asynchronous, so a client + * should chain them (`liveClient.ts` does). + */ +export async function unpackLivePayload(payload: unknown): Promise { + const bytes = + payload instanceof ArrayBuffer + ? new Uint8Array(payload) + : ArrayBuffer.isView(payload) + ? new Uint8Array(payload.buffer, payload.byteOffset, payload.byteLength) + : null; + if (!bytes) return payload; + // The web streams API, present in browsers and Node ≥ 18; typed locally so + // the SDK keeps compiling without the DOM library. + const web = globalThis as unknown as { + Blob?: new (parts: unknown[]) => { stream(): { pipeThrough(transform: unknown): unknown } }; + DecompressionStream?: new (format: 'gzip') => unknown; + Response?: new (body: unknown) => { text(): Promise }; + }; + if (!web.Blob || !web.DecompressionStream || !web.Response) return null; + try { + const stream = new web.Blob([bytes]).stream().pipeThrough(new web.DecompressionStream('gzip')); + return JSON.parse(await new web.Response(stream).text()) as unknown; + } catch { + return null; + } +} + /** A `live:snapshot` payload, or `null`. */ export function parseLiveSnapshot(payload: unknown): LiveSnapshot | null { return parsed(liveSnapshotSchema, payload); diff --git a/packages/sdk/test/live.test.ts b/packages/sdk/test/live.test.ts index f1e387a3..9f68137c 100644 --- a/packages/sdk/test/live.test.ts +++ b/packages/sdk/test/live.test.ts @@ -4,13 +4,18 @@ import { LIVE_SOCKET_EVENTS, LIVE_SOCKET_NAMESPACE, liveSubscription, + LIVE_REGIONAL_TRANSIT_MODES, + liveRoomZoomsOf, liveTileOf, + liveTilesOf, liveTilesFor, parseLiveDelta, parseLiveSnapshot, parseLiveSources, parseLiveSubscriptionState, + unpackLivePayload, } from '../src/live'; +import { gzipSync } from 'node:zlib'; import { fakeFetch, rejection } from './helpers'; const source = { @@ -111,14 +116,22 @@ describe('@goway.to/sdk/live', () => { })?.removes, ).toEqual(['renfe-cercanias:1']); expect(parseLiveSources({ items: [source], nextCursor: null })?.items).toHaveLength(1); + expect( + parseLiveSubscriptionState({ + state: 'ok', + tiles: { transit: ['7/64/47'], aircraft: ['7/64/47'] }, + retryAfterSeconds: null, + })?.tiles.transit, + ).toEqual(['7/64/47']); + // The old flat shape, which let one kind's tile keep another kind's room. expect( parseLiveSubscriptionState({ state: 'ok', tiles: ['7/64/47'], kinds: ['transit'], retryAfterSeconds: null, - })?.state, - ).toBe('ok'); + }), + ).toBeNull(); for (const junk of [ null, 'x', @@ -151,10 +164,43 @@ describe('@goway.to/sdk/live', () => { ).toThrow(TypeError); }); - it('computes the same rooms the server does', () => { - expect(liveTileOf({ latitude: 41.39, longitude: 2.17 })).toBe('7/64/47'); - expect(liveTilesFor({ west: 2.05, south: 41.32, east: 2.25, north: 41.47 })).toEqual([ + it('computes the same rooms the server does, at each kind’s zoom', () => { + expect(liveTileOf({ latitude: 41.39, longitude: 2.17 }, 'aircraft')).toBe('7/64/47'); + expect(liveTileOf({ latitude: 41.39, longitude: 2.17 }, 'transit')).toBe('11/1036/764'); + expect( + liveTilesFor({ west: 2.05, south: 41.32, east: 2.25, north: 41.47 }, 'aircraft'), + ).toEqual(['7/64/47']); + }); + + it('puts a train in every transit level, a bus only at city zoom', () => { + const barcelona = { latitude: 41.39, longitude: 2.17 }; + expect(liveTilesOf(barcelona, 'transit', 'rail')).toEqual([ '7/64/47', + '9/259/191', + '11/1036/764', ]); + expect(liveTilesOf(barcelona, 'transit', 'bus')).toEqual(['11/1036/764']); + expect(liveRoomZoomsOf({ kind: 'transit', mode: 'tram' })).toEqual([11]); + expect(LIVE_REGIONAL_TRANSIT_MODES).toEqual(['rail', 'subway']); + }); + + it('inflates a packed payload and passes JSON through', async () => { + const delta = { + tile: '11/1036/764', + kind: 'transit', + at: '2026-10-12T12:00:00.000Z', + upserts: [], + removes: [], + }; + expect(await unpackLivePayload(delta)).toBe(delta); + const packed = gzipSync(Buffer.from(JSON.stringify(delta))); + const unpacked = await unpackLivePayload(new Uint8Array(packed)); + expect(parseLiveDelta(unpacked)).toEqual({ ...delta, moves: [] }); + expect( + await unpackLivePayload( + packed.buffer.slice(packed.byteOffset, packed.byteOffset + packed.byteLength), + ), + ).toEqual(delta); + expect(await unpackLivePayload(new Uint8Array([1, 2, 3]))).toBeNull(); }); });