diff --git a/e2e/collab.e2e.ts b/e2e/collab.e2e.ts index 04ec33efd..0e70015f1 100644 --- a/e2e/collab.e2e.ts +++ b/e2e/collab.e2e.ts @@ -11,7 +11,7 @@ * login session cookie. They still meet in one repository-authorized channel. */ -import { content, expect, ready, test } from "./room"; +import { content, expect, ready, test, written } from "./room"; test("an edit by one appears for the other", async ({ join }) => { let ana = await join("ana"); @@ -30,6 +30,21 @@ test("an edit by one appears for the other", async ({ join }) => { await expect(content(ana)).toContainText("Readable and diffable."); }); +test("typing faster than the server takes updates still reaches everyone", async ({ join, room }) => { + let ana = await join("ana"); + let bo = await join("bo"); + + // Without a delay this is well past the 200 updates a second the server + // accepts from one socket. The excess used to be dropped, and everything + // typed afterwards was stranded behind it. + await content(ana).click(); + await ana.keyboard.type("Typed faster than anybody types. ".repeat(8).trim()); + await ana.keyboard.type(" Then the rest.", { delay: 50 }); + + await expect(content(bo)).toContainText("anybody types. Then the rest."); + await written(ana, room, "anybody types. Then the rest."); +}); + test("the header represents everyone here as faces", async ({ join }) => { let ana = await join("ana"); let bo = await join("bo"); diff --git a/packages/editor/src/provider.test.ts b/packages/editor/src/provider.test.ts index bca47cf6e..ad7e727ec 100644 --- a/packages/editor/src/provider.test.ts +++ b/packages/editor/src/provider.test.ts @@ -11,7 +11,7 @@ * layout, and happy-dom returns zero for every measurement. */ -import { describe, expect, it } from "bun:test"; +import { afterEach, describe, expect, it, jest } from "bun:test"; import * as Y from "yjs"; import { PlanProvider } from "./provider"; @@ -302,3 +302,205 @@ describe("opening the plan", () => { }); }); }); + +/** + * A server that applies, acknowledges and relays updates, and drops some + * without a word, as the real one does past its rate limit. + * + * The plan service accepts at most 200 updates a second from one socket and + * silently ignores the rest. A dropped update also strands every later one + * from the same client, because Yjs holds an update whose predecessor never + * arrived instead of integrating it. + */ +function server(drop: (index: number) => boolean) { + let doc = new Y.Doc(); + let transport = wire(); + let received: Array<{ id: string; bytes: number }> = []; + let delivered = 0; + let send = transport.send; + let room = { + doc, + transport, + received, + drop, + get delivered() { + return delivered; + }, + }; + transport.send = (kind: string, payload: Record = {}) => { + send.call(transport, kind); + if (kind !== "plan:update") return; + let binary = atob(payload.update as string); + let update = Uint8Array.from(binary, char => char.charCodeAt(0)); + received.push({ id: payload.id as string, bytes: update.byteLength }); + if (room.drop(received.length - 1)) return; + delivered++; + Y.applyUpdate(doc, update); + transport.emit("plan:ack", { kind: "plan:ack", id: payload.id }); + }; + return room; +} + +function typing(doc: Y.Doc, count: number) { + let text = doc.getText("typed"); + for (let i = 0; i < count; i++) text.insert(text.length, "x"); +} + +describe("delivering local edits", () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it("does not send faster than the server accepts", async () => { + jest.useFakeTimers(); + // Past 200 a second the real server drops the rest. + let room = server(() => room.delivered >= 200); + let doc = new Y.Doc(); + let provider = new PlanProvider({ wire: room.transport, doc }); + await provider.connect(); + + typing(doc, 300); + jest.advanceTimersByTime(1_000); + + expect(room.doc.getText("typed").toString()).toBe("x".repeat(300)); + expect(room.delivered).toBeLessThan(200); + }); + + it("resends an edit the server never acknowledged", async () => { + jest.useFakeTimers(); + let room = server(index => index === 0); + let doc = new Y.Doc(); + let provider = new PlanProvider({ wire: room.transport, doc }); + await provider.connect(); + + typing(doc, 1); + jest.advanceTimersByTime(100); + typing(doc, 1); + jest.advanceTimersByTime(100); + // The second keystroke arrived but cannot be applied without the first. + expect(room.doc.getText("typed").toString()).toBe(""); + + jest.advanceTimersByTime(10_000); + expect(room.doc.getText("typed").toString()).toBe("xx"); + }); + + it("leaves acknowledged edits alone", async () => { + jest.useFakeTimers(); + let room = server(() => false); + let doc = new Y.Doc(); + let provider = new PlanProvider({ wire: room.transport, doc }); + await provider.connect(); + + typing(doc, 1); + jest.advanceTimersByTime(10_000); + + expect(room.delivered).toBe(1); + }); + + it("backs off and then stops resending an edit nobody acknowledges", async () => { + jest.useFakeTimers(); + let room = server(() => true); + let doc = new Y.Doc(); + let provider = new PlanProvider({ wire: room.transport, doc }); + await provider.connect(); + + typing(doc, 1); + // Sent at once, resent at 4s, then at 8s, 16s and 32s. + jest.advanceTimersByTime(7_000); + expect(room.received).toHaveLength(2); + jest.advanceTimersByTime(7_000); + expect(room.received).toHaveLength(3); + + jest.advanceTimersByTime(10 * 60_000); + expect(room.received).toHaveLength(5); + }); + + it("does not repeat an edit the server refused", async () => { + jest.useFakeTimers(); + let room = server(() => true); + let doc = new Y.Doc(); + let provider = new PlanProvider({ wire: room.transport, doc }); + await provider.connect(); + + typing(doc, 1); + room.transport.emit("session:error", { + kind: "session:error", + message: "implementation is active", + }); + typing(doc, 1); + // Too large, in the real server: refused without rotating the epoch. + jest.advanceTimersByTime(100); + room.transport.emit("plan:reset", { + kind: "plan:reset", + epoch: "01K0N4TR8K7JGM4R1J7PW4R8YJ", + reason: "rebuilt", + }); + jest.advanceTimersByTime(10 * 60_000); + + expect(room.received).toHaveLength(2); + }); + + it("stops resending once the epoch is replaced", async () => { + jest.useFakeTimers(); + let room = server(() => true); + let doc = new Y.Doc(); + let provider = new PlanProvider({ wire: room.transport, doc }); + await provider.connect(); + + typing(doc, 1); + room.transport.emit("plan:reset", { + kind: "plan:reset", + epoch: "01K0N4TR8K7JGM4R1J7PW4R8YK", + reason: "rebuilt", + }); + jest.advanceTimersByTime(10 * 60_000); + + expect(room.received).toHaveLength(1); + }); + + it("keeps a merged resend outstanding when the old ids are acknowledged late", async () => { + jest.useFakeTimers(); + let room = server(() => true); + let doc = new Y.Doc(); + let provider = new PlanProvider({ wire: room.transport, doc }); + await provider.connect(); + + typing(doc, 1); + jest.advanceTimersByTime(100); + typing(doc, 1); + jest.advanceTimersByTime(4_000); + let [first, second, merged] = room.received.map(entry => entry.id); + expect(merged).toStartWith("merged-"); + + for (let id of [first, second]) { + room.transport.emit("plan:ack", { kind: "plan:ack", id }); + } + room.drop = () => false; + jest.advanceTimersByTime(60_000); + + expect(room.received.at(-1)?.id).toBe(merged); + expect(room.doc.getText("typed").toString()).toBe("xx"); + }); + + it("replays a large backlog in chunks the server accepts", async () => { + jest.useFakeTimers(); + let room = server(() => true); + let doc = new Y.Doc(); + let provider = new PlanProvider({ wire: room.transport, doc }); + await provider.connect(); + let text = doc.getText("typed"); + for (let i = 0; i < 64; i++) { + text.insert(text.length, "y".repeat(4_096)); + jest.advanceTimersByTime(20); + } + + room.drop = () => false; + let before = room.received.length; + await provider.resume(); + let replayed = room.received.slice(before); + + expect(replayed.length).toBeGreaterThan(1); + for (let entry of replayed) expect(entry.bytes).toBeLessThanOrEqual(64 * 1024); + expect(room.doc.getText("typed").length).toBe(64 * 4_096); + }); +}); diff --git a/packages/editor/src/provider.ts b/packages/editor/src/provider.ts index b0d12baa4..ca951f5d8 100644 --- a/packages/editor/src/provider.ts +++ b/packages/editor/src/provider.ts @@ -81,6 +81,32 @@ export type PlanProviderOptions = { const MAX_OUTBOX_BYTES = 2 * 1024 * 1024; const MAX_OUTBOX_ITEMS = 1_000; +/** + * Pacing for outgoing updates. + * + * The server takes at most 200 updates a second from one socket and drops the + * rest without replying. Typing faster than that (a script, a held key with a + * fast repeat) lost the excess, and Yjs then held back every later update from + * this client until the missing one arrived, so the rest of the session never + * reached anyone. Updates made within one interval go out merged, which keeps + * this client well under the limit, and anything still unacknowledged after a + * while is sent again in case it was dropped anyway. + */ +const SEND_MS = 10; +/** + * Resends back off from this, doubling up to `MAX_RESEND_MS`, so a server that + * is slow to commit is not also asked to take every backlog again. + */ +const RESEND_MS = 2_000; +const MAX_RESEND_MS = 30_000; +/** + * An update sent this many times without an acknowledgement is left for the + * next open to replay. Something other than a dropped frame is refusing it. + */ +const MAX_SENDS = 5; +/** Merged updates stay far below the server's per-update limit. */ +const MERGE_BYTES = 64 * 1024; + export class PlanProvider implements Provider { /** * y-protocols' `Awareness` is structurally what Lexical wants, but its state @@ -98,6 +124,15 @@ export class PlanProvider implements Provider { /** Updates sent but not yet acknowledged, replayed after a reconnect. */ readonly #outbox = new Map(); #outboxBytes = 0; + /** Outbox entries waiting for the current send interval to end. */ + readonly #unsent = new Set(); + /** Outbox entries already outstanding when the resend timer last fired. */ + #overdue = new Set(); + /** How often each outbox entry has been sent since the last open. */ + readonly #sends = new Map(); + #resendDelay = RESEND_MS; + #pacing: ReturnType | undefined; + #resending: ReturnType | undefined; #epoch: string | undefined; #synced = false; @@ -131,11 +166,53 @@ export class PlanProvider implements Provider { * never be acknowledged, because they were never received. */ #coalesce(): void { - if (this.#outbox.size < 2) return; - let merged = Y.mergeUpdates([...this.#outbox.values()]); - this.#outbox.clear(); - this.#outbox.set(`merged-${this.#counter++}`, merged); - this.#outboxBytes = merged.byteLength; + this.#merge([...this.#outbox.keys()]); + } + + /** + * Replace outbox entries with as few merged ones as stay under + * `MERGE_BYTES`, and return the ids that now hold them. + * + * A merged entry carries a fresh id: acknowledgements for the ones it + * replaces may still arrive and are ignored, and the merged one is only + * settled by its own. + */ + #merge(ids: string[]): string[] { + let groups: string[][] = []; + let size = 0; + for (let id of ids) { + let bytes = this.#outbox.get(id)?.byteLength; + if (bytes === undefined) continue; + let group = groups.at(-1); + if (!group || size + bytes > MERGE_BYTES) { + groups.push([id]); + size = bytes; + } else { + group.push(id); + size += bytes; + } + } + + return groups.map(group => { + if (group.length === 1) return group[0]!; + let merged = Y.mergeUpdates(group.map(id => this.#outbox.get(id)!)); + let id = `merged-${this.#counter++}`; + let unsent = false; + let sends = 0; + for (let part of group) { + this.#outboxBytes -= this.#outbox.get(part)!.byteLength; + this.#outbox.delete(part); + if (this.#unsent.delete(part)) unsent = true; + this.#overdue.delete(part); + sends = Math.max(sends, this.#sends.get(part) ?? 0); + this.#sends.delete(part); + } + this.#outbox.set(id, merged); + this.#outboxBytes += merged.byteLength; + if (unsent) this.#unsent.add(id); + if (sends > 0) this.#sends.set(id, sends); + return id; + }); } // -- provider surface ---------------------------------------------------- @@ -199,6 +276,10 @@ export class PlanProvider implements Provider { this.#wire.on("plan:ack", event => this.#settle(event.id)), this.#wire.on("plan:awareness", event => this.#presence(event)), this.#wire.on("plan:reset", event => this.#reset(event)), + // Refused outright rather than dropped: repeating it changes nothing. + this.#wire.on<{ message?: string }>("session:error", event => { + if (event.message === "implementation is active") this.#park(); + }), this.#wire.on("plan:changes", event => { if (event.epoch === this.#epoch) this.#options.onChanges?.(event.changes); }), @@ -242,6 +323,7 @@ export class PlanProvider implements Provider { this.#doc.off("update", this.#local); this.awareness.off("update", this.#announce); this.awareness.destroy(); + this.#stopTimers(); this.#wire.send("plan:close", {}); this.#emit("status", { status: "disconnected" }); @@ -278,6 +360,9 @@ export class PlanProvider implements Provider { if (rotated) { this.#outbox.clear(); this.#outboxBytes = 0; + this.#unsent.clear(); + this.#overdue.clear(); + this.#sends.clear(); } else { this.#replay(); } @@ -321,9 +406,14 @@ export class PlanProvider implements Provider { * * Yjs updates are idempotent, so a duplicate is harmless — losing one is * not, which is why the outbox survives a reconnect on the same epoch. + * Merged first, so a long backlog does not arrive faster than the server + * accepts it. */ #replay(): void { - for (let [id, update] of this.#outbox) this.#send(id, update); + this.#unsent.clear(); + this.#sends.clear(); + this.#resendDelay = RESEND_MS; + for (let id of this.#merge([...this.#outbox.keys()])) this.#send(id); } #local = (update: Uint8Array, origin: unknown): void => { @@ -333,22 +423,85 @@ export class PlanProvider implements Provider { let id = `${Date.now().toString(36)}-${this.#counter++}`; this.#outbox.set(id, update); this.#outboxBytes += update.byteLength; + this.#unsent.add(id); if (this.saturated) this.#coalesce(); - this.#send(id, update); + if (!this.#pacing) this.#flush(); }; + /** Send what this interval produced, and hold anything newer until it ends. */ + #flush(): void { + if (!this.#epoch || this.#unsent.size === 0) return; + let ids = this.#merge([...this.#unsent]); + this.#unsent.clear(); + for (let id of ids) this.#send(id); + this.#pacing = setTimeout(() => { + this.#pacing = undefined; + this.#flush(); + }, SEND_MS); + } + + /** + * Send again whatever has gone a whole interval without an acknowledgement. + * + * Yjs updates are idempotent, so resending one that was merely slow costs + * a duplicate; not resending one that was dropped costs everything after it. + */ + #resend(): void { + this.#resending = undefined; + if (!this.#connected || !this.#synced || !this.#epoch) { + // The next open replays the whole outbox. + this.#overdue.clear(); + return; + } + let overdue = [...this.#outbox.keys()].filter(id => + this.#overdue.has(id) && !this.#unsent.has(id) && (this.#sends.get(id) ?? 0) < MAX_SENDS + ); + if (overdue.length > 0) { + this.#resendDelay = Math.min(this.#resendDelay * 2, MAX_RESEND_MS); + for (let id of this.#merge(overdue)) this.#send(id); + } + this.#overdue = new Set(this.#outbox.keys()); + let waiting = [...this.#outbox.keys()].some(id => (this.#sends.get(id) ?? 0) < MAX_SENDS); + if (waiting) this.#armResend(); + } + + #armResend(): void { + this.#resending ??= setTimeout(() => this.#resend(), this.#resendDelay); + } + + /** Stop resending what is outstanding now. The next open still replays it. */ + #park(): void { + for (let id of this.#outbox.keys()) { + if (!this.#unsent.has(id)) this.#sends.set(id, MAX_SENDS); + } + } + + #stopTimers(): void { + clearTimeout(this.#pacing); + clearTimeout(this.#resending); + this.#pacing = undefined; + this.#resending = undefined; + this.#unsent.clear(); + this.#overdue.clear(); + this.#sends.clear(); + this.#resendDelay = RESEND_MS; + } + /** * Send one update. * * Fire-and-forget rather than a correlated request: acknowledgements arrive * as `plan:ack` and are matched by `id`, so a keystroke does not cost a - * pending promise, and an update whose ack is lost simply stays in the - * outbox until the next open replays it. + * pending promise, and an update whose ack never arrives stays in the + * outbox until `#resend` or the next open sends it again. */ - #send(id: string, update: Uint8Array): void { - if (!this.#epoch) return; + #send(id: string): void { + let update = this.#outbox.get(id); + if (!this.#epoch || !update) return; this.#wire.send("plan:update", { epoch: this.#epoch, id, update: encode(update) }); + this.#sends.set(id, (this.#sends.get(id) ?? 0) + 1); + this.#armResend(); } #settle(id: string): void { @@ -356,6 +509,10 @@ export class PlanProvider implements Provider { if (!update) return; this.#outbox.delete(id); this.#outboxBytes -= update.byteLength; + this.#overdue.delete(id); + this.#sends.delete(id); + // The server is accepting again, so the next resend need not wait long. + this.#resendDelay = RESEND_MS; } #remote(event: Plan.Update): void { @@ -395,9 +552,12 @@ export class PlanProvider implements Provider { * boundary where continuity ends. */ #reset(event: Plan.Reset): void { - if (event.epoch === this.#epoch) return; + // The same epoch is the server refusing an oversized update while + // keeping the document. Sending it again would be refused again. + if (event.epoch === this.#epoch) return this.#park(); this.#outbox.clear(); + this.#stopTimers(); this.#generation++; this.#outboxBytes = 0; this.#epoch = undefined;