From 9b1dc110114eee0ca7d504e6d6a1731748942709 Mon Sep 17 00:00:00 2001 From: QQ Date: Sat, 14 Nov 2026 15:00:11 +0800 Subject: [PATCH] Import packages/plugin/tests/client-reconnect.test.ts --- .../plugin/tests/client-reconnect.test.ts | 326 ++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 packages/plugin/tests/client-reconnect.test.ts diff --git a/packages/plugin/tests/client-reconnect.test.ts b/packages/plugin/tests/client-reconnect.test.ts new file mode 100644 index 0000000..3f9fef1 --- /dev/null +++ b/packages/plugin/tests/client-reconnect.test.ts @@ -0,0 +1,326 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { RemoteClientCore } from '@dsh-remote/client-core' +import { BaseTransport } from '@dsh-remote/webrtc' +import { createEvent, createRpcResponse, decodeMessage, encodeMessage } from '@dsh-remote/protocol' +import { afterEach, expect, it, vi } from 'vitest' +import { runScenarios } from '../../../scripts/test-scenarios.mjs' +import { ClientModeRuntime, remoteHostFeatures } from '../src/client-runtime.js' +import type { ConnectedRemote } from '../src/client-runtime-contract.js' +import { resolveConfig } from '../src/config.js' +import { IdentityStore } from '../src/identity-store.js' +import type { SafeLogger } from '../src/logging.js' +import type { ClientServerApi } from '../src/server-api.js' +import type { TypertGatewayLike } from '../src/typert-gateway-contract.js' +import { subscribeRemoteRecovery } from '../src/client-recovery.js' +import type { RemoteStatus } from '../src/client/types.js' + +class Carrier extends BaseTransport { + connected = true + reply = true + sendGate?: Promise + responseFor?: (request: ReturnType) => unknown + sent: ReturnType[] = [] + async connect() {} + async close() { this.connected = false } + async send(bytes: Uint8Array) { + const request = decodeMessage(bytes) + this.sent.push(request) + await this.sendGate + if (this.reply) this.emit(encodeMessage(createRpcResponse(request.id, this.responseFor?.(request) ?? { ok: true, value: 'existing-host-session' }))) + } + getStats() { return { connected: this.connected, mode: this.connected ? 'Relay' as const : 'Disconnected' as const } } + drop(error?: Error) { this.connected = false; this.emitClose(error) } + pushEvent(event: 'harness.remote.frame', data: unknown) { this.emit(encodeMessage(createEvent(event, data))) } +} + +let directory: string | undefined +let runtime: ClientModeRuntime | undefined +const cleanup = async () => { + await runtime?.close() + runtime = undefined + vi.useRealTimers() + vi.restoreAllMocks() + if (directory !== undefined) await rm(directory, { recursive: true, force: true }) + directory = undefined +} +afterEach(cleanup) + +async function fixture(enabled = true) { + directory = await mkdtemp(join(tmpdir(), 'remote-reconnect-')) + const localInvoke = vi.fn(async () => 'local') + const gateway: TypertGatewayLike & { + dispatchRpc: (_endpoint: string, _payload: unknown, _signal: AbortSignal) => Promise<{ ok: true }> + openWireStream: (_endpoint: string, _payload: unknown, _signal: AbortSignal) => Promise> + } = { + invoke: localInvoke, + dispatchRpc: async () => ({ ok: true }), + openWireStream: async () => (async function* () {})(), + wireStream: { open: async () => (async function* () {})(), failure: error => { + const failure = error as { code?: unknown; message?: unknown } + return { + code: typeof failure?.code === 'string' ? failure.code : 'internal', + message: typeof failure?.message === 'string' ? failure.message : '', + details: {}, + } + } }, + } + runtime = new ClientModeRuntime( + resolveConfig({ reconnect: enabled ? { initialDelayMs: 100, maxDelayMs: 1000, jitter: 0 } : false }), + new IdentityStore({ directory }), + { bindIdentity: vi.fn(), revokeCurrentDevice: vi.fn(async () => undefined), authorizeWithAccount: vi.fn(async () => ({})) } as unknown as ClientServerApi, + gateway, { info: vi.fn(), warn: vi.fn(), debug: vi.fn(), error: vi.fn() } as unknown as SafeLogger, + ) + await runtime.start() + const remotes: Array<{ remote: ConnectedRemote; carrier: Carrier }> = [] + const makeRemote = async (deviceId: string) => { + const carrier = new Carrier() + const client = new RemoteClientCore(carrier) + await client.connect() + const remote = { + client, target: { deviceId, name: deviceId }, progressRunId: remotes.length + 1, + features: { ...remoteHostFeatures(), remoteGateway: true }, + } as ConnectedRemote + remotes.push({ remote, carrier }) + return remote + } + const connection = (runtime as unknown as { connection: { connect(id: string, signal?: AbortSignal): Promise } }).connection + const connect = vi.spyOn(connection, 'connect').mockImplementation(makeRemote) + await runtime.setMode('remote', 'host-1') + vi.useFakeTimers() + // ClientReconnect draws its FULL-JITTER backoff from Math.random(); pin it to + // the top of each window so the deterministic timer assertions below mean + // "waits the full window", not "waits somewhere in it". 1 also keeps the + // legacy 0-draw behaviour of the first attempt out of the sequence: the first + // retry now waits a full initialDelayMs. + vi.spyOn(Math, 'random').mockReturnValue(0.999999) + return { runtime, gateway, localInvoke, connect, remotes, makeRemote } +} + +it('recovers the selected Host with bounded backoff and fresh reads without replaying in-flight mutations', async () => { + const f = await fixture() + const first = f.remotes[0]!.carrier + first.reply = false + const prompt = f.gateway.invoke({ namespace: 'session', method: 'prompt', args: { sessionId: 'existing-host-session' } }) + const approval = f.gateway.dispatchRpc('$events/result', { args: { + clientId: 'event-client', eventId: 'approval-1', outcome: { kind: 'result', value: 'allowed-once' }, + } }, new AbortController().signal) + const rejectedPrompt = expect(prompt).rejects.toMatchObject({ code: 'TRANSPORT_CLOSED' }) + const rejectedApproval = expect(approval).resolves.toMatchObject({ ok: false, error: { code: 'TRANSPORT_CLOSED' } }) + expect(first.sent[1]).toMatchObject({ payload: { method: 'harness.remote.call', params: { endpoint: '$events/result' } } }) + f.connect.mockRejectedValueOnce(new TypeError('fetch failed')) + .mockRejectedValueOnce(Object.assign(new Error('HTTP 503'), { code: 'INTERNAL_ERROR', retryable: true })) + .mockRejectedValueOnce(new Error('offline')).mockRejectedValueOnce(new Error('offline')).mockRejectedValueOnce(new Error('offline')) + first.drop() + await Promise.all([rejectedPrompt, rejectedApproval]) + expect(f.runtime.status()).toMatchObject({ mode: 'remote', connected: false, target: { deviceId: 'host-1' } }) + await expect(f.gateway.invoke({ namespace: 'session', method: 'list', args: {} })).rejects.toMatchObject({ code: 'remote/disconnected' }) + await expect(f.gateway.dispatchRpc('$events/result', { args: { + clientId: 'event-client', eventId: 'approval-1', outcome: { kind: 'result', value: 'allowed-once' }, + } }, new AbortController().signal)).resolves.toMatchObject({ ok: false, error: { code: 'remote/disconnected' } }) + expect(first.sent).toHaveLength(2) + // Every retry inside ClientReconnect waits a full-jitter window over + // [0, maxDelayMs] (1000ms here, Math.random() pinned to the top). Assert the + // property that matters: no two attempts land inside the same window. + const sample = async (span: number): Promise => { + const marks: number[] = [] + const startCalls = f.connect.mock.calls.length + for (let elapsed = 0; elapsed < span; elapsed += 100) { + await vi.advanceTimersByTimeAsync(100) + if (f.connect.mock.calls.length > startCalls + marks.length) marks.push(elapsed + 100) + } + return marks + } + const marks = await sample(3_100) + expect(marks.length).toBeGreaterThanOrEqual(2) + // Pinned to the top of the window, attempts land ~1000ms apart. + for (let index = 1; index < marks.length; index++) { + expect(marks[index]! - marks[index - 1]!).toBeGreaterThanOrEqual(1000) + } + // The mocked attempts keep failing (three queued failures) until one lands; + // the recovery only completes once a fresh carrier is up. + await vi.advanceTimersByTimeAsync(10_000) + expect(f.runtime.status()).toMatchObject({ connected: true, recoveryGeneration: 1, target: { deviceId: 'host-1' } }) + expect(f.remotes.at(-1)!.carrier.sent).toEqual([]) + await expect(f.gateway.invoke({ namespace: 'session', method: 'list', args: {} })).resolves.toBe('existing-host-session') + expect(f.localInvoke).not.toHaveBeenCalled() + expect(first.sent).toHaveLength(2) + f.remotes.at(-1)!.carrier.drop() + await vi.advanceTimersByTimeAsync(1000) + expect(f.runtime.status()).toMatchObject({ connected: true, recoveryGeneration: 2 }) +}) + +it.each(['close', 'retry'] as const)('exposes terminal client authorization on %s and clears it after switching back', async phase => { + const f = await fixture() + const rejection = Object.assign(new Error('credential rejected'), { code: 'AUTH_INVALID', retryable: false }) + if (phase === 'retry') f.connect.mockRejectedValueOnce(rejection) + f.remotes[0]!.carrier.drop(phase === 'close' ? rejection : undefined) + if (phase === 'retry') expect(f.runtime.status()).not.toHaveProperty('clientAuthRequired') + await vi.advanceTimersByTimeAsync(60_000) + expect(f.runtime.status()).toMatchObject({ connected: false, clientAuthRequired: true, target: { deviceId: 'host-1' } }) + expect(f.connect).toHaveBeenCalledTimes(phase === 'close' ? 1 : 2) + await expect(f.gateway.invoke({ namespace: 'session', method: 'list', args: {} })).rejects.toMatchObject({ code: 'remote/disconnected' }) + if (phase === 'close') await f.runtime.setMode('local') + else { + await f.runtime.authorizeClientWithAccount('test@example.invalid', 'test') + expect(f.runtime.status()).not.toHaveProperty('clientAuthRequired') + await f.runtime.setMode('remote', 'host-1') + expect(f.runtime.status()).toMatchObject({ connected: true, target: { deviceId: 'host-1' } }) + } + expect(f.runtime.status()).not.toHaveProperty('clientAuthRequired') +}) + +it('does not replay an approval whose old carrier send settles after recovery', async () => { + const f = await fixture() + const first = f.remotes[0]!.carrier + let finishSend!: () => void + first.sendGate = new Promise(resolve => { finishSend = resolve }) + const approval = f.gateway.dispatchRpc('$events/result', { args: { + clientId: 'event-client', eventId: 'approval-2', outcome: { kind: 'result', value: 'allowed-once' }, + } }, new AbortController().signal) + const rejected = expect(approval).resolves.toMatchObject({ ok: false, error: { code: 'TRANSPORT_CLOSED' } }) + expect(first.sent).toHaveLength(1) + expect(first.sent[0]).toMatchObject({ payload: { method: 'harness.remote.call', params: { endpoint: '$events/result' } } }) + + first.drop() + await rejected + await vi.advanceTimersByTimeAsync(1000) + expect(f.runtime.status()).toMatchObject({ connected: true, recoveryGeneration: 1 }) + expect(f.remotes[1]!.carrier.sent).toEqual([]) + + finishSend() + await vi.advanceTimersByTimeAsync(0) + expect(f.remotes[1]!.carrier.sent).toEqual([]) + expect(f.localInvoke).not.toHaveBeenCalled() +}) + +it('reopens a read stream and loads Host history from the recovered carrier', async () => { + const f = await fixture() + const first = f.remotes[0]!.carrier + const record = (seq: number) => ({ type: 'event', event: { seq, time: seq, type: 'test/entry', data: { text: `host-${seq}` } } }) + const respond = (cursor: number) => (request: ReturnType): unknown => { + const payload = request.payload as { method: string; params?: { endpoint?: string } } + if (payload.method === 'harness.transport.describe') return { capabilities: [] } + if (payload.method === 'harness.remote.stream.open') return { opened: true } + if (payload.method === 'harness.remote.call' && payload.params?.endpoint === 'session/history') { + return { ok: true, value: { records: [record(cursor)], throughSeq: cursor } } + } + return { ok: true, value: 'existing-host-session' } + } + const streamId = (carrier: Carrier): string => { + const opening = carrier.sent.find(request => (request.payload as { method?: string }).method === 'harness.remote.stream.open') + return (opening?.payload as { params?: { streamId?: string } } | undefined)?.params?.streamId ?? '' + } + const request = { args: { request: { sessionId: 'existing-host-session' } } } + const oldLifetime = new AbortController() + const signal = new AbortController().signal + first.responseFor = respond(3) + const oldStream = await f.gateway.openWireStream('session/follow', request, oldLifetime.signal) + const oldIterator = oldStream[Symbol.asyncIterator]() + const oldId = streamId(first) + expect(oldId).not.toBe('') + first.pushEvent('harness.remote.frame', { streamId: oldId, hasValue: true, value: { + type: 'snapshot', cursor: 3, records: [record(3)], hasMore: false, + } }) + await expect(oldIterator.next()).resolves.toMatchObject({ value: { type: 'snapshot', cursor: 3 } }) + const settled = vi.fn() + const interrupted = oldIterator.next().catch(error => { settled(); throw error }) + const released = expect(interrupted).rejects.toMatchObject({ code: 'TRANSPORT_CLOSED' }) + + first.drop() + // Main's full-jitter backoff draws from [0, maxDelayMs] even for the first + // retry; the fixture pins Math.random() to the top of the window. + await vi.advanceTimersByTimeAsync(1000) + expect(f.runtime.status()).toMatchObject({ connected: true, recoveryGeneration: 1 }) + expect(settled).not.toHaveBeenCalled() + // The native carrier reset cancels its old readers after authentication. + oldLifetime.abort() + await released + + const next = f.remotes[1]!.carrier + next.responseFor = respond(8) + const newStream = await f.gateway.openWireStream('session/follow', request, signal) + const newIterator = newStream[Symbol.asyncIterator]() + const newId = streamId(next) + expect(newId).not.toBe('') + expect(newId).not.toBe(oldId) + next.pushEvent('harness.remote.frame', { streamId: newId, hasValue: true, value: { + type: 'snapshot', cursor: 8, records: [record(8)], hasMore: false, + } }) + await expect(newIterator.next()).resolves.toMatchObject({ value: { type: 'snapshot', cursor: 8, records: [record(8)] } }) + await expect(f.gateway.dispatchRpc('session/history', request, signal)).resolves.toMatchObject({ + ok: true, value: { throughSeq: 8, records: [record(8)] }, + }) + expect(next.sent.filter(frame => (frame.payload as { method?: string }).method === 'harness.remote.call')).toHaveLength(1) + expect(f.localInvoke).not.toHaveBeenCalled() + await newIterator.return?.() +}) + +it('stops timers and retires late candidates on exit, sign-out, shutdown and Host replacement', async () => { + await runScenarios((['exit', 'sign-out', 'shutdown', 'switch'] as const).flatMap(action => + (['waiting', 'connecting'] as const).map(phase => ({ name: `${action}: ${phase}`, run: async () => { + const f = await fixture() + let resolve: ((remote: ConnectedRemote) => void) | undefined + let signal: AbortSignal | undefined + f.connect.mockImplementationOnce((_id, requestSignal) => { + signal = requestSignal + return new Promise(yes => { resolve = yes }) + }) + f.remotes[0]!.carrier.drop() + if (phase === 'connecting') await vi.advanceTimersByTimeAsync(1000) + else f.connect.mockReset().mockImplementation(f.makeRemote) + if (action === 'exit') await f.runtime.setMode('local') + else if (action === 'sign-out') await f.runtime.clearClientAuthorization() + else if (action === 'shutdown') await f.runtime.close() + else await f.runtime.setMode('remote', 'host-2') + if (resolve !== undefined) { + expect(signal?.aborted).toBe(true) + const late = await f.makeRemote('host-1') + resolve(late) + await vi.advanceTimersByTimeAsync(0) + expect(late.client.getStats().connected).toBe(false) + } + const count = f.connect.mock.calls.length + await vi.advanceTimersByTimeAsync(5000) + expect(f.connect).toHaveBeenCalledTimes(count) + expect(f.runtime.status()).toMatchObject(action === 'switch' + ? { mode: 'remote', connected: true, target: { deviceId: 'host-2' } } + : { mode: 'local', connected: false }) + } }))), cleanup) +}) + +it('does not reclaim replaced identities, revoked membership, rejected encryption or disabled reconnect', async () => { + await runScenarios(['CONNECTION_REPLACED', 'PEER_IDENTITY_MISMATCH', 'MEMBERSHIP_REQUIRED', 'DEVICE_REVOKED', 'SECURE_CHANNEL_FAILED', 'CLIENT_CLOSED', 'disabled'].flatMap(code => + ['drop', 'retry'].map(phase => ({ name: `${code}: ${phase}`, run: async () => { + const f = await fixture(code !== 'disabled') + const failure = Object.assign(new Error('rejected'), { code, retryable: true }) + if (phase === 'retry') f.connect.mockRejectedValueOnce(failure) + f.remotes[0]!.carrier.drop(phase === 'drop' ? failure : undefined) + await vi.advanceTimersByTimeAsync(5000) + expect(f.connect).toHaveBeenCalledTimes(phase === 'retry' && code !== 'disabled' ? 2 : 1) + expect(f.runtime.status()).toMatchObject({ mode: 'remote', connected: false, recoveryGeneration: 0 }) + expect(f.localInvoke).not.toHaveBeenCalled() + } }))), cleanup) +}) + +it('reopens native read subscriptions once per recovered generation, without replay or page reload', () => { + let listener!: (status: RemoteStatus) => void + const unsubscribe = vi.fn() + const reconnect = vi.fn() + const stop = subscribeRemoteRecovery({ subscribe: next => { listener = next; return unsubscribe }, getSnapshot: () => undefined, close() {} }, reconnect) + const status = (id: string, connected: boolean, recoveryGeneration: number): RemoteStatus => ({ mode: 'remote', target: { deviceId: id, name: id }, connected, recoveryGeneration, available: true, hostAuthorizationAvailable: false }) + listener(status('host-1', true, 0)) + listener(status('host-1', false, 0)) + listener(status('host-1', true, 1)) + listener(status('host-1', true, 1)) + expect(reconnect).toHaveBeenCalledTimes(1) + listener(status('host-1', true, 2)) // Even when the short outage was between status samples. + expect(reconnect).toHaveBeenCalledTimes(2) + listener(status('host-2', true, 3)) + listener({ ...status('host-2', false, 3), mode: 'local' }) + listener(status('host-2', true, 4)) + expect(reconnect).toHaveBeenCalledTimes(2) + stop() + expect(unsubscribe).toHaveBeenCalledOnce() +})