import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { RemoteClientCore } from '@dsh-remote/client-core' import { BaseTransport } from '@dsh-remote/webrtc' import { createEvent, createRpcResponse, decodeMessage, encodeMessage } from '@dsh-remote/protocol' import { afterEach, expect, it, vi } from 'vitest' import { runScenarios } from '../../../scripts/test-scenarios.mjs' import { ClientModeRuntime, remoteHostFeatures } from '../src/client-runtime.js' import type { ConnectedRemote } from '../src/client-runtime-contract.js' import { resolveConfig } from '../src/config.js' import { IdentityStore } from '../src/identity-store.js' import type { SafeLogger } from '../src/logging.js' import type { ClientServerApi } from '../src/server-api.js' import type { TypertGatewayLike } from '../src/typert-gateway-contract.js' import { subscribeRemoteRecovery } from '../src/client-recovery.js' import type { RemoteStatus } from '../src/client/types.js' class Carrier extends BaseTransport { connected = true reply = true sendGate?: Promise responseFor?: (request: ReturnType) => unknown sent: ReturnType[] = [] async connect() {} async close() { this.connected = false } async send(bytes: Uint8Array) { const request = decodeMessage(bytes) this.sent.push(request) await this.sendGate if (this.reply) this.emit(encodeMessage(createRpcResponse(request.id, this.responseFor?.(request) ?? { ok: true, value: 'existing-host-session' }))) } getStats() { return { connected: this.connected, mode: this.connected ? 'Relay' as const : 'Disconnected' as const } } drop(error?: Error) { this.connected = false; this.emitClose(error) } pushEvent(event: 'harness.remote.frame', data: unknown) { this.emit(encodeMessage(createEvent(event, data))) } } let directory: string | undefined let runtime: ClientModeRuntime | undefined const cleanup = async () => { await runtime?.close() runtime = undefined vi.useRealTimers() vi.restoreAllMocks() if (directory !== undefined) await rm(directory, { recursive: true, force: true }) directory = undefined } afterEach(cleanup) async function fixture(enabled = true) { directory = await mkdtemp(join(tmpdir(), 'remote-reconnect-')) const localInvoke = vi.fn(async () => 'local') const gateway: TypertGatewayLike & { dispatchRpc: (_endpoint: string, _payload: unknown, _signal: AbortSignal) => Promise<{ ok: true }> openWireStream: (_endpoint: string, _payload: unknown, _signal: AbortSignal) => Promise> } = { invoke: localInvoke, dispatchRpc: async () => ({ ok: true }), openWireStream: async () => (async function* () {})(), wireStream: { open: async () => (async function* () {})(), failure: error => { const failure = error as { code?: unknown; message?: unknown } return { code: typeof failure?.code === 'string' ? failure.code : 'internal', message: typeof failure?.message === 'string' ? failure.message : '', details: {}, } } }, } runtime = new ClientModeRuntime( resolveConfig({ reconnect: enabled ? { initialDelayMs: 100, maxDelayMs: 1000, jitter: 0 } : false }), new IdentityStore({ directory }), { bindIdentity: vi.fn(), revokeCurrentDevice: vi.fn(async () => undefined), authorizeWithAccount: vi.fn(async () => ({})) } as unknown as ClientServerApi, gateway, { info: vi.fn(), warn: vi.fn(), debug: vi.fn(), error: vi.fn() } as unknown as SafeLogger, ) await runtime.start() const remotes: Array<{ remote: ConnectedRemote; carrier: Carrier }> = [] const makeRemote = async (deviceId: string) => { const carrier = new Carrier() const client = new RemoteClientCore(carrier) await client.connect() const remote = { client, target: { deviceId, name: deviceId }, progressRunId: remotes.length + 1, features: { ...remoteHostFeatures(), remoteGateway: true }, } as ConnectedRemote remotes.push({ remote, carrier }) return remote } const connection = (runtime as unknown as { connection: { connect(id: string, signal?: AbortSignal): Promise } }).connection const connect = vi.spyOn(connection, 'connect').mockImplementation(makeRemote) await runtime.setMode('remote', 'host-1') vi.useFakeTimers() // ClientReconnect draws its FULL-JITTER backoff from Math.random(); pin it to // the top of each window so the deterministic timer assertions below mean // "waits the full window", not "waits somewhere in it". 1 also keeps the // legacy 0-draw behaviour of the first attempt out of the sequence: the first // retry now waits a full initialDelayMs. vi.spyOn(Math, 'random').mockReturnValue(0.999999) return { runtime, gateway, localInvoke, connect, remotes, makeRemote } } it('recovers the selected Host with bounded backoff and fresh reads without replaying in-flight mutations', async () => { const f = await fixture() const first = f.remotes[0]!.carrier first.reply = false const prompt = f.gateway.invoke({ namespace: 'session', method: 'prompt', args: { sessionId: 'existing-host-session' } }) const approval = f.gateway.dispatchRpc('$events/result', { args: { clientId: 'event-client', eventId: 'approval-1', outcome: { kind: 'result', value: 'allowed-once' }, } }, new AbortController().signal) const rejectedPrompt = expect(prompt).rejects.toMatchObject({ code: 'TRANSPORT_CLOSED' }) const rejectedApproval = expect(approval).resolves.toMatchObject({ ok: false, error: { code: 'TRANSPORT_CLOSED' } }) expect(first.sent[1]).toMatchObject({ payload: { method: 'harness.remote.call', params: { endpoint: '$events/result' } } }) f.connect.mockRejectedValueOnce(new TypeError('fetch failed')) .mockRejectedValueOnce(Object.assign(new Error('HTTP 503'), { code: 'INTERNAL_ERROR', retryable: true })) .mockRejectedValueOnce(new Error('offline')).mockRejectedValueOnce(new Error('offline')).mockRejectedValueOnce(new Error('offline')) first.drop() await Promise.all([rejectedPrompt, rejectedApproval]) expect(f.runtime.status()).toMatchObject({ mode: 'remote', connected: false, target: { deviceId: 'host-1' } }) await expect(f.gateway.invoke({ namespace: 'session', method: 'list', args: {} })).rejects.toMatchObject({ code: 'remote/disconnected' }) await expect(f.gateway.dispatchRpc('$events/result', { args: { clientId: 'event-client', eventId: 'approval-1', outcome: { kind: 'result', value: 'allowed-once' }, } }, new AbortController().signal)).resolves.toMatchObject({ ok: false, error: { code: 'remote/disconnected' } }) expect(first.sent).toHaveLength(2) // Every retry inside ClientReconnect waits a full-jitter window over // [0, maxDelayMs] (1000ms here, Math.random() pinned to the top). Assert the // property that matters: no two attempts land inside the same window. const sample = async (span: number): Promise => { const marks: number[] = [] const startCalls = f.connect.mock.calls.length for (let elapsed = 0; elapsed < span; elapsed += 100) { await vi.advanceTimersByTimeAsync(100) if (f.connect.mock.calls.length > startCalls + marks.length) marks.push(elapsed + 100) } return marks } const marks = await sample(3_100) expect(marks.length).toBeGreaterThanOrEqual(2) // Pinned to the top of the window, attempts land ~1000ms apart. for (let index = 1; index < marks.length; index++) { expect(marks[index]! - marks[index - 1]!).toBeGreaterThanOrEqual(1000) } // The mocked attempts keep failing (three queued failures) until one lands; // the recovery only completes once a fresh carrier is up. await vi.advanceTimersByTimeAsync(10_000) expect(f.runtime.status()).toMatchObject({ connected: true, recoveryGeneration: 1, target: { deviceId: 'host-1' } }) expect(f.remotes.at(-1)!.carrier.sent).toEqual([]) await expect(f.gateway.invoke({ namespace: 'session', method: 'list', args: {} })).resolves.toBe('existing-host-session') expect(f.localInvoke).not.toHaveBeenCalled() expect(first.sent).toHaveLength(2) f.remotes.at(-1)!.carrier.drop() await vi.advanceTimersByTimeAsync(1000) expect(f.runtime.status()).toMatchObject({ connected: true, recoveryGeneration: 2 }) }) it.each(['close', 'retry'] as const)('exposes terminal client authorization on %s and clears it after switching back', async phase => { const f = await fixture() const rejection = Object.assign(new Error('credential rejected'), { code: 'AUTH_INVALID', retryable: false }) if (phase === 'retry') f.connect.mockRejectedValueOnce(rejection) f.remotes[0]!.carrier.drop(phase === 'close' ? rejection : undefined) if (phase === 'retry') expect(f.runtime.status()).not.toHaveProperty('clientAuthRequired') await vi.advanceTimersByTimeAsync(60_000) expect(f.runtime.status()).toMatchObject({ connected: false, clientAuthRequired: true, target: { deviceId: 'host-1' } }) expect(f.connect).toHaveBeenCalledTimes(phase === 'close' ? 1 : 2) await expect(f.gateway.invoke({ namespace: 'session', method: 'list', args: {} })).rejects.toMatchObject({ code: 'remote/disconnected' }) if (phase === 'close') await f.runtime.setMode('local') else { await f.runtime.authorizeClientWithAccount('test@example.invalid', 'test') expect(f.runtime.status()).not.toHaveProperty('clientAuthRequired') await f.runtime.setMode('remote', 'host-1') expect(f.runtime.status()).toMatchObject({ connected: true, target: { deviceId: 'host-1' } }) } expect(f.runtime.status()).not.toHaveProperty('clientAuthRequired') }) it('does not replay an approval whose old carrier send settles after recovery', async () => { const f = await fixture() const first = f.remotes[0]!.carrier let finishSend!: () => void first.sendGate = new Promise(resolve => { finishSend = resolve }) const approval = f.gateway.dispatchRpc('$events/result', { args: { clientId: 'event-client', eventId: 'approval-2', outcome: { kind: 'result', value: 'allowed-once' }, } }, new AbortController().signal) const rejected = expect(approval).resolves.toMatchObject({ ok: false, error: { code: 'TRANSPORT_CLOSED' } }) expect(first.sent).toHaveLength(1) expect(first.sent[0]).toMatchObject({ payload: { method: 'harness.remote.call', params: { endpoint: '$events/result' } } }) first.drop() await rejected await vi.advanceTimersByTimeAsync(1000) expect(f.runtime.status()).toMatchObject({ connected: true, recoveryGeneration: 1 }) expect(f.remotes[1]!.carrier.sent).toEqual([]) finishSend() await vi.advanceTimersByTimeAsync(0) expect(f.remotes[1]!.carrier.sent).toEqual([]) expect(f.localInvoke).not.toHaveBeenCalled() }) it('reopens a read stream and loads Host history from the recovered carrier', async () => { const f = await fixture() const first = f.remotes[0]!.carrier const record = (seq: number) => ({ type: 'event', event: { seq, time: seq, type: 'test/entry', data: { text: `host-${seq}` } } }) const respond = (cursor: number) => (request: ReturnType): unknown => { const payload = request.payload as { method: string; params?: { endpoint?: string } } if (payload.method === 'harness.transport.describe') return { capabilities: [] } if (payload.method === 'harness.remote.stream.open') return { opened: true } if (payload.method === 'harness.remote.call' && payload.params?.endpoint === 'session/history') { return { ok: true, value: { records: [record(cursor)], throughSeq: cursor } } } return { ok: true, value: 'existing-host-session' } } const streamId = (carrier: Carrier): string => { const opening = carrier.sent.find(request => (request.payload as { method?: string }).method === 'harness.remote.stream.open') return (opening?.payload as { params?: { streamId?: string } } | undefined)?.params?.streamId ?? '' } const request = { args: { request: { sessionId: 'existing-host-session' } } } const oldLifetime = new AbortController() const signal = new AbortController().signal first.responseFor = respond(3) const oldStream = await f.gateway.openWireStream('session/follow', request, oldLifetime.signal) const oldIterator = oldStream[Symbol.asyncIterator]() const oldId = streamId(first) expect(oldId).not.toBe('') first.pushEvent('harness.remote.frame', { streamId: oldId, hasValue: true, value: { type: 'snapshot', cursor: 3, records: [record(3)], hasMore: false, } }) await expect(oldIterator.next()).resolves.toMatchObject({ value: { type: 'snapshot', cursor: 3 } }) const settled = vi.fn() const interrupted = oldIterator.next().catch(error => { settled(); throw error }) const released = expect(interrupted).rejects.toMatchObject({ code: 'TRANSPORT_CLOSED' }) first.drop() // Main's full-jitter backoff draws from [0, maxDelayMs] even for the first // retry; the fixture pins Math.random() to the top of the window. await vi.advanceTimersByTimeAsync(1000) expect(f.runtime.status()).toMatchObject({ connected: true, recoveryGeneration: 1 }) expect(settled).not.toHaveBeenCalled() // The native carrier reset cancels its old readers after authentication. oldLifetime.abort() await released const next = f.remotes[1]!.carrier next.responseFor = respond(8) const newStream = await f.gateway.openWireStream('session/follow', request, signal) const newIterator = newStream[Symbol.asyncIterator]() const newId = streamId(next) expect(newId).not.toBe('') expect(newId).not.toBe(oldId) next.pushEvent('harness.remote.frame', { streamId: newId, hasValue: true, value: { type: 'snapshot', cursor: 8, records: [record(8)], hasMore: false, } }) await expect(newIterator.next()).resolves.toMatchObject({ value: { type: 'snapshot', cursor: 8, records: [record(8)] } }) await expect(f.gateway.dispatchRpc('session/history', request, signal)).resolves.toMatchObject({ ok: true, value: { throughSeq: 8, records: [record(8)] }, }) expect(next.sent.filter(frame => (frame.payload as { method?: string }).method === 'harness.remote.call')).toHaveLength(1) expect(f.localInvoke).not.toHaveBeenCalled() await newIterator.return?.() }) it('stops timers and retires late candidates on exit, sign-out, shutdown and Host replacement', async () => { await runScenarios((['exit', 'sign-out', 'shutdown', 'switch'] as const).flatMap(action => (['waiting', 'connecting'] as const).map(phase => ({ name: `${action}: ${phase}`, run: async () => { const f = await fixture() let resolve: ((remote: ConnectedRemote) => void) | undefined let signal: AbortSignal | undefined f.connect.mockImplementationOnce((_id, requestSignal) => { signal = requestSignal return new Promise(yes => { resolve = yes }) }) f.remotes[0]!.carrier.drop() if (phase === 'connecting') await vi.advanceTimersByTimeAsync(1000) else f.connect.mockReset().mockImplementation(f.makeRemote) if (action === 'exit') await f.runtime.setMode('local') else if (action === 'sign-out') await f.runtime.clearClientAuthorization() else if (action === 'shutdown') await f.runtime.close() else await f.runtime.setMode('remote', 'host-2') if (resolve !== undefined) { expect(signal?.aborted).toBe(true) const late = await f.makeRemote('host-1') resolve(late) await vi.advanceTimersByTimeAsync(0) expect(late.client.getStats().connected).toBe(false) } const count = f.connect.mock.calls.length await vi.advanceTimersByTimeAsync(5000) expect(f.connect).toHaveBeenCalledTimes(count) expect(f.runtime.status()).toMatchObject(action === 'switch' ? { mode: 'remote', connected: true, target: { deviceId: 'host-2' } } : { mode: 'local', connected: false }) } }))), cleanup) }) it('does not reclaim replaced identities, revoked membership, rejected encryption or disabled reconnect', async () => { await runScenarios(['CONNECTION_REPLACED', 'PEER_IDENTITY_MISMATCH', 'MEMBERSHIP_REQUIRED', 'DEVICE_REVOKED', 'SECURE_CHANNEL_FAILED', 'CLIENT_CLOSED', 'disabled'].flatMap(code => ['drop', 'retry'].map(phase => ({ name: `${code}: ${phase}`, run: async () => { const f = await fixture(code !== 'disabled') const failure = Object.assign(new Error('rejected'), { code, retryable: true }) if (phase === 'retry') f.connect.mockRejectedValueOnce(failure) f.remotes[0]!.carrier.drop(phase === 'drop' ? failure : undefined) await vi.advanceTimersByTimeAsync(5000) expect(f.connect).toHaveBeenCalledTimes(phase === 'retry' && code !== 'disabled' ? 2 : 1) expect(f.runtime.status()).toMatchObject({ mode: 'remote', connected: false, recoveryGeneration: 0 }) expect(f.localInvoke).not.toHaveBeenCalled() } }))), cleanup) }) it('reopens native read subscriptions once per recovered generation, without replay or page reload', () => { let listener!: (status: RemoteStatus) => void const unsubscribe = vi.fn() const reconnect = vi.fn() const stop = subscribeRemoteRecovery({ subscribe: next => { listener = next; return unsubscribe }, getSnapshot: () => undefined, close() {} }, reconnect) const status = (id: string, connected: boolean, recoveryGeneration: number): RemoteStatus => ({ mode: 'remote', target: { deviceId: id, name: id }, connected, recoveryGeneration, available: true, hostAuthorizationAvailable: false }) listener(status('host-1', true, 0)) listener(status('host-1', false, 0)) listener(status('host-1', true, 1)) listener(status('host-1', true, 1)) expect(reconnect).toHaveBeenCalledTimes(1) listener(status('host-1', true, 2)) // Even when the short outage was between status samples. expect(reconnect).toHaveBeenCalledTimes(2) listener(status('host-2', true, 3)) listener({ ...status('host-2', false, 3), mode: 'local' }) listener(status('host-2', true, 4)) expect(reconnect).toHaveBeenCalledTimes(2) stop() expect(unsubscribe).toHaveBeenCalledOnce() })