Import packages/plugin/src/remote-compaction-operations.ts

This commit is contained in:
QQ 2026-11-17 08:48:15 +08:00
parent 3719143bc4
commit da92aee355
1 changed files with 481 additions and 0 deletions

View File

@ -0,0 +1,481 @@
import { randomUUID } from 'node:crypto'
import type {
RemoteCompactionState,
RemoteCompactionStatusParams,
RemoteCompactionStatusResult,
RemoteCompactionSubmitParams,
RemoteCompactionSubmitResult,
} from '@dsh-remote/protocol'
import type { SafeLogger } from './logging.js'
import { RpcError } from './safe-error.js'
import type { TypertRpcResult } from './typert-gateway-contract.js'
/**
* Host-side registry for asynchronous Remote compaction operations.
*
* Harness forwards `commands/execute` through `withAbort()`: once the caller's
* signal aborts, the RPC stops waiting for the handler while the handler keeps
* running. Therefore "the command call returned" does not prove the compaction
* handler, its maintenance lock and its persistence work are finished. This
* registry separates those facts:
*
* - `cancelling`: a stop was requested; the command call may not have settled.
* - `unconfirmed`: the run committed, but cancellation/timing made the final
* persistence result unobservable.
* - `cancelled`/`failed`/`succeeded`: the Host observed a trustworthy final
* result (command result, or `compaction/end` for the same command id).
*
* Terminal states come from the background Harness call. A `compaction/end`
* event alone never fabricates success.
*/
export interface RemoteCompactionOwner {
deviceId: string
connectionId: string
}
/** Runs the Host's fixed `/compact` command for one operation. */
export type RemoteCompactionRunner = (sessionId: string, signal: AbortSignal) => Promise<TypertRpcResult>
/** Outcome of inspecting the Session log for the compaction lifecycle. */
export type RemoteCompactionConfirmation =
| { kind: 'finished'; error?: string }
| { kind: 'not-yet' }
| { kind: 'unknown' }
/**
* Confirms whether the compaction handler actually finished. Implementations
* read the Session log (`command/run`, `compaction/start|end`) by command id.
*/
export type RemoteCompactionConfirmer = (
sessionId: string,
acceptedAt: number,
commandId: string | undefined,
) => Promise<RemoteCompactionConfirmation>
export interface RemoteCompactionOperationsOptions {
confirm?: RemoteCompactionConfirmer
logger?: SafeLogger
now?: () => number
}
type SettledState = 'succeeded' | 'failed' | 'cancelled' | 'unconfirmed'
interface CompactionOperation {
readonly operationId: string
readonly requestId: string
readonly sessionId: string
readonly ownerDeviceId: string
readonly budgetMs: number
readonly acceptedAt: number
readonly controller: AbortController
readonly commandSettled: Promise<void>
resolveCommandSettled: () => void
state: RemoteCompactionState
startedAt?: number
finishedAt?: number
message?: string
cancelled: boolean
timedOut: boolean
commandSettledFlag: boolean
commandId?: string
confirmation: 'pending' | 'finished' | 'unknown'
confirmError?: string
timer?: ReturnType<typeof setTimeout>
}
const MIN_BUDGET_MS = 60_000
const MAX_BUDGET_MS = 7_200_000
const DEFAULT_BUDGET_MS = 1_800_000
/** Keep settled operations readable long enough for a reconnect to collect them. */
const RETENTION_MS = 600_000
/** A cancel request waits this long for the command call to settle. */
const CANCEL_SETTLE_MS = 15_000
/** Bound bound records so a client cannot create unbounded tasks. */
const MAX_OPERATIONS = 32
function requestIndexKey(ownerDeviceId: string, sessionId: string, requestId: string): string {
return `${ownerDeviceId}\u0000${sessionId}\u0000${requestId}`
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
function commandIdOf(result: TypertRpcResult | undefined): string | undefined {
if (result?.ok !== true || !isRecord(result.value)) return undefined
return typeof result.value.commandId === 'string' ? result.value.commandId : undefined
}
export class RemoteCompactionOperations {
private readonly operations = new Map<string, CompactionOperation>()
private readonly requestIndex = new Map<string, string>()
private readonly confirm?: RemoteCompactionConfirmer
private readonly logger?: SafeLogger
private readonly now: () => number
private closed = false
constructor(
private readonly runner: RemoteCompactionRunner,
options: RemoteCompactionOperationsOptions = {},
) {
this.confirm = options.confirm
this.logger = options.logger
this.now = options.now ?? Date.now
}
submit(params: RemoteCompactionSubmitParams, owner: RemoteCompactionOwner): RemoteCompactionSubmitResult {
if (this.closed) throw new RpcError('HARNESS_UNAVAILABLE', 'The Remote Host is shutting down.')
this.prune()
const key = requestIndexKey(owner.deviceId, params.sessionId, params.requestId)
const existing = this.requestIndex.get(key)
if (existing !== undefined) {
const operation = this.operations.get(existing)
if (operation !== undefined) return { operationId: operation.operationId, state: operation.state }
this.requestIndex.delete(key)
}
// One unfinished compaction per Session: new request ids must not stack
// work on top of a run whose handler may still be finishing.
for (const operation of this.operations.values()) {
if (operation.sessionId === params.sessionId && operation.finishedAt === undefined) {
throw new RpcError('AGENT_BUSY', 'A compaction operation is already running for this Session.')
}
}
if (this.operations.size >= MAX_OPERATIONS) {
throw new RpcError('RATE_LIMITED', 'Too many Remote compaction operations are tracked by this Host.', undefined, true)
}
const budgetMs = clampBudget(params.budgetMs)
const operation = this.createOperation(params, owner, budgetMs)
this.operations.set(operation.operationId, operation)
this.requestIndex.set(key, operation.operationId)
this.logger?.info('remote compaction accepted', {
operationId: operation.operationId,
sessionId: operation.sessionId,
budgetMs,
})
void this.run(operation)
return { operationId: operation.operationId, state: 'accepted' }
}
async status(params: RemoteCompactionStatusParams, owner: RemoteCompactionOwner): Promise<RemoteCompactionStatusResult> {
this.prune()
const operation = this.find(params, owner)
if (operation === undefined) return this.interrupted(params)
if (operation.finishedAt === undefined && operation.commandSettledFlag
&& (operation.state === 'cancelling' || operation.state === 'unconfirmed')) {
await this.reconcile(operation)
}
return this.view(operation)
}
async cancel(params: RemoteCompactionStatusParams, owner: RemoteCompactionOwner): Promise<RemoteCompactionStatusResult> {
this.prune()
const operation = this.find(params, owner)
if (operation === undefined) return this.interrupted(params)
if (operation.finishedAt !== undefined) return this.view(operation)
if (!operation.cancelled) {
operation.cancelled = true
operation.state = 'cancelling'
operation.controller.abort(new Error('Remote compaction cancelled by the client.'))
this.logger?.info('remote compaction cancellation requested', { operationId: operation.operationId })
}
// `withAbort` settles the command call early; reconciliation below waits for
// the compaction lifecycle (or reports that it is still unconfirmed).
await Promise.race([operation.commandSettled, delay(CANCEL_SETTLE_MS)])
if (operation.commandSettledFlag) await this.reconcile(operation)
return this.view(operation)
}
/** Host shutdown: stop work and report every unfinished operation as interrupted. */
closeAll(): void {
this.closed = true
for (const operation of this.operations.values()) {
if (operation.finishedAt !== undefined) continue
this.settle(operation, 'interrupted', 'The Remote Host shut down before the compaction finished.')
operation.controller.abort(new Error('Remote Host shut down.'))
}
}
private createOperation(
params: RemoteCompactionSubmitParams,
owner: RemoteCompactionOwner,
budgetMs: number,
): CompactionOperation {
let resolveCommandSettled = () => {}
const commandSettled = new Promise<void>(resolve => { resolveCommandSettled = resolve })
return {
operationId: randomUUID(),
requestId: params.requestId,
sessionId: params.sessionId,
ownerDeviceId: owner.deviceId,
budgetMs,
acceptedAt: this.now(),
controller: new AbortController(),
commandSettled,
resolveCommandSettled,
state: 'accepted',
cancelled: false,
timedOut: false,
commandSettledFlag: false,
confirmation: 'pending',
}
}
private async run(operation: CompactionOperation): Promise<void> {
operation.state = 'running'
operation.startedAt = this.now()
operation.timer = setTimeout(() => {
operation.timedOut = true
operation.controller.abort(new Error('Remote compaction exceeded its budget.'))
}, operation.budgetMs)
let result: TypertRpcResult | undefined
let thrown: unknown
try {
result = await this.runner(operation.sessionId, operation.controller.signal)
} catch (error) {
thrown = error
} finally {
if (operation.timer !== undefined) {
clearTimeout(operation.timer)
operation.timer = undefined
}
operation.commandSettledFlag = true
operation.resolveCommandSettled()
}
operation.commandId = commandIdOf(result)
if (result?.ok === true && isCommandSuccess(result)) {
this.settle(operation, 'succeeded', commandText(result) ?? undefined)
return
}
// A stop request (cancel or budget) must not let us claim the handler is
// gone: `withAbort` returns before the handler finishes. Inspect the Session
// lifecycle instead, and keep `cancelling` while that is unconfirmed.
if (operation.cancelled || operation.timedOut) {
await this.reconcile(operation)
return
}
this.settle(operation, 'failed', this.commandFailureMessage(operation, result, thrown))
}
/**
* Inspect the compaction lifecycle for a settled command call. A confirmed
* `compaction/end` with an error marks the run stopped; a confirmed end
* without error means it committed, but the final persistence result is no
* longer observable after the abort.
*/
private async reconcile(operation: CompactionOperation): Promise<void> {
if (operation.finishedAt !== undefined) return
if (!operation.commandSettledFlag) return
if (this.confirm === undefined) {
operation.state = 'unconfirmed'
operation.message = 'The Host cannot observe the compaction lifecycle for this run.'
operation.finishedAt = this.now()
return
}
let confirmation: RemoteCompactionConfirmation
try {
confirmation = await this.confirm(operation.sessionId, operation.acceptedAt, operation.commandId)
} catch {
confirmation = { kind: 'unknown' }
}
this.logger?.info('remote compaction confirmation', {
operationId: operation.operationId,
kind: confirmation.kind,
...(confirmation.kind === 'finished' ? { failed: confirmation.error !== undefined } : {}),
})
if (confirmation.kind === 'finished') {
operation.confirmation = 'finished'
operation.confirmError = confirmation.error
if (confirmation.error !== undefined) {
this.settle(
operation,
operation.cancelled ? 'cancelled' : 'failed',
operation.cancelled ? 'Compaction cancelled and the Host confirmed the run stopped.' : sanitizeMessage(confirmation.error),
)
return
}
// The run committed, but cancellation/timeout raced with its completion.
// Do not report a clean success when the persistence result is unknown.
this.settle(
operation,
'unconfirmed',
'Compaction committed despite the stop request; the final persistence result could not be confirmed.',
)
return
}
if (confirmation.kind === 'not-yet' || confirmation.kind === 'unknown') {
operation.confirmation = 'unknown'
operation.state = 'cancelling'
operation.message = operation.cancelled
? 'Cancellation was requested; the Host is still waiting for the compaction to finish.'
: 'The compaction budget expired; the Host is still waiting for the compaction to finish.'
return
}
}
private commandFailureMessage(
operation: CompactionOperation,
result: TypertRpcResult | undefined,
thrown: unknown,
): string {
if (result?.ok === true) {
const text = commandText(result)
return text === undefined ? 'The Host rejected the compaction command.' : sanitizeMessage(text)
}
if (result !== undefined && result.ok === false) return sanitizeMessage(result.error.message)
if (operation.timedOut) return `Compaction exceeded its ${operation.budgetMs}ms budget.`
if (thrown instanceof RpcError) return sanitizeMessage(thrown.message)
this.logger?.warn('remote compaction failed', {
operationId: operation.operationId,
reason: thrown instanceof Error ? thrown.name : 'unknown',
})
return 'The Host failed to run the compaction command.'
}
private settle(operation: CompactionOperation, state: SettledState | 'interrupted', message?: string): void {
if (operation.finishedAt !== undefined) return
if (operation.timer !== undefined) {
clearTimeout(operation.timer)
operation.timer = undefined
}
operation.state = state
operation.finishedAt = this.now()
if (message !== undefined) operation.message = message
this.logger?.info('remote compaction settled', {
operationId: operation.operationId,
state,
durationMs: operation.startedAt === undefined ? 0 : operation.finishedAt - operation.startedAt,
})
this.prune()
}
private find(params: RemoteCompactionStatusParams, owner: RemoteCompactionOwner): CompactionOperation | undefined {
const operation = this.operations.get(params.operationId)
if (operation === undefined) return undefined
if (operation.ownerDeviceId !== owner.deviceId) {
throw new RpcError('PERMISSION_DENIED', 'This compaction operation belongs to another device.')
}
if (operation.sessionId !== params.sessionId) {
throw new RpcError('INVALID_MESSAGE', 'The compaction operation does not belong to this Session.')
}
return operation
}
private view(operation: CompactionOperation): RemoteCompactionStatusResult {
return {
operationId: operation.operationId,
sessionId: operation.sessionId,
state: operation.state,
acceptedAt: operation.acceptedAt,
...(operation.startedAt === undefined ? {} : { startedAt: operation.startedAt }),
...(operation.finishedAt === undefined ? {} : { finishedAt: operation.finishedAt }),
...(operation.message === undefined ? {} : { message: operation.message }),
}
}
private interrupted(params: RemoteCompactionStatusParams): RemoteCompactionStatusResult {
return {
operationId: params.operationId,
sessionId: params.sessionId,
state: 'interrupted',
message: 'The Host no longer tracks this operation; it may have restarted.',
}
}
/** Drop settled operations past retention; unfinished ones are never pruned. */
private prune(): void {
const cutoff = this.now() - RETENTION_MS
for (const [operationId, operation] of this.operations) {
if (operation.finishedAt === undefined || operation.finishedAt > cutoff) continue
this.operations.delete(operationId)
this.requestIndex.delete(requestIndexKey(operation.ownerDeviceId, operation.sessionId, operation.requestId))
}
}
}
function clampBudget(value: number | undefined): number {
if (value === undefined) return DEFAULT_BUDGET_MS
if (!Number.isFinite(value) || !Number.isSafeInteger(value)) {
throw new RpcError('INVALID_MESSAGE', 'budgetMs must be a safe integer.')
}
return Math.min(Math.max(value, MIN_BUDGET_MS), MAX_BUDGET_MS)
}
function isCommandSuccess(result: TypertRpcResult): boolean {
if (result.ok !== true || !isRecord(result.value)) return false
const command = isRecord(result.value.result) ? result.value.result : undefined
return command?.kind === 'success'
}
function commandText(result: TypertRpcResult): string | undefined {
if (result.ok !== true || !isRecord(result.value)) return undefined
const command = isRecord(result.value.result) ? result.value.result : undefined
return typeof command?.text === 'string' && command.text.trim() !== '' ? command.text : undefined
}
function sanitizeMessage(message: string): string {
return message.length <= 2_000 ? message : `${message.slice(0, 2_000)}…`
}
function delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms))
}
/**
* Read the tail of a `session/events` result and decide whether the compaction
* behind `commandId` finished. Handles both the 0.2.2 `records[].event` shape
* and the legacy `events[]` shape used by older bridges.
*/
export function readCompactionConfirmation(
result: TypertRpcResult,
acceptedAt: number,
commandId: string | undefined,
): RemoteCompactionConfirmation {
if (result.ok !== true || !isRecord(result.value)) return { kind: 'unknown' }
const records = Array.isArray(result.value.records) ? result.value.records : []
const legacy = Array.isArray(result.value.events) ? result.value.events : []
const events: Array<Record<string, unknown>> = []
for (const record of [...records, ...legacy]) {
const event = isRecord(record) && isRecord(record.event) ? record.event : record
if (isRecord(event) && typeof event.type === 'string') events.push(event)
}
if (events.length === 0) return { kind: 'unknown' }
const resolvedCommandId = commandId ?? latestCompactCommandId(events, acceptedAt)
if (resolvedCommandId === undefined) return { kind: 'not-yet' }
const end = [...events].reverse().find(event => {
if (event.type !== 'compaction/end') return false
const data = isRecord(event.data) ? event.data : undefined
return data?.sourceCommandId === resolvedCommandId
})
if (end === undefined) return { kind: 'not-yet' }
const data = isRecord(end.data) ? end.data : {}
const error = data.error === undefined ? undefined : errorText(data.error)
return error === undefined ? { kind: 'finished' } : { kind: 'finished', error }
}
function latestCompactCommandId(events: Array<Record<string, unknown>>, acceptedAt: number): string | undefined {
for (const event of [...events].reverse()) {
if (event.type !== 'command/run') continue
const data = isRecord(event.data) ? event.data : undefined
if (data?.name !== 'compact' || typeof data.commandId !== 'string') continue
const time = typeof event.time === 'number' ? event.time : 0
if (time !== 0 && time < acceptedAt - 5_000) continue
return data.commandId
}
return undefined
}
function errorText(error: unknown): string {
if (typeof error === 'string') return sanitizeMessage(error)
if (isRecord(error)) {
const message = error.message ?? error.error ?? error.reason
if (typeof message === 'string' && message.trim() !== '') return sanitizeMessage(message)
try {
return sanitizeMessage(JSON.stringify(error))
} catch {
return 'The compaction failed.'
}
}
return 'The compaction failed.'
}