Import packages/plugin/src/remote-compaction-operations.ts
This commit is contained in:
parent
3719143bc4
commit
da92aee355
|
|
@ -0,0 +1,481 @@
|
|||
import { randomUUID } from 'node:crypto'
|
||||
import type {
|
||||
RemoteCompactionState,
|
||||
RemoteCompactionStatusParams,
|
||||
RemoteCompactionStatusResult,
|
||||
RemoteCompactionSubmitParams,
|
||||
RemoteCompactionSubmitResult,
|
||||
} from '@dsh-remote/protocol'
|
||||
import type { SafeLogger } from './logging.js'
|
||||
import { RpcError } from './safe-error.js'
|
||||
import type { TypertRpcResult } from './typert-gateway-contract.js'
|
||||
|
||||
/**
|
||||
* Host-side registry for asynchronous Remote compaction operations.
|
||||
*
|
||||
* Harness forwards `commands/execute` through `withAbort()`: once the caller's
|
||||
* signal aborts, the RPC stops waiting for the handler while the handler keeps
|
||||
* running. Therefore "the command call returned" does not prove the compaction
|
||||
* handler, its maintenance lock and its persistence work are finished. This
|
||||
* registry separates those facts:
|
||||
*
|
||||
* - `cancelling`: a stop was requested; the command call may not have settled.
|
||||
* - `unconfirmed`: the run committed, but cancellation/timing made the final
|
||||
* persistence result unobservable.
|
||||
* - `cancelled`/`failed`/`succeeded`: the Host observed a trustworthy final
|
||||
* result (command result, or `compaction/end` for the same command id).
|
||||
*
|
||||
* Terminal states come from the background Harness call. A `compaction/end`
|
||||
* event alone never fabricates success.
|
||||
*/
|
||||
|
||||
export interface RemoteCompactionOwner {
|
||||
deviceId: string
|
||||
connectionId: string
|
||||
}
|
||||
|
||||
/** Runs the Host's fixed `/compact` command for one operation. */
|
||||
export type RemoteCompactionRunner = (sessionId: string, signal: AbortSignal) => Promise<TypertRpcResult>
|
||||
|
||||
/** Outcome of inspecting the Session log for the compaction lifecycle. */
|
||||
export type RemoteCompactionConfirmation =
|
||||
| { kind: 'finished'; error?: string }
|
||||
| { kind: 'not-yet' }
|
||||
| { kind: 'unknown' }
|
||||
|
||||
/**
|
||||
* Confirms whether the compaction handler actually finished. Implementations
|
||||
* read the Session log (`command/run`, `compaction/start|end`) by command id.
|
||||
*/
|
||||
export type RemoteCompactionConfirmer = (
|
||||
sessionId: string,
|
||||
acceptedAt: number,
|
||||
commandId: string | undefined,
|
||||
) => Promise<RemoteCompactionConfirmation>
|
||||
|
||||
export interface RemoteCompactionOperationsOptions {
|
||||
confirm?: RemoteCompactionConfirmer
|
||||
logger?: SafeLogger
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
type SettledState = 'succeeded' | 'failed' | 'cancelled' | 'unconfirmed'
|
||||
|
||||
interface CompactionOperation {
|
||||
readonly operationId: string
|
||||
readonly requestId: string
|
||||
readonly sessionId: string
|
||||
readonly ownerDeviceId: string
|
||||
readonly budgetMs: number
|
||||
readonly acceptedAt: number
|
||||
readonly controller: AbortController
|
||||
readonly commandSettled: Promise<void>
|
||||
resolveCommandSettled: () => void
|
||||
state: RemoteCompactionState
|
||||
startedAt?: number
|
||||
finishedAt?: number
|
||||
message?: string
|
||||
cancelled: boolean
|
||||
timedOut: boolean
|
||||
commandSettledFlag: boolean
|
||||
commandId?: string
|
||||
confirmation: 'pending' | 'finished' | 'unknown'
|
||||
confirmError?: string
|
||||
timer?: ReturnType<typeof setTimeout>
|
||||
}
|
||||
|
||||
const MIN_BUDGET_MS = 60_000
|
||||
const MAX_BUDGET_MS = 7_200_000
|
||||
const DEFAULT_BUDGET_MS = 1_800_000
|
||||
/** Keep settled operations readable long enough for a reconnect to collect them. */
|
||||
const RETENTION_MS = 600_000
|
||||
/** A cancel request waits this long for the command call to settle. */
|
||||
const CANCEL_SETTLE_MS = 15_000
|
||||
/** Bound bound records so a client cannot create unbounded tasks. */
|
||||
const MAX_OPERATIONS = 32
|
||||
|
||||
function requestIndexKey(ownerDeviceId: string, sessionId: string, requestId: string): string {
|
||||
return `${ownerDeviceId}\u0000${sessionId}\u0000${requestId}`
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function commandIdOf(result: TypertRpcResult | undefined): string | undefined {
|
||||
if (result?.ok !== true || !isRecord(result.value)) return undefined
|
||||
return typeof result.value.commandId === 'string' ? result.value.commandId : undefined
|
||||
}
|
||||
|
||||
export class RemoteCompactionOperations {
|
||||
private readonly operations = new Map<string, CompactionOperation>()
|
||||
private readonly requestIndex = new Map<string, string>()
|
||||
private readonly confirm?: RemoteCompactionConfirmer
|
||||
private readonly logger?: SafeLogger
|
||||
private readonly now: () => number
|
||||
private closed = false
|
||||
|
||||
constructor(
|
||||
private readonly runner: RemoteCompactionRunner,
|
||||
options: RemoteCompactionOperationsOptions = {},
|
||||
) {
|
||||
this.confirm = options.confirm
|
||||
this.logger = options.logger
|
||||
this.now = options.now ?? Date.now
|
||||
}
|
||||
|
||||
submit(params: RemoteCompactionSubmitParams, owner: RemoteCompactionOwner): RemoteCompactionSubmitResult {
|
||||
if (this.closed) throw new RpcError('HARNESS_UNAVAILABLE', 'The Remote Host is shutting down.')
|
||||
this.prune()
|
||||
const key = requestIndexKey(owner.deviceId, params.sessionId, params.requestId)
|
||||
const existing = this.requestIndex.get(key)
|
||||
if (existing !== undefined) {
|
||||
const operation = this.operations.get(existing)
|
||||
if (operation !== undefined) return { operationId: operation.operationId, state: operation.state }
|
||||
this.requestIndex.delete(key)
|
||||
}
|
||||
// One unfinished compaction per Session: new request ids must not stack
|
||||
// work on top of a run whose handler may still be finishing.
|
||||
for (const operation of this.operations.values()) {
|
||||
if (operation.sessionId === params.sessionId && operation.finishedAt === undefined) {
|
||||
throw new RpcError('AGENT_BUSY', 'A compaction operation is already running for this Session.')
|
||||
}
|
||||
}
|
||||
if (this.operations.size >= MAX_OPERATIONS) {
|
||||
throw new RpcError('RATE_LIMITED', 'Too many Remote compaction operations are tracked by this Host.', undefined, true)
|
||||
}
|
||||
const budgetMs = clampBudget(params.budgetMs)
|
||||
const operation = this.createOperation(params, owner, budgetMs)
|
||||
this.operations.set(operation.operationId, operation)
|
||||
this.requestIndex.set(key, operation.operationId)
|
||||
this.logger?.info('remote compaction accepted', {
|
||||
operationId: operation.operationId,
|
||||
sessionId: operation.sessionId,
|
||||
budgetMs,
|
||||
})
|
||||
void this.run(operation)
|
||||
return { operationId: operation.operationId, state: 'accepted' }
|
||||
}
|
||||
|
||||
async status(params: RemoteCompactionStatusParams, owner: RemoteCompactionOwner): Promise<RemoteCompactionStatusResult> {
|
||||
this.prune()
|
||||
const operation = this.find(params, owner)
|
||||
if (operation === undefined) return this.interrupted(params)
|
||||
if (operation.finishedAt === undefined && operation.commandSettledFlag
|
||||
&& (operation.state === 'cancelling' || operation.state === 'unconfirmed')) {
|
||||
await this.reconcile(operation)
|
||||
}
|
||||
return this.view(operation)
|
||||
}
|
||||
|
||||
async cancel(params: RemoteCompactionStatusParams, owner: RemoteCompactionOwner): Promise<RemoteCompactionStatusResult> {
|
||||
this.prune()
|
||||
const operation = this.find(params, owner)
|
||||
if (operation === undefined) return this.interrupted(params)
|
||||
if (operation.finishedAt !== undefined) return this.view(operation)
|
||||
if (!operation.cancelled) {
|
||||
operation.cancelled = true
|
||||
operation.state = 'cancelling'
|
||||
operation.controller.abort(new Error('Remote compaction cancelled by the client.'))
|
||||
this.logger?.info('remote compaction cancellation requested', { operationId: operation.operationId })
|
||||
}
|
||||
// `withAbort` settles the command call early; reconciliation below waits for
|
||||
// the compaction lifecycle (or reports that it is still unconfirmed).
|
||||
await Promise.race([operation.commandSettled, delay(CANCEL_SETTLE_MS)])
|
||||
if (operation.commandSettledFlag) await this.reconcile(operation)
|
||||
return this.view(operation)
|
||||
}
|
||||
|
||||
/** Host shutdown: stop work and report every unfinished operation as interrupted. */
|
||||
closeAll(): void {
|
||||
this.closed = true
|
||||
for (const operation of this.operations.values()) {
|
||||
if (operation.finishedAt !== undefined) continue
|
||||
this.settle(operation, 'interrupted', 'The Remote Host shut down before the compaction finished.')
|
||||
operation.controller.abort(new Error('Remote Host shut down.'))
|
||||
}
|
||||
}
|
||||
|
||||
private createOperation(
|
||||
params: RemoteCompactionSubmitParams,
|
||||
owner: RemoteCompactionOwner,
|
||||
budgetMs: number,
|
||||
): CompactionOperation {
|
||||
let resolveCommandSettled = () => {}
|
||||
const commandSettled = new Promise<void>(resolve => { resolveCommandSettled = resolve })
|
||||
return {
|
||||
operationId: randomUUID(),
|
||||
requestId: params.requestId,
|
||||
sessionId: params.sessionId,
|
||||
ownerDeviceId: owner.deviceId,
|
||||
budgetMs,
|
||||
acceptedAt: this.now(),
|
||||
controller: new AbortController(),
|
||||
commandSettled,
|
||||
resolveCommandSettled,
|
||||
state: 'accepted',
|
||||
cancelled: false,
|
||||
timedOut: false,
|
||||
commandSettledFlag: false,
|
||||
confirmation: 'pending',
|
||||
}
|
||||
}
|
||||
|
||||
private async run(operation: CompactionOperation): Promise<void> {
|
||||
operation.state = 'running'
|
||||
operation.startedAt = this.now()
|
||||
operation.timer = setTimeout(() => {
|
||||
operation.timedOut = true
|
||||
operation.controller.abort(new Error('Remote compaction exceeded its budget.'))
|
||||
}, operation.budgetMs)
|
||||
let result: TypertRpcResult | undefined
|
||||
let thrown: unknown
|
||||
try {
|
||||
result = await this.runner(operation.sessionId, operation.controller.signal)
|
||||
} catch (error) {
|
||||
thrown = error
|
||||
} finally {
|
||||
if (operation.timer !== undefined) {
|
||||
clearTimeout(operation.timer)
|
||||
operation.timer = undefined
|
||||
}
|
||||
operation.commandSettledFlag = true
|
||||
operation.resolveCommandSettled()
|
||||
}
|
||||
operation.commandId = commandIdOf(result)
|
||||
|
||||
if (result?.ok === true && isCommandSuccess(result)) {
|
||||
this.settle(operation, 'succeeded', commandText(result) ?? undefined)
|
||||
return
|
||||
}
|
||||
// A stop request (cancel or budget) must not let us claim the handler is
|
||||
// gone: `withAbort` returns before the handler finishes. Inspect the Session
|
||||
// lifecycle instead, and keep `cancelling` while that is unconfirmed.
|
||||
if (operation.cancelled || operation.timedOut) {
|
||||
await this.reconcile(operation)
|
||||
return
|
||||
}
|
||||
this.settle(operation, 'failed', this.commandFailureMessage(operation, result, thrown))
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect the compaction lifecycle for a settled command call. A confirmed
|
||||
* `compaction/end` with an error marks the run stopped; a confirmed end
|
||||
* without error means it committed, but the final persistence result is no
|
||||
* longer observable after the abort.
|
||||
*/
|
||||
private async reconcile(operation: CompactionOperation): Promise<void> {
|
||||
if (operation.finishedAt !== undefined) return
|
||||
if (!operation.commandSettledFlag) return
|
||||
if (this.confirm === undefined) {
|
||||
operation.state = 'unconfirmed'
|
||||
operation.message = 'The Host cannot observe the compaction lifecycle for this run.'
|
||||
operation.finishedAt = this.now()
|
||||
return
|
||||
}
|
||||
let confirmation: RemoteCompactionConfirmation
|
||||
try {
|
||||
confirmation = await this.confirm(operation.sessionId, operation.acceptedAt, operation.commandId)
|
||||
} catch {
|
||||
confirmation = { kind: 'unknown' }
|
||||
}
|
||||
this.logger?.info('remote compaction confirmation', {
|
||||
operationId: operation.operationId,
|
||||
kind: confirmation.kind,
|
||||
...(confirmation.kind === 'finished' ? { failed: confirmation.error !== undefined } : {}),
|
||||
})
|
||||
if (confirmation.kind === 'finished') {
|
||||
operation.confirmation = 'finished'
|
||||
operation.confirmError = confirmation.error
|
||||
if (confirmation.error !== undefined) {
|
||||
this.settle(
|
||||
operation,
|
||||
operation.cancelled ? 'cancelled' : 'failed',
|
||||
operation.cancelled ? 'Compaction cancelled and the Host confirmed the run stopped.' : sanitizeMessage(confirmation.error),
|
||||
)
|
||||
return
|
||||
}
|
||||
// The run committed, but cancellation/timeout raced with its completion.
|
||||
// Do not report a clean success when the persistence result is unknown.
|
||||
this.settle(
|
||||
operation,
|
||||
'unconfirmed',
|
||||
'Compaction committed despite the stop request; the final persistence result could not be confirmed.',
|
||||
)
|
||||
return
|
||||
}
|
||||
if (confirmation.kind === 'not-yet' || confirmation.kind === 'unknown') {
|
||||
operation.confirmation = 'unknown'
|
||||
operation.state = 'cancelling'
|
||||
operation.message = operation.cancelled
|
||||
? 'Cancellation was requested; the Host is still waiting for the compaction to finish.'
|
||||
: 'The compaction budget expired; the Host is still waiting for the compaction to finish.'
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
private commandFailureMessage(
|
||||
operation: CompactionOperation,
|
||||
result: TypertRpcResult | undefined,
|
||||
thrown: unknown,
|
||||
): string {
|
||||
if (result?.ok === true) {
|
||||
const text = commandText(result)
|
||||
return text === undefined ? 'The Host rejected the compaction command.' : sanitizeMessage(text)
|
||||
}
|
||||
if (result !== undefined && result.ok === false) return sanitizeMessage(result.error.message)
|
||||
if (operation.timedOut) return `Compaction exceeded its ${operation.budgetMs}ms budget.`
|
||||
if (thrown instanceof RpcError) return sanitizeMessage(thrown.message)
|
||||
this.logger?.warn('remote compaction failed', {
|
||||
operationId: operation.operationId,
|
||||
reason: thrown instanceof Error ? thrown.name : 'unknown',
|
||||
})
|
||||
return 'The Host failed to run the compaction command.'
|
||||
}
|
||||
|
||||
private settle(operation: CompactionOperation, state: SettledState | 'interrupted', message?: string): void {
|
||||
if (operation.finishedAt !== undefined) return
|
||||
if (operation.timer !== undefined) {
|
||||
clearTimeout(operation.timer)
|
||||
operation.timer = undefined
|
||||
}
|
||||
operation.state = state
|
||||
operation.finishedAt = this.now()
|
||||
if (message !== undefined) operation.message = message
|
||||
this.logger?.info('remote compaction settled', {
|
||||
operationId: operation.operationId,
|
||||
state,
|
||||
durationMs: operation.startedAt === undefined ? 0 : operation.finishedAt - operation.startedAt,
|
||||
})
|
||||
this.prune()
|
||||
}
|
||||
|
||||
private find(params: RemoteCompactionStatusParams, owner: RemoteCompactionOwner): CompactionOperation | undefined {
|
||||
const operation = this.operations.get(params.operationId)
|
||||
if (operation === undefined) return undefined
|
||||
if (operation.ownerDeviceId !== owner.deviceId) {
|
||||
throw new RpcError('PERMISSION_DENIED', 'This compaction operation belongs to another device.')
|
||||
}
|
||||
if (operation.sessionId !== params.sessionId) {
|
||||
throw new RpcError('INVALID_MESSAGE', 'The compaction operation does not belong to this Session.')
|
||||
}
|
||||
return operation
|
||||
}
|
||||
|
||||
private view(operation: CompactionOperation): RemoteCompactionStatusResult {
|
||||
return {
|
||||
operationId: operation.operationId,
|
||||
sessionId: operation.sessionId,
|
||||
state: operation.state,
|
||||
acceptedAt: operation.acceptedAt,
|
||||
...(operation.startedAt === undefined ? {} : { startedAt: operation.startedAt }),
|
||||
...(operation.finishedAt === undefined ? {} : { finishedAt: operation.finishedAt }),
|
||||
...(operation.message === undefined ? {} : { message: operation.message }),
|
||||
}
|
||||
}
|
||||
|
||||
private interrupted(params: RemoteCompactionStatusParams): RemoteCompactionStatusResult {
|
||||
return {
|
||||
operationId: params.operationId,
|
||||
sessionId: params.sessionId,
|
||||
state: 'interrupted',
|
||||
message: 'The Host no longer tracks this operation; it may have restarted.',
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop settled operations past retention; unfinished ones are never pruned. */
|
||||
private prune(): void {
|
||||
const cutoff = this.now() - RETENTION_MS
|
||||
for (const [operationId, operation] of this.operations) {
|
||||
if (operation.finishedAt === undefined || operation.finishedAt > cutoff) continue
|
||||
this.operations.delete(operationId)
|
||||
this.requestIndex.delete(requestIndexKey(operation.ownerDeviceId, operation.sessionId, operation.requestId))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function clampBudget(value: number | undefined): number {
|
||||
if (value === undefined) return DEFAULT_BUDGET_MS
|
||||
if (!Number.isFinite(value) || !Number.isSafeInteger(value)) {
|
||||
throw new RpcError('INVALID_MESSAGE', 'budgetMs must be a safe integer.')
|
||||
}
|
||||
return Math.min(Math.max(value, MIN_BUDGET_MS), MAX_BUDGET_MS)
|
||||
}
|
||||
|
||||
function isCommandSuccess(result: TypertRpcResult): boolean {
|
||||
if (result.ok !== true || !isRecord(result.value)) return false
|
||||
const command = isRecord(result.value.result) ? result.value.result : undefined
|
||||
return command?.kind === 'success'
|
||||
}
|
||||
|
||||
function commandText(result: TypertRpcResult): string | undefined {
|
||||
if (result.ok !== true || !isRecord(result.value)) return undefined
|
||||
const command = isRecord(result.value.result) ? result.value.result : undefined
|
||||
return typeof command?.text === 'string' && command.text.trim() !== '' ? command.text : undefined
|
||||
}
|
||||
|
||||
function sanitizeMessage(message: string): string {
|
||||
return message.length <= 2_000 ? message : `${message.slice(0, 2_000)}…`
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the tail of a `session/events` result and decide whether the compaction
|
||||
* behind `commandId` finished. Handles both the 0.2.2 `records[].event` shape
|
||||
* and the legacy `events[]` shape used by older bridges.
|
||||
*/
|
||||
export function readCompactionConfirmation(
|
||||
result: TypertRpcResult,
|
||||
acceptedAt: number,
|
||||
commandId: string | undefined,
|
||||
): RemoteCompactionConfirmation {
|
||||
if (result.ok !== true || !isRecord(result.value)) return { kind: 'unknown' }
|
||||
const records = Array.isArray(result.value.records) ? result.value.records : []
|
||||
const legacy = Array.isArray(result.value.events) ? result.value.events : []
|
||||
const events: Array<Record<string, unknown>> = []
|
||||
for (const record of [...records, ...legacy]) {
|
||||
const event = isRecord(record) && isRecord(record.event) ? record.event : record
|
||||
if (isRecord(event) && typeof event.type === 'string') events.push(event)
|
||||
}
|
||||
if (events.length === 0) return { kind: 'unknown' }
|
||||
const resolvedCommandId = commandId ?? latestCompactCommandId(events, acceptedAt)
|
||||
if (resolvedCommandId === undefined) return { kind: 'not-yet' }
|
||||
const end = [...events].reverse().find(event => {
|
||||
if (event.type !== 'compaction/end') return false
|
||||
const data = isRecord(event.data) ? event.data : undefined
|
||||
return data?.sourceCommandId === resolvedCommandId
|
||||
})
|
||||
if (end === undefined) return { kind: 'not-yet' }
|
||||
const data = isRecord(end.data) ? end.data : {}
|
||||
const error = data.error === undefined ? undefined : errorText(data.error)
|
||||
return error === undefined ? { kind: 'finished' } : { kind: 'finished', error }
|
||||
}
|
||||
|
||||
function latestCompactCommandId(events: Array<Record<string, unknown>>, acceptedAt: number): string | undefined {
|
||||
for (const event of [...events].reverse()) {
|
||||
if (event.type !== 'command/run') continue
|
||||
const data = isRecord(event.data) ? event.data : undefined
|
||||
if (data?.name !== 'compact' || typeof data.commandId !== 'string') continue
|
||||
const time = typeof event.time === 'number' ? event.time : 0
|
||||
if (time !== 0 && time < acceptedAt - 5_000) continue
|
||||
return data.commandId
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function errorText(error: unknown): string {
|
||||
if (typeof error === 'string') return sanitizeMessage(error)
|
||||
if (isRecord(error)) {
|
||||
const message = error.message ?? error.error ?? error.reason
|
||||
if (typeof message === 'string' && message.trim() !== '') return sanitizeMessage(message)
|
||||
try {
|
||||
return sanitizeMessage(JSON.stringify(error))
|
||||
} catch {
|
||||
return 'The compaction failed.'
|
||||
}
|
||||
}
|
||||
return 'The compaction failed.'
|
||||
}
|
||||
Loading…
Reference in New Issue