diff --git a/src/skill-sync.ts b/src/skill-sync.ts new file mode 100644 index 0000000..73fff03 --- /dev/null +++ b/src/skill-sync.ts @@ -0,0 +1,545 @@ +/** + * Bidirectional skill sync between the cloud skill center and the local + * DSH skills directory (`$DSH_HOME/skills` by default). + * + * The cloud center is the hub, the local directory is a working copy, and + * the DSH skills manager keeps owning local management — this module only + * moves files, in both directions: + * + * - DOWN: cloud skills are installed exactly like the platform's own + * `skills-sync` tool does it — bundle hash and per-file checksums are + * verified before anything lands, writes are atomic per file, and a + * per-skill `.manifest.json` records what the cloud delivered. + * - UP: skills that originated locally (installed by the skills manager, + * imported, or written by hand) are published to the center as new + * revisions, guarded by the center's `expected_revision` check. + * + * Safety rules (deliberate, in order): + * 1. A cloud-delivered skill is never re-uploaded. If its local files no + * longer match the delivery manifest, the skill is flagged as a + * conflict and left completely alone. + * 2. A local deletion never deletes the cloud copy. + * 3. A name that already exists on the other side is never hijacked; + * it is reported as a conflict for a human to resolve. + * 4. Bundles whose scripts changed on the cloud side are not installed + * until the change is acknowledged in the dashboard, mirroring the + * center's own review flag. + * 5. Only the center's file layout is uploadable (SKILL.md at the root, + * everything else under references/, scripts/ or assets/, with the + * center's extension allowlist). Anything else is reported, never + * silently trimmed. + */ +import { createHash, randomUUID } from 'node:crypto' +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises' +import { dirname, join, resolve } from 'node:path' +import type { CloudClient, SkillBundle, SkillSummary, SkillUploadFile } from './api.ts' +import { describeError } from './api.ts' +import { dshHome } from './store.ts' + +/** Cloud skill names: lowercase kebab, at most 64 characters. */ +export const CLOUD_SKILL_NAME = /^[a-z0-9][a-z0-9-]{0,63}$/ + +/** Layout contract of the cloud skill center (mirrors its store rules). */ +const TEXT_EXT = new Set(['.md', '.txt', '.json', '.yaml', '.yml', '.py', '.sh', '.ps1', '.mjs', '.js', '.ts', '.tsx', '.css', '.html', '.csv']) +const BINARY_EXT = new Set(['.docx', '.xlsx', '.pptx', '.pdf', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.zip']) +const SCRIPT_EXT = new Set(['.py', '.sh', '.ps1', '.mjs']) +const SKILL_DIRS = new Set(['references', 'scripts', 'assets']) +const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/ +const MAX_FILES = 100 +const MAX_FILE_BYTES = 512 * 1024 +const MAX_SKILL_BYTES = 2 * 1024 * 1024 +const MAX_DEPTH = 3 + +/** SHA-256 of a buffer or UTF-8 string, hex-encoded. */ +export function sha256Hex(data: Buffer | string): string { + return createHash('sha256').update(data).digest('hex') +} + +/** + * The center's linked bundle manifest: one `path\0sha256` line per file, + * sorted, under a version header. Byte-identical to the platform's own + * computation, so locally computed manifests compare directly. + */ +export function linkedSkillManifest(files: readonly { path: string; sha256: string }[]): string { + const lines = files.map((file) => `${file.path}\0${file.sha256}`).sort() + return sha256Hex(Buffer.from(`cloud-memory-skill-v1\n${lines.join('\n')}\n`, 'utf8')) +} + +/** True when the cloud center would accept this relative file path. */ +export function isCloudSkillPath(rel: string): boolean { + if (rel === '' || rel.length > 200 || rel.startsWith('/') || rel.includes('\\') || rel.includes('//')) return false + const segments = rel.split('/') + if (segments.length > MAX_DEPTH) return false + for (const segment of segments) { + if (!SEGMENT.test(segment) || segment === '.' || segment === '..') return false + } + const dot = rel.lastIndexOf('.') + const ext = dot === -1 ? '' : rel.slice(dot).toLowerCase() + if (!TEXT_EXT.has(ext) && !BINARY_EXT.has(ext)) return false + const first = segments[0] + if (segments.length === 1) return rel === 'SKILL.md' + if (first === undefined || !SKILL_DIRS.has(first)) return false + if (first === 'scripts' && !SCRIPT_EXT.has(ext)) return false + return true +} + +/** Minimal frontmatter reader for `name` / `description` (plain scalars). */ +export function parseSkillFrontmatter(markdown: string): { name?: string; description?: string } { + if (!markdown.startsWith('---')) return {} + const end = markdown.indexOf('\n---', 3) + if (end === -1) return {} + const out: { name?: string; description?: string } = {} + for (const line of markdown.slice(3, end).split('\n')) { + const match = /^\s*(name|description)\s*:\s*(.+?)\s*$/.exec(line) + if (match === null) continue + const value = (match[2] ?? '').replace(/^["']|["']$/g, '') + if (match[1] === 'name' && value !== '') out.name = value + if (match[1] === 'description' && value !== '') out.description = value + } + return out +} + +/** One sync conflict a human should look at; sync never forces these. */ +export interface SkillSyncConflict { + name: string + reason: string +} + +/** Outcome of the most recent (or running) sync pass. */ +export interface SkillSyncStats { + running: boolean + lastRunAt?: string + lastError?: string + downloaded: string[] + uploaded: string[] + updated: string[] + conflicts: SkillSyncConflict[] + skipped: number +} + +/** What the sidecar remembers about one locally-originated upload. */ +interface UploadRecord { + folder: string + revision: number + manifest: string + syncedAt: string +} + +/** Sidecar state: `$DSH_HOME/cloud-memory-skills.json`. */ +interface SyncState { + version: 1 + uploaded: Record + /** Cloud skills currently delivered locally, by delivered manifest. */ + delivered: Record + /** + * Skills the user deleted locally, by the cloud manifest at deletion + * time. A deleted skill stays deleted until the cloud publishes a + * different manifest; the cloud copy itself is never removed. + */ + removed: Record +} + +/** One local file read from disk. */ +interface LocalFile { + rel: string + buffer: Buffer +} + +/** Dependencies of one sync engine instance. */ +export interface SkillSyncDeps { + client: () => CloudClient + settings: () => { enabled: boolean; skillsSyncEnabled: boolean; skillsDir: string } + log: (message: string) => void +} + +/** The per-skill delivery record the platform's own tooling writes. */ +interface DeliveryManifest { + name?: string + revision?: number + manifest_sha256?: string + files?: { path: string; sha256: string }[] +} + +export class SkillSync { + private running = false + private current: SkillSyncStats = { + running: false, downloaded: [], uploaded: [], updated: [], conflicts: [], skipped: 0, + } + + constructor(private readonly deps: SkillSyncDeps) {} + + /** Stats of the latest pass, for the status endpoint. */ + stats(): SkillSyncStats { + return this.current + } + + /** The local skills root this engine syncs. */ + skillsRoot(): string { + const dir = this.deps.settings().skillsDir + return dir !== '' ? resolve(dir) : join(dshHome(), 'skills') + } + + private statePath(): string { + return join(dshHome(), 'cloud-memory-skills.json') + } + + private async loadState(): Promise { + const clean = { version: 1 as const, uploaded: {}, delivered: {}, removed: {} } + try { + const parsed: unknown = JSON.parse(await readFile(this.statePath(), 'utf8')) + if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) { + const raw = parsed as { uploaded?: unknown; delivered?: unknown; removed?: unknown } + const map = (value: unknown): Record => + value !== null && typeof value === 'object' && !Array.isArray(value) ? (value as Record) : {} + return { + version: 1, + uploaded: map(raw.uploaded) as Record, + delivered: map(raw.delivered) as Record, + removed: map(raw.removed) as Record, + } + } + } catch { + // Missing or corrupt sidecar: start clean; the cloud is the truth. + } + return clean + } + + private async saveState(state: SyncState): Promise { + const path = this.statePath() + await mkdir(dirname(path), { recursive: true }) + const temporary = `${path}.tmp-${process.pid}` + await writeFile(temporary, JSON.stringify(state, null, 2)) + await rename(temporary, path) + } + + /** Read every syncable file of one skill directory (unsorted walk). */ + private async readSkillFiles(dir: string): Promise { + const out: LocalFile[] = [] + const walk = async (sub: string): Promise => { + const entries = await readdir(join(dir, sub), { withFileTypes: true }) + for (const entry of entries) { + const rel = sub === '' ? entry.name : `${sub}/${entry.name}` + if (entry.name.startsWith('.')) continue + if (entry.isDirectory()) { + await walk(rel) + } else if (entry.isFile()) { + out.push({ rel, buffer: await readFile(join(dir, rel)) }) + } + } + } + await walk('') + return out.sort((a, b) => (a.rel < b.rel ? -1 : 1)) + } + + /** True when a cloud-delivered directory still matches its manifest. */ + private async deliveredIntact(dir: string, delivery: DeliveryManifest): Promise { + const recorded = delivery.files ?? [] + let files: LocalFile[] + try { + files = await this.readSkillFiles(dir) + } catch { + return false + } + const current = files.filter((file) => file.rel !== '.manifest.json') + if (current.length !== recorded.length) return false + const byPath = new Map(current.map((file) => [file.rel, sha256Hex(file.buffer)])) + return recorded.every((file) => byPath.get(file.path) === file.sha256) + } + + /** Install or update one cloud skill into the local directory. */ + private async install(item: SkillSummary, bundle: SkillBundle, root: string): Promise { + const dir = join(root, item.name) + const previous = await this.readDelivery(dir) + const keep = new Set(bundle.files.map((file) => file.path)) + await mkdir(dir, { recursive: true }) + const written: { path: string; sha256: string; size: number }[] = [] + for (const file of bundle.files) { + const buffer = Buffer.from(file.data, 'base64') + if (sha256Hex(buffer) !== file.sha256) throw new Error(`checksum mismatch for ${file.path}`) + const target = join(dir, ...file.path.split('/')) + await mkdir(dirname(target), { recursive: true }) + const temporary = `${target}.tmp-${process.pid}` + await writeFile(temporary, buffer) + await rename(temporary, target) + written.push({ path: file.path, sha256: file.sha256, size: buffer.length }) + } + for (const file of previous?.files ?? []) { + if (!keep.has(file.path)) await rm(join(dir, ...file.path.split('/')), { force: true }) + } + const delivery: DeliveryManifest & { synced_at: string } = { + name: item.name, + revision: bundle.revision, + manifest_sha256: bundle.manifest_sha256, + files: written, + synced_at: new Date().toISOString(), + } + await writeFile(join(dir, '.manifest.json'), JSON.stringify(delivery, null, 2)) + } + + private async readDelivery(dir: string): Promise { + try { + const parsed: unknown = JSON.parse(await readFile(join(dir, '.manifest.json'), 'utf8')) + if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) { + return parsed as DeliveryManifest + } + } catch { + // Not a cloud-delivered skill. + } + return undefined + } + + /** One full pass; never throws — outcomes land in the stats. */ + async syncOnce(): Promise { + if (this.running) return this.current + const settings = this.deps.settings() + if (!settings.enabled || !settings.skillsSyncEnabled) return this.current + this.running = true + const stats: SkillSyncStats = { + running: true, downloaded: [], uploaded: [], updated: [], conflicts: [], skipped: 0, + } + this.current = stats + try { + await this.pass(stats) + stats.lastRunAt = new Date().toISOString() + } catch (error) { + stats.lastError = error instanceof Error ? error.message : String(error) + this.deps.log(`skill sync failed: ${stats.lastError}`) + } finally { + stats.running = false + this.running = false + this.current = stats + } + return this.current + } + + /** Record a conflict once per pass, however many rules trip over it. */ + private conflict(stats: SkillSyncStats, name: string, reason: string): void { + if (!stats.conflicts.some((item) => item.name === name && item.reason === reason)) { + stats.conflicts.push({ name, reason }) + } + } + + private async pass(stats: SkillSyncStats): Promise { + const client = this.deps.client() + const listing = await client.listSkills() + if (!listing.ok) throw new Error(`skill listing failed: ${describeError(listing.error)}`) + const items = (listing.data.items ?? []).filter( + (item): item is SkillSummary & { name: string } => typeof item.name === 'string' && CLOUD_SKILL_NAME.test(item.name), + ) + const cloudByName = new Map(items.map((item) => [item.name, item])) + const root = this.skillsRoot() + const state = await this.loadState() + let stateDirty = false + + // ---- DOWN: cloud center -> local directory ------------------------- + for (const item of items) { + if (item.enabled === false) { stats.skipped += 1; continue } + // Locally-originated skills belong to the UP pass, even though the + // center lists them too once they are published. + if (state.uploaded[item.name] !== undefined) { stats.skipped += 1; continue } + const dir = join(root, item.name) + const exists = await stat(dir).then((s) => s.isDirectory()).catch(() => false) + const removed = state.removed[item.name] + if (removed !== undefined && exists) { + // The user put it back by hand; the removal is over. + delete state.removed[item.name] + stateDirty = true + } else if (removed !== undefined && item.manifest_sha256 === removed.manifest) { + // Deleted locally at this exact cloud version: keep it deleted. + stats.skipped += 1 + continue + } else if (removed !== undefined) { + // The cloud published a different version since the deletion. + delete state.removed[item.name] + stateDirty = true + } + const delivery = await this.readDelivery(dir) + if (exists && delivery !== undefined) { + // Delivered before: local integrity is judged before any download. + if (!(await this.deliveredIntact(dir, delivery))) { + this.conflict(stats, item.name, 'locally_modified') + continue + } + if (delivery.manifest_sha256 === item.manifest_sha256) { + if (state.delivered[item.name]?.manifest !== item.manifest_sha256) { + state.delivered[item.name] = { manifest: item.manifest_sha256 ?? '' } + stateDirty = true + } + stats.skipped += 1 + continue + } + // Intact but the cloud moved on: fall through and install the update. + } + if (exists && delivery === undefined) { + this.conflict(stats, item.name, 'name_collision') + continue + } + if (item.script_changed === true) { + // The listing already carries the review flag; don't even fetch. + this.conflict(stats, item.name, 'script_review_required') + continue + } + const bundle = await client.skillBundle(item.name) + if (!bundle.ok) { + this.conflict(stats, item.name, `bundle_failed: ${describeError(bundle.error)}`) + continue + } + if (linkedSkillManifest(bundle.data.files) !== bundle.data.manifest_sha256) { + this.conflict(stats, item.name, 'manifest_mismatch') + continue + } + if (bundle.data.script_changed === true) { + this.conflict(stats, item.name, 'script_review_required') + continue + } + await this.install(item, bundle.data, root) + state.delivered[item.name] = { manifest: bundle.data.manifest_sha256 } + stateDirty = true + ;(exists ? stats.updated : stats.downloaded).push(item.name) + } + + // ---- UP: local originals -> cloud center --------------------------- + const folders = await readdir(root, { withFileTypes: true }).catch(() => []) + const seenFolders = new Set() + for (const entry of folders) { + if (!entry.isDirectory() || entry.name.startsWith('.')) continue + seenFolders.add(entry.name) + const dir = join(root, entry.name) + if ((await this.readDelivery(dir)) !== undefined) continue // cloud-origin: down pass owns it + const skill = await this.readLocalSkill(dir, entry.name) + if (skill === undefined) continue + if (typeof skill === 'string') { + this.conflict(stats, entry.name, skill) + continue + } + const record = state.uploaded[skill.name] + const cloud = cloudByName.get(skill.name) + if (cloud !== undefined && record === undefined) { + this.conflict(stats, skill.name, 'name_collision') + continue + } + if (record !== undefined && record.manifest === skill.manifest) { stats.skipped += 1; continue } + const files: SkillUploadFile[] = skill.files.map((file) => { + const ext = file.rel.slice(file.rel.lastIndexOf('.')).toLowerCase() + return TEXT_EXT.has(ext) + ? { path: file.rel, content: file.buffer.toString('utf8') } + : { path: file.rel, base64: file.buffer.toString('base64') } + }) + const operationId = `dsh-skill-${randomUUID()}` + if (record === undefined || cloud === undefined) { + const created = await client.createSkill({ + name: skill.name, + description: skill.description, + execution_mode: skill.hasScripts ? 'local-bundle' : 'prompt-only', + files, + expected_revision: 0, + operation_id: operationId, + }) + if (!created.ok) { + this.conflict(stats, skill.name, `upload_failed: ${describeError(created.error)}`) + continue + } + state.uploaded[skill.name] = { + folder: entry.name, + revision: created.data.skill?.revision ?? created.data.revision ?? 1, + manifest: skill.manifest, + syncedAt: new Date().toISOString(), + } + stateDirty = true + stats.uploaded.push(skill.name) + } else { + if (cloud.revision !== undefined && cloud.revision !== record.revision) { + this.conflict(stats, skill.name, 'cloud_newer') + continue + } + const updated = await client.updateSkill(skill.name, { + description: skill.description, + files, + expected_revision: record.revision, + operation_id: operationId, + }) + if (!updated.ok) { + const reason = updated.error.code === 'revision_conflict' ? 'cloud_newer' : `upload_failed: ${describeError(updated.error)}` + this.conflict(stats, skill.name, reason) + continue + } + state.uploaded[skill.name] = { + folder: entry.name, + revision: updated.data.skill?.revision ?? updated.data.revision ?? record.revision + 1, + manifest: skill.manifest, + syncedAt: new Date().toISOString(), + } + stateDirty = true + stats.updated.push(skill.name) + } + } + + // A vanished local folder is remembered as a deliberate removal (so + // the next pass does not resurrect it); the cloud copy always stays. + for (const [name, record] of Object.entries(state.uploaded)) { + if (!seenFolders.has(record.folder)) { + state.removed[name] = { manifest: cloudByName.get(name)?.manifest_sha256 ?? record.manifest } + delete state.uploaded[name] + stateDirty = true + } + } + for (const [name, record] of Object.entries(state.delivered)) { + if (!seenFolders.has(name)) { + state.removed[name] = { manifest: record.manifest } + delete state.delivered[name] + stateDirty = true + } + } + if (stateDirty) await this.saveState(state) + const changed = stats.downloaded.length + stats.uploaded.length + stats.updated.length + if (changed > 0 || stats.conflicts.length > 0) { + this.deps.log( + `skill sync: ${stats.downloaded.length} downloaded, ${stats.uploaded.length} uploaded, ` + + `${stats.updated.length} updated, ${stats.conflicts.length} conflict(s)`, + ) + } + } + + /** + * Read one locally-originated skill for upload. Returns the skill, a + * conflict reason string, or undefined when the folder is not a skill. + */ + private async readLocalSkill( + dir: string, + folder: string, + ): Promise<{ name: string; description: string; files: LocalFile[]; manifest: string; hasScripts: boolean } | string | undefined> { + let files: LocalFile[] + try { + files = await this.readSkillFiles(dir) + } catch { + return undefined + } + const markdown = files.find((file) => file.rel === 'SKILL.md') + if (markdown === undefined) return undefined + if (files.length > MAX_FILES) return 'too_many_files' + let total = 0 + for (const file of files) { + if (file.buffer.length > MAX_FILE_BYTES) return 'file_too_large' + total += file.buffer.length + if (!isCloudSkillPath(file.rel)) return 'unsupported_layout' + } + if (total > MAX_SKILL_BYTES) return 'skill_too_large' + const frontmatter = parseSkillFrontmatter(markdown.buffer.toString('utf8')) + const name = frontmatter.name !== undefined && CLOUD_SKILL_NAME.test(frontmatter.name) + ? frontmatter.name + : CLOUD_SKILL_NAME.test(folder) + ? folder + : undefined + if (name === undefined) return 'invalid_name' + const heading = /^#\s+(.+)$/m.exec(markdown.buffer.toString('utf8'))?.[1]?.trim() + const description = (frontmatter.description ?? heading ?? name).slice(0, 500) + const manifest = linkedSkillManifest(files.map((file) => ({ path: file.rel, sha256: sha256Hex(file.buffer) }))) + return { + name, + description, + files, + manifest, + hasScripts: files.some((file) => file.rel.startsWith('scripts/')), + } + } +}