/** * Bidirectional skill sync between the cloud skill center and the local * DSH skills directory (`$DSH_HOME/skills` by default). * * The cloud center is the hub, the local directory is a working copy, and * the DSH skills manager keeps owning local management — this module only * moves files, in both directions: * * - DOWN: cloud skills are installed exactly like the platform's own * `skills-sync` tool does it — bundle hash and per-file checksums are * verified before anything lands, writes are atomic per file, and a * per-skill `.manifest.json` records what the cloud delivered. * - UP: skills that originated locally (installed by the skills manager, * imported, or written by hand) are published to the center as new * revisions, guarded by the center's `expected_revision` check. * * Safety rules (deliberate, in order): * 1. A cloud-delivered skill is never re-uploaded. If its local files no * longer match the delivery manifest, the skill is flagged as a * conflict and left completely alone. * 2. A local deletion never deletes the cloud copy. * 3. A name that already exists on the other side is never hijacked; * it is reported as a conflict for a human to resolve. * 4. Bundles whose scripts changed on the cloud side are not installed * until the change is acknowledged in the dashboard, mirroring the * center's own review flag. * 5. Only the center's file layout is uploadable (SKILL.md at the root, * everything else under references/, scripts/ or assets/, with the * center's extension allowlist). Anything else is reported, never * silently trimmed. */ import { createHash, randomUUID } from 'node:crypto'; import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'; import { dirname, join, resolve } from 'node:path'; import { describeError } from "./api.js"; import { dshHome } from "./store.js"; /** Cloud skill names: lowercase kebab, at most 64 characters. */ export const CLOUD_SKILL_NAME = /^[a-z0-9][a-z0-9-]{0,63}$/; /** Layout contract of the cloud skill center (mirrors its store rules). */ const TEXT_EXT = new Set(['.md', '.txt', '.json', '.yaml', '.yml', '.py', '.sh', '.ps1', '.mjs', '.js', '.ts', '.tsx', '.css', '.html', '.csv']); const BINARY_EXT = new Set(['.docx', '.xlsx', '.pptx', '.pdf', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.zip']); const SCRIPT_EXT = new Set(['.py', '.sh', '.ps1', '.mjs']); const SKILL_DIRS = new Set(['references', 'scripts', 'assets']); const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; const MAX_FILES = 100; const MAX_FILE_BYTES = 512 * 1024; const MAX_SKILL_BYTES = 2 * 1024 * 1024; const MAX_DEPTH = 3; /** SHA-256 of a buffer or UTF-8 string, hex-encoded. */ export function sha256Hex(data) { return createHash('sha256').update(data).digest('hex'); } /** * The center's linked bundle manifest: one `path\0sha256` line per file, * sorted, under a version header. Byte-identical to the platform's own * computation, so locally computed manifests compare directly. */ export function linkedSkillManifest(files) { const lines = files.map((file) => `${file.path}\0${file.sha256}`).sort(); return sha256Hex(Buffer.from(`cloud-memory-skill-v1\n${lines.join('\n')}\n`, 'utf8')); } /** True when the cloud center would accept this relative file path. */ export function isCloudSkillPath(rel) { if (rel === '' || rel.length > 200 || rel.startsWith('/') || rel.includes('\\') || rel.includes('//')) return false; const segments = rel.split('/'); if (segments.length > MAX_DEPTH) return false; for (const segment of segments) { if (!SEGMENT.test(segment) || segment === '.' || segment === '..') return false; } const dot = rel.lastIndexOf('.'); const ext = dot === -1 ? '' : rel.slice(dot).toLowerCase(); if (!TEXT_EXT.has(ext) && !BINARY_EXT.has(ext)) return false; const first = segments[0]; if (segments.length === 1) return rel === 'SKILL.md'; if (first === undefined || !SKILL_DIRS.has(first)) return false; if (first === 'scripts' && !SCRIPT_EXT.has(ext)) return false; return true; } /** Minimal frontmatter reader for `name` / `description` (plain scalars). */ export function parseSkillFrontmatter(markdown) { if (!markdown.startsWith('---')) return {}; const end = markdown.indexOf('\n---', 3); if (end === -1) return {}; const out = {}; for (const line of markdown.slice(3, end).split('\n')) { const match = /^\s*(name|description)\s*:\s*(.+?)\s*$/.exec(line); if (match === null) continue; const value = (match[2] ?? '').replace(/^["']|["']$/g, ''); if (match[1] === 'name' && value !== '') out.name = value; if (match[1] === 'description' && value !== '') out.description = value; } return out; } async function isDirectory(path) { return stat(path).then((value) => value.isDirectory()).catch(() => false); } export class SkillSync { deps; running = false; current = { running: false, downloaded: [], uploaded: [], updated: [], conflicts: [], skipped: 0, }; constructor(deps) { this.deps = deps; } /** Stats of the latest pass, for the status endpoint. */ stats() { return this.current; } /** The local skills root this engine syncs. */ skillsRoot() { const dir = this.deps.settings().skillsDir; return dir !== '' ? resolve(dir) : join(dshHome(), 'skills'); } /** The parking area where disabled skills wait, outside the root. */ holdingRoot() { return this.deps.holdingDir?.() ?? join(dshHome(), 'skills-disabled'); } statePath() { return join(dshHome(), 'cloud-memory-skills.json'); } async loadState() { const clean = { version: 1, uploaded: {}, delivered: {}, removed: {} }; try { const parsed = JSON.parse(await readFile(this.statePath(), 'utf8')); if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) { const raw = parsed; const map = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {}; return { version: 1, uploaded: map(raw.uploaded), delivered: map(raw.delivered), removed: map(raw.removed), }; } } catch { // Missing or corrupt sidecar: start clean; the cloud is the truth. } return clean; } async saveState(state) { const path = this.statePath(); await mkdir(dirname(path), { recursive: true }); const temporary = `${path}.tmp-${process.pid}`; await writeFile(temporary, JSON.stringify(state, null, 2)); await rename(temporary, path); } /** Read every syncable file of one skill directory (unsorted walk). */ async readSkillFiles(dir) { const out = []; const walk = async (sub) => { const entries = await readdir(join(dir, sub), { withFileTypes: true }); for (const entry of entries) { const rel = sub === '' ? entry.name : `${sub}/${entry.name}`; if (entry.name.startsWith('.')) continue; if (entry.isDirectory()) { await walk(rel); } else if (entry.isFile()) { out.push({ rel, buffer: await readFile(join(dir, rel)) }); } } }; await walk(''); return out.sort((a, b) => (a.rel < b.rel ? -1 : 1)); } /** True when a cloud-delivered directory still matches its manifest. */ async deliveredIntact(dir, delivery) { const recorded = delivery.files ?? []; let files; try { files = await this.readSkillFiles(dir); } catch { return false; } const current = files.filter((file) => file.rel !== '.manifest.json'); if (current.length !== recorded.length) return false; const byPath = new Map(current.map((file) => [file.rel, sha256Hex(file.buffer)])); return recorded.every((file) => byPath.get(file.path) === file.sha256); } /** Install or update one cloud skill into the local directory. */ async install(item, bundle, root) { const dir = join(root, item.name); const previous = await this.readDelivery(dir); const keep = new Set(bundle.files.map((file) => file.path)); await mkdir(dir, { recursive: true }); const written = []; for (const file of bundle.files) { const buffer = Buffer.from(file.data, 'base64'); if (sha256Hex(buffer) !== file.sha256) throw new Error(`checksum mismatch for ${file.path}`); const target = join(dir, ...file.path.split('/')); await mkdir(dirname(target), { recursive: true }); const temporary = `${target}.tmp-${process.pid}`; await writeFile(temporary, buffer); await rename(temporary, target); written.push({ path: file.path, sha256: file.sha256, size: buffer.length }); } for (const file of previous?.files ?? []) { if (!keep.has(file.path)) await rm(join(dir, ...file.path.split('/')), { force: true }); } const delivery = { name: item.name, revision: bundle.revision, manifest_sha256: bundle.manifest_sha256, files: written, synced_at: new Date().toISOString(), }; await writeFile(join(dir, '.manifest.json'), JSON.stringify(delivery, null, 2)); } async readDelivery(dir) { try { const parsed = JSON.parse(await readFile(join(dir, '.manifest.json'), 'utf8')); if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) { return parsed; } } catch { // Not a cloud-delivered skill. } return undefined; } /** One full pass; never throws — outcomes land in the stats. */ async syncOnce() { if (this.running) return this.current; const settings = this.deps.settings(); if (!settings.enabled || !settings.skillsSyncEnabled) return this.current; this.running = true; const stats = { running: true, downloaded: [], uploaded: [], updated: [], conflicts: [], skipped: 0, }; this.current = stats; try { await this.pass(stats); stats.lastRunAt = new Date().toISOString(); } catch (error) { stats.lastError = error instanceof Error ? error.message : String(error); this.deps.log(`skill sync failed: ${stats.lastError}`); } finally { stats.running = false; this.running = false; this.current = stats; } return this.current; } /** Record a conflict once per pass, however many rules trip over it. */ conflict(stats, name, reason) { if (!stats.conflicts.some((item) => item.name === name && item.reason === reason)) { stats.conflicts.push({ name, reason }); } } async pass(stats) { const client = this.deps.client(); const listing = await client.listSkills(); if (!listing.ok) throw new Error(`skill listing failed: ${describeError(listing.error)}`); const items = (listing.data.items ?? []).filter((item) => typeof item.name === 'string' && CLOUD_SKILL_NAME.test(item.name)); const cloudByName = new Map(items.map((item) => [item.name, item])); const root = this.skillsRoot(); const state = await this.loadState(); let stateDirty = false; // ---- DOWN: cloud center -> local directory ------------------------- for (const item of items) { if (item.enabled === false) { stats.skipped += 1; continue; } // Locally-originated skills belong to the UP pass, even though the // center lists them too once they are published. if (state.uploaded[item.name] !== undefined) { stats.skipped += 1; continue; } const dir = join(root, item.name); const parkedDir = join(this.holdingRoot(), item.name); const inRoot = await isDirectory(dir); const parked = !inRoot && (await isDirectory(parkedDir)); const home = parked ? parkedDir : dir; const exists = inRoot || parked; const removed = state.removed[item.name]; if (removed !== undefined && exists) { // The user put it back by hand; the removal is over. delete state.removed[item.name]; stateDirty = true; } else if (removed !== undefined && item.manifest_sha256 === removed.manifest) { // Deleted locally at this exact cloud version: keep it deleted. stats.skipped += 1; continue; } else if (removed !== undefined) { // The cloud published a different version since the deletion. delete state.removed[item.name]; stateDirty = true; } const delivery = await this.readDelivery(home); if (exists && delivery !== undefined) { // Delivered before: local integrity is judged before any download. if (!(await this.deliveredIntact(home, delivery))) { this.conflict(stats, item.name, 'locally_modified'); continue; } if (delivery.manifest_sha256 === item.manifest_sha256) { if (state.delivered[item.name]?.manifest !== item.manifest_sha256) { state.delivered[item.name] = { manifest: item.manifest_sha256 ?? '' }; stateDirty = true; } stats.skipped += 1; continue; } // Intact but the cloud moved on: fall through and install the update. } if (exists && delivery === undefined) { this.conflict(stats, item.name, 'name_collision'); continue; } if (item.script_changed === true) { // The listing already carries the review flag; don't even fetch. this.conflict(stats, item.name, 'script_review_required'); continue; } const bundle = await client.skillBundle(item.name); if (!bundle.ok) { this.conflict(stats, item.name, `bundle_failed: ${describeError(bundle.error)}`); continue; } if (linkedSkillManifest(bundle.data.files) !== bundle.data.manifest_sha256) { this.conflict(stats, item.name, 'manifest_mismatch'); continue; } if (bundle.data.script_changed === true) { this.conflict(stats, item.name, 'script_review_required'); continue; } await this.install(item, bundle.data, parked ? this.holdingRoot() : root); state.delivered[item.name] = { manifest: bundle.data.manifest_sha256 }; stateDirty = true; (exists ? stats.updated : stats.downloaded).push(item.name); } // ---- UP: local originals -> cloud center --------------------------- const folders = await readdir(root, { withFileTypes: true }).catch(() => []); const seenFolders = new Set(); for (const entry of folders) { if (!entry.isDirectory() || entry.name.startsWith('.')) continue; seenFolders.add(entry.name); const dir = join(root, entry.name); if ((await this.readDelivery(dir)) !== undefined) continue; // cloud-origin: down pass owns it const skill = await this.readLocalSkill(dir, entry.name); if (skill === undefined) continue; if (typeof skill === 'string') { this.conflict(stats, entry.name, skill); continue; } const record = state.uploaded[skill.name]; const cloud = cloudByName.get(skill.name); if (cloud !== undefined && record === undefined) { this.conflict(stats, skill.name, 'name_collision'); continue; } if (record !== undefined && record.manifest === skill.manifest) { stats.skipped += 1; continue; } const files = skill.files.map((file) => { const ext = file.rel.slice(file.rel.lastIndexOf('.')).toLowerCase(); return TEXT_EXT.has(ext) ? { path: file.rel, content: file.buffer.toString('utf8') } : { path: file.rel, base64: file.buffer.toString('base64') }; }); const operationId = `dsh-skill-${randomUUID()}`; if (record === undefined || cloud === undefined) { const created = await client.createSkill({ name: skill.name, description: skill.description, execution_mode: skill.hasScripts ? 'local-bundle' : 'prompt-only', files, expected_revision: 0, operation_id: operationId, }); if (!created.ok) { this.conflict(stats, skill.name, `upload_failed: ${describeError(created.error)}`); continue; } state.uploaded[skill.name] = { folder: entry.name, revision: created.data.skill?.revision ?? created.data.revision ?? 1, manifest: skill.manifest, syncedAt: new Date().toISOString(), }; stateDirty = true; stats.uploaded.push(skill.name); } else { if (cloud.revision !== undefined && cloud.revision !== record.revision) { this.conflict(stats, skill.name, 'cloud_newer'); continue; } const updated = await client.updateSkill(skill.name, { description: skill.description, files, expected_revision: record.revision, operation_id: operationId, }); if (!updated.ok) { const reason = updated.error.code === 'revision_conflict' ? 'cloud_newer' : `upload_failed: ${describeError(updated.error)}`; this.conflict(stats, skill.name, reason); continue; } state.uploaded[skill.name] = { folder: entry.name, revision: updated.data.skill?.revision ?? updated.data.revision ?? record.revision + 1, manifest: skill.manifest, syncedAt: new Date().toISOString(), }; stateDirty = true; stats.updated.push(skill.name); } } // Presence counts the parking area too: a parked skill is disabled, // not deleted. Anything missing from both is a deliberate removal // (remembered, so it is not resurrected); the cloud copy stays. const presentFolders = new Set(seenFolders); for (const entry of await readdir(this.holdingRoot(), { withFileTypes: true }).catch(() => [])) { if (entry.isDirectory()) presentFolders.add(entry.name); } for (const [name, record] of Object.entries(state.uploaded)) { if (!presentFolders.has(record.folder)) { state.removed[name] = { manifest: cloudByName.get(name)?.manifest_sha256 ?? record.manifest }; delete state.uploaded[name]; stateDirty = true; } } for (const [name, record] of Object.entries(state.delivered)) { if (!presentFolders.has(name)) { state.removed[name] = { manifest: record.manifest }; delete state.delivered[name]; stateDirty = true; } } if (stateDirty) await this.saveState(state); const changed = stats.downloaded.length + stats.uploaded.length + stats.updated.length; if (changed > 0 || stats.conflicts.length > 0) { this.deps.log(`skill sync: ${stats.downloaded.length} downloaded, ${stats.uploaded.length} uploaded, ` + `${stats.updated.length} updated, ${stats.conflicts.length} conflict(s)`); } } /** * Read one locally-originated skill for upload. Returns the skill, a * conflict reason string, or undefined when the folder is not a skill. */ async readLocalSkill(dir, folder) { let files; try { files = await this.readSkillFiles(dir); } catch { return undefined; } const markdown = files.find((file) => file.rel === 'SKILL.md'); if (markdown === undefined) return undefined; if (files.length > MAX_FILES) return 'too_many_files'; let total = 0; for (const file of files) { if (file.buffer.length > MAX_FILE_BYTES) return 'file_too_large'; total += file.buffer.length; if (!isCloudSkillPath(file.rel)) return 'unsupported_layout'; } if (total > MAX_SKILL_BYTES) return 'skill_too_large'; const frontmatter = parseSkillFrontmatter(markdown.buffer.toString('utf8')); const name = frontmatter.name !== undefined && CLOUD_SKILL_NAME.test(frontmatter.name) ? frontmatter.name : CLOUD_SKILL_NAME.test(folder) ? folder : undefined; if (name === undefined) return 'invalid_name'; const heading = /^#\s+(.+)$/m.exec(markdown.buffer.toString('utf8'))?.[1]?.trim(); const description = (frontmatter.description ?? heading ?? name).slice(0, 500); const manifest = linkedSkillManifest(files.map((file) => ({ path: file.rel, sha256: sha256Hex(file.buffer) }))); return { name, description, files, manifest, hasScripts: files.some((file) => file.rel.startsWith('scripts/')), }; } }