dsh-brain/lib/skill-sync.js

520 lines
23 KiB
JavaScript

/**
* Bidirectional skill sync between the cloud skill center and the local
* DSH skills directory (`$DSH_HOME/skills` by default).
*
* The cloud center is the hub, the local directory is a working copy, and
* the DSH skills manager keeps owning local management — this module only
* moves files, in both directions:
*
* - DOWN: cloud skills are installed exactly like the platform's own
* `skills-sync` tool does it — bundle hash and per-file checksums are
* verified before anything lands, writes are atomic per file, and a
* per-skill `.manifest.json` records what the cloud delivered.
* - UP: skills that originated locally (installed by the skills manager,
* imported, or written by hand) are published to the center as new
* revisions, guarded by the center's `expected_revision` check.
*
* Safety rules (deliberate, in order):
* 1. A cloud-delivered skill is never re-uploaded. If its local files no
* longer match the delivery manifest, the skill is flagged as a
* conflict and left completely alone.
* 2. A local deletion never deletes the cloud copy.
* 3. A name that already exists on the other side is never hijacked;
* it is reported as a conflict for a human to resolve.
* 4. Bundles whose scripts changed on the cloud side are not installed
* until the change is acknowledged in the dashboard, mirroring the
* center's own review flag.
* 5. Only the center's file layout is uploadable (SKILL.md at the root,
* everything else under references/, scripts/ or assets/, with the
* center's extension allowlist). Anything else is reported, never
* silently trimmed.
*/
import { createHash, randomUUID } from 'node:crypto';
import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
import { dirname, join, resolve } from 'node:path';
import { describeError } from "./api.js";
import { dshHome } from "./store.js";
/** Cloud skill names: lowercase kebab, at most 64 characters. */
export const CLOUD_SKILL_NAME = /^[a-z0-9][a-z0-9-]{0,63}$/;
/** Layout contract of the cloud skill center (mirrors its store rules). */
const TEXT_EXT = new Set(['.md', '.txt', '.json', '.yaml', '.yml', '.py', '.sh', '.ps1', '.mjs', '.js', '.ts', '.tsx', '.css', '.html', '.csv']);
const BINARY_EXT = new Set(['.docx', '.xlsx', '.pptx', '.pdf', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.zip']);
const SCRIPT_EXT = new Set(['.py', '.sh', '.ps1', '.mjs']);
const SKILL_DIRS = new Set(['references', 'scripts', 'assets']);
const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
const MAX_FILES = 100;
const MAX_FILE_BYTES = 512 * 1024;
const MAX_SKILL_BYTES = 2 * 1024 * 1024;
const MAX_DEPTH = 3;
/** SHA-256 of a buffer or UTF-8 string, hex-encoded. */
export function sha256Hex(data) {
return createHash('sha256').update(data).digest('hex');
}
/**
* The center's linked bundle manifest: one `path\0sha256` line per file,
* sorted, under a version header. Byte-identical to the platform's own
* computation, so locally computed manifests compare directly.
*/
export function linkedSkillManifest(files) {
const lines = files.map((file) => `${file.path}\0${file.sha256}`).sort();
return sha256Hex(Buffer.from(`cloud-memory-skill-v1\n${lines.join('\n')}\n`, 'utf8'));
}
/** True when the cloud center would accept this relative file path. */
export function isCloudSkillPath(rel) {
if (rel === '' || rel.length > 200 || rel.startsWith('/') || rel.includes('\\') || rel.includes('//'))
return false;
const segments = rel.split('/');
if (segments.length > MAX_DEPTH)
return false;
for (const segment of segments) {
if (!SEGMENT.test(segment) || segment === '.' || segment === '..')
return false;
}
const dot = rel.lastIndexOf('.');
const ext = dot === -1 ? '' : rel.slice(dot).toLowerCase();
if (!TEXT_EXT.has(ext) && !BINARY_EXT.has(ext))
return false;
const first = segments[0];
if (segments.length === 1)
return rel === 'SKILL.md';
if (first === undefined || !SKILL_DIRS.has(first))
return false;
if (first === 'scripts' && !SCRIPT_EXT.has(ext))
return false;
return true;
}
/** Minimal frontmatter reader for `name` / `description` (plain scalars). */
export function parseSkillFrontmatter(markdown) {
if (!markdown.startsWith('---'))
return {};
const end = markdown.indexOf('\n---', 3);
if (end === -1)
return {};
const out = {};
for (const line of markdown.slice(3, end).split('\n')) {
const match = /^\s*(name|description)\s*:\s*(.+?)\s*$/.exec(line);
if (match === null)
continue;
const value = (match[2] ?? '').replace(/^["']|["']$/g, '');
if (match[1] === 'name' && value !== '')
out.name = value;
if (match[1] === 'description' && value !== '')
out.description = value;
}
return out;
}
async function isDirectory(path) {
return stat(path).then((value) => value.isDirectory()).catch(() => false);
}
export class SkillSync {
deps;
running = false;
current = {
running: false, downloaded: [], uploaded: [], updated: [], conflicts: [], skipped: 0,
};
constructor(deps) {
this.deps = deps;
}
/** Stats of the latest pass, for the status endpoint. */
stats() {
return this.current;
}
/** The local skills root this engine syncs. */
skillsRoot() {
const dir = this.deps.settings().skillsDir;
return dir !== '' ? resolve(dir) : join(dshHome(), 'skills');
}
/** The parking area where disabled skills wait, outside the root. */
holdingRoot() {
return this.deps.holdingDir?.() ?? join(dshHome(), 'skills-disabled');
}
statePath() {
return join(dshHome(), 'cloud-memory-skills.json');
}
async loadState() {
const clean = { version: 1, uploaded: {}, delivered: {}, removed: {} };
try {
const parsed = JSON.parse(await readFile(this.statePath(), 'utf8'));
if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {
const raw = parsed;
const map = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
return {
version: 1,
uploaded: map(raw.uploaded),
delivered: map(raw.delivered),
removed: map(raw.removed),
};
}
}
catch {
// Missing or corrupt sidecar: start clean; the cloud is the truth.
}
return clean;
}
async saveState(state) {
const path = this.statePath();
await mkdir(dirname(path), { recursive: true });
const temporary = `${path}.tmp-${process.pid}`;
await writeFile(temporary, JSON.stringify(state, null, 2));
await rename(temporary, path);
}
/** Read every syncable file of one skill directory (unsorted walk). */
async readSkillFiles(dir) {
const out = [];
const walk = async (sub) => {
const entries = await readdir(join(dir, sub), { withFileTypes: true });
for (const entry of entries) {
const rel = sub === '' ? entry.name : `${sub}/${entry.name}`;
if (entry.name.startsWith('.'))
continue;
if (entry.isDirectory()) {
await walk(rel);
}
else if (entry.isFile()) {
out.push({ rel, buffer: await readFile(join(dir, rel)) });
}
}
};
await walk('');
return out.sort((a, b) => (a.rel < b.rel ? -1 : 1));
}
/** True when a cloud-delivered directory still matches its manifest. */
async deliveredIntact(dir, delivery) {
const recorded = delivery.files ?? [];
let files;
try {
files = await this.readSkillFiles(dir);
}
catch {
return false;
}
const current = files.filter((file) => file.rel !== '.manifest.json');
if (current.length !== recorded.length)
return false;
const byPath = new Map(current.map((file) => [file.rel, sha256Hex(file.buffer)]));
return recorded.every((file) => byPath.get(file.path) === file.sha256);
}
/** Install or update one cloud skill into the local directory. */
async install(item, bundle, root) {
const dir = join(root, item.name);
const previous = await this.readDelivery(dir);
const keep = new Set(bundle.files.map((file) => file.path));
await mkdir(dir, { recursive: true });
const written = [];
for (const file of bundle.files) {
const buffer = Buffer.from(file.data, 'base64');
if (sha256Hex(buffer) !== file.sha256)
throw new Error(`checksum mismatch for ${file.path}`);
const target = join(dir, ...file.path.split('/'));
await mkdir(dirname(target), { recursive: true });
const temporary = `${target}.tmp-${process.pid}`;
await writeFile(temporary, buffer);
await rename(temporary, target);
written.push({ path: file.path, sha256: file.sha256, size: buffer.length });
}
for (const file of previous?.files ?? []) {
if (!keep.has(file.path))
await rm(join(dir, ...file.path.split('/')), { force: true });
}
const delivery = {
name: item.name,
revision: bundle.revision,
manifest_sha256: bundle.manifest_sha256,
files: written,
synced_at: new Date().toISOString(),
};
await writeFile(join(dir, '.manifest.json'), JSON.stringify(delivery, null, 2));
}
async readDelivery(dir) {
try {
const parsed = JSON.parse(await readFile(join(dir, '.manifest.json'), 'utf8'));
if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {
return parsed;
}
}
catch {
// Not a cloud-delivered skill.
}
return undefined;
}
/** One full pass; never throws — outcomes land in the stats. */
async syncOnce() {
if (this.running)
return this.current;
const settings = this.deps.settings();
if (!settings.enabled || !settings.skillsSyncEnabled)
return this.current;
this.running = true;
const stats = {
running: true, downloaded: [], uploaded: [], updated: [], conflicts: [], skipped: 0,
};
this.current = stats;
try {
await this.pass(stats);
stats.lastRunAt = new Date().toISOString();
}
catch (error) {
stats.lastError = error instanceof Error ? error.message : String(error);
this.deps.log(`skill sync failed: ${stats.lastError}`);
}
finally {
stats.running = false;
this.running = false;
this.current = stats;
}
return this.current;
}
/** Record a conflict once per pass, however many rules trip over it. */
conflict(stats, name, reason) {
if (!stats.conflicts.some((item) => item.name === name && item.reason === reason)) {
stats.conflicts.push({ name, reason });
}
}
async pass(stats) {
const client = this.deps.client();
const listing = await client.listSkills();
if (!listing.ok)
throw new Error(`skill listing failed: ${describeError(listing.error)}`);
const items = (listing.data.items ?? []).filter((item) => typeof item.name === 'string' && CLOUD_SKILL_NAME.test(item.name));
const cloudByName = new Map(items.map((item) => [item.name, item]));
const root = this.skillsRoot();
const state = await this.loadState();
let stateDirty = false;
// ---- DOWN: cloud center -> local directory -------------------------
for (const item of items) {
if (item.enabled === false) {
stats.skipped += 1;
continue;
}
// Locally-originated skills belong to the UP pass, even though the
// center lists them too once they are published.
if (state.uploaded[item.name] !== undefined) {
stats.skipped += 1;
continue;
}
const dir = join(root, item.name);
const parkedDir = join(this.holdingRoot(), item.name);
const inRoot = await isDirectory(dir);
const parked = !inRoot && (await isDirectory(parkedDir));
const home = parked ? parkedDir : dir;
const exists = inRoot || parked;
const removed = state.removed[item.name];
if (removed !== undefined && exists) {
// The user put it back by hand; the removal is over.
delete state.removed[item.name];
stateDirty = true;
}
else if (removed !== undefined && item.manifest_sha256 === removed.manifest) {
// Deleted locally at this exact cloud version: keep it deleted.
stats.skipped += 1;
continue;
}
else if (removed !== undefined) {
// The cloud published a different version since the deletion.
delete state.removed[item.name];
stateDirty = true;
}
const delivery = await this.readDelivery(home);
if (exists && delivery !== undefined) {
// Delivered before: local integrity is judged before any download.
if (!(await this.deliveredIntact(home, delivery))) {
this.conflict(stats, item.name, 'locally_modified');
continue;
}
if (delivery.manifest_sha256 === item.manifest_sha256) {
if (state.delivered[item.name]?.manifest !== item.manifest_sha256) {
state.delivered[item.name] = { manifest: item.manifest_sha256 ?? '' };
stateDirty = true;
}
stats.skipped += 1;
continue;
}
// Intact but the cloud moved on: fall through and install the update.
}
if (exists && delivery === undefined) {
this.conflict(stats, item.name, 'name_collision');
continue;
}
if (item.script_changed === true) {
// The listing already carries the review flag; don't even fetch.
this.conflict(stats, item.name, 'script_review_required');
continue;
}
const bundle = await client.skillBundle(item.name);
if (!bundle.ok) {
this.conflict(stats, item.name, `bundle_failed: ${describeError(bundle.error)}`);
continue;
}
if (linkedSkillManifest(bundle.data.files) !== bundle.data.manifest_sha256) {
this.conflict(stats, item.name, 'manifest_mismatch');
continue;
}
if (bundle.data.script_changed === true) {
this.conflict(stats, item.name, 'script_review_required');
continue;
}
await this.install(item, bundle.data, parked ? this.holdingRoot() : root);
state.delivered[item.name] = { manifest: bundle.data.manifest_sha256 };
stateDirty = true;
(exists ? stats.updated : stats.downloaded).push(item.name);
}
// ---- UP: local originals -> cloud center ---------------------------
const folders = await readdir(root, { withFileTypes: true }).catch(() => []);
const seenFolders = new Set();
for (const entry of folders) {
if (!entry.isDirectory() || entry.name.startsWith('.'))
continue;
seenFolders.add(entry.name);
const dir = join(root, entry.name);
if ((await this.readDelivery(dir)) !== undefined)
continue; // cloud-origin: down pass owns it
const skill = await this.readLocalSkill(dir, entry.name);
if (skill === undefined)
continue;
if (typeof skill === 'string') {
this.conflict(stats, entry.name, skill);
continue;
}
const record = state.uploaded[skill.name];
const cloud = cloudByName.get(skill.name);
if (cloud !== undefined && record === undefined) {
this.conflict(stats, skill.name, 'name_collision');
continue;
}
if (record !== undefined && record.manifest === skill.manifest) {
stats.skipped += 1;
continue;
}
const files = skill.files.map((file) => {
const ext = file.rel.slice(file.rel.lastIndexOf('.')).toLowerCase();
return TEXT_EXT.has(ext)
? { path: file.rel, content: file.buffer.toString('utf8') }
: { path: file.rel, base64: file.buffer.toString('base64') };
});
const operationId = `dsh-skill-${randomUUID()}`;
if (record === undefined || cloud === undefined) {
const created = await client.createSkill({
name: skill.name,
description: skill.description,
execution_mode: skill.hasScripts ? 'local-bundle' : 'prompt-only',
files,
expected_revision: 0,
operation_id: operationId,
});
if (!created.ok) {
this.conflict(stats, skill.name, `upload_failed: ${describeError(created.error)}`);
continue;
}
state.uploaded[skill.name] = {
folder: entry.name,
revision: created.data.skill?.revision ?? created.data.revision ?? 1,
manifest: skill.manifest,
syncedAt: new Date().toISOString(),
};
stateDirty = true;
stats.uploaded.push(skill.name);
}
else {
if (cloud.revision !== undefined && cloud.revision !== record.revision) {
this.conflict(stats, skill.name, 'cloud_newer');
continue;
}
const updated = await client.updateSkill(skill.name, {
description: skill.description,
files,
expected_revision: record.revision,
operation_id: operationId,
});
if (!updated.ok) {
const reason = updated.error.code === 'revision_conflict' ? 'cloud_newer' : `upload_failed: ${describeError(updated.error)}`;
this.conflict(stats, skill.name, reason);
continue;
}
state.uploaded[skill.name] = {
folder: entry.name,
revision: updated.data.skill?.revision ?? updated.data.revision ?? record.revision + 1,
manifest: skill.manifest,
syncedAt: new Date().toISOString(),
};
stateDirty = true;
stats.updated.push(skill.name);
}
}
// Presence counts the parking area too: a parked skill is disabled,
// not deleted. Anything missing from both is a deliberate removal
// (remembered, so it is not resurrected); the cloud copy stays.
const presentFolders = new Set(seenFolders);
for (const entry of await readdir(this.holdingRoot(), { withFileTypes: true }).catch(() => [])) {
if (entry.isDirectory())
presentFolders.add(entry.name);
}
for (const [name, record] of Object.entries(state.uploaded)) {
if (!presentFolders.has(record.folder)) {
state.removed[name] = { manifest: cloudByName.get(name)?.manifest_sha256 ?? record.manifest };
delete state.uploaded[name];
stateDirty = true;
}
}
for (const [name, record] of Object.entries(state.delivered)) {
if (!presentFolders.has(name)) {
state.removed[name] = { manifest: record.manifest };
delete state.delivered[name];
stateDirty = true;
}
}
if (stateDirty)
await this.saveState(state);
const changed = stats.downloaded.length + stats.uploaded.length + stats.updated.length;
if (changed > 0 || stats.conflicts.length > 0) {
this.deps.log(`skill sync: ${stats.downloaded.length} downloaded, ${stats.uploaded.length} uploaded, ` +
`${stats.updated.length} updated, ${stats.conflicts.length} conflict(s)`);
}
}
/**
* Read one locally-originated skill for upload. Returns the skill, a
* conflict reason string, or undefined when the folder is not a skill.
*/
async readLocalSkill(dir, folder) {
let files;
try {
files = await this.readSkillFiles(dir);
}
catch {
return undefined;
}
const markdown = files.find((file) => file.rel === 'SKILL.md');
if (markdown === undefined)
return undefined;
if (files.length > MAX_FILES)
return 'too_many_files';
let total = 0;
for (const file of files) {
if (file.buffer.length > MAX_FILE_BYTES)
return 'file_too_large';
total += file.buffer.length;
if (!isCloudSkillPath(file.rel))
return 'unsupported_layout';
}
if (total > MAX_SKILL_BYTES)
return 'skill_too_large';
const frontmatter = parseSkillFrontmatter(markdown.buffer.toString('utf8'));
const name = frontmatter.name !== undefined && CLOUD_SKILL_NAME.test(frontmatter.name)
? frontmatter.name
: CLOUD_SKILL_NAME.test(folder)
? folder
: undefined;
if (name === undefined)
return 'invalid_name';
const heading = /^#\s+(.+)$/m.exec(markdown.buffer.toString('utf8'))?.[1]?.trim();
const description = (frontmatter.description ?? heading ?? name).slice(0, 500);
const manifest = linkedSkillManifest(files.map((file) => ({ path: file.rel, sha256: sha256Hex(file.buffer) })));
return {
name,
description,
files,
manifest,
hasScripts: files.some((file) => file.rel.startsWith('scripts/')),
};
}
}