520 lines
23 KiB
JavaScript
520 lines
23 KiB
JavaScript
/**
|
|
* Bidirectional skill sync between the cloud skill center and the local
|
|
* DSH skills directory (`$DSH_HOME/skills` by default).
|
|
*
|
|
* The cloud center is the hub, the local directory is a working copy, and
|
|
* the DSH skills manager keeps owning local management — this module only
|
|
* moves files, in both directions:
|
|
*
|
|
* - DOWN: cloud skills are installed exactly like the platform's own
|
|
* `skills-sync` tool does it — bundle hash and per-file checksums are
|
|
* verified before anything lands, writes are atomic per file, and a
|
|
* per-skill `.manifest.json` records what the cloud delivered.
|
|
* - UP: skills that originated locally (installed by the skills manager,
|
|
* imported, or written by hand) are published to the center as new
|
|
* revisions, guarded by the center's `expected_revision` check.
|
|
*
|
|
* Safety rules (deliberate, in order):
|
|
* 1. A cloud-delivered skill is never re-uploaded. If its local files no
|
|
* longer match the delivery manifest, the skill is flagged as a
|
|
* conflict and left completely alone.
|
|
* 2. A local deletion never deletes the cloud copy.
|
|
* 3. A name that already exists on the other side is never hijacked;
|
|
* it is reported as a conflict for a human to resolve.
|
|
* 4. Bundles whose scripts changed on the cloud side are not installed
|
|
* until the change is acknowledged in the dashboard, mirroring the
|
|
* center's own review flag.
|
|
* 5. Only the center's file layout is uploadable (SKILL.md at the root,
|
|
* everything else under references/, scripts/ or assets/, with the
|
|
* center's extension allowlist). Anything else is reported, never
|
|
* silently trimmed.
|
|
*/
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
|
|
import { dirname, join, resolve } from 'node:path';
|
|
import { describeError } from "./api.js";
|
|
import { dshHome } from "./store.js";
|
|
/** Cloud skill names: lowercase kebab, at most 64 characters. */
|
|
export const CLOUD_SKILL_NAME = /^[a-z0-9][a-z0-9-]{0,63}$/;
|
|
/** Layout contract of the cloud skill center (mirrors its store rules). */
|
|
const TEXT_EXT = new Set(['.md', '.txt', '.json', '.yaml', '.yml', '.py', '.sh', '.ps1', '.mjs', '.js', '.ts', '.tsx', '.css', '.html', '.csv']);
|
|
const BINARY_EXT = new Set(['.docx', '.xlsx', '.pptx', '.pdf', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.zip']);
|
|
const SCRIPT_EXT = new Set(['.py', '.sh', '.ps1', '.mjs']);
|
|
const SKILL_DIRS = new Set(['references', 'scripts', 'assets']);
|
|
const SEGMENT = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
|
|
const MAX_FILES = 100;
|
|
const MAX_FILE_BYTES = 512 * 1024;
|
|
const MAX_SKILL_BYTES = 2 * 1024 * 1024;
|
|
const MAX_DEPTH = 3;
|
|
/** SHA-256 of a buffer or UTF-8 string, hex-encoded. */
|
|
export function sha256Hex(data) {
|
|
return createHash('sha256').update(data).digest('hex');
|
|
}
|
|
/**
|
|
* The center's linked bundle manifest: one `path\0sha256` line per file,
|
|
* sorted, under a version header. Byte-identical to the platform's own
|
|
* computation, so locally computed manifests compare directly.
|
|
*/
|
|
export function linkedSkillManifest(files) {
|
|
const lines = files.map((file) => `${file.path}\0${file.sha256}`).sort();
|
|
return sha256Hex(Buffer.from(`cloud-memory-skill-v1\n${lines.join('\n')}\n`, 'utf8'));
|
|
}
|
|
/** True when the cloud center would accept this relative file path. */
|
|
export function isCloudSkillPath(rel) {
|
|
if (rel === '' || rel.length > 200 || rel.startsWith('/') || rel.includes('\\') || rel.includes('//'))
|
|
return false;
|
|
const segments = rel.split('/');
|
|
if (segments.length > MAX_DEPTH)
|
|
return false;
|
|
for (const segment of segments) {
|
|
if (!SEGMENT.test(segment) || segment === '.' || segment === '..')
|
|
return false;
|
|
}
|
|
const dot = rel.lastIndexOf('.');
|
|
const ext = dot === -1 ? '' : rel.slice(dot).toLowerCase();
|
|
if (!TEXT_EXT.has(ext) && !BINARY_EXT.has(ext))
|
|
return false;
|
|
const first = segments[0];
|
|
if (segments.length === 1)
|
|
return rel === 'SKILL.md';
|
|
if (first === undefined || !SKILL_DIRS.has(first))
|
|
return false;
|
|
if (first === 'scripts' && !SCRIPT_EXT.has(ext))
|
|
return false;
|
|
return true;
|
|
}
|
|
/** Minimal frontmatter reader for `name` / `description` (plain scalars). */
|
|
export function parseSkillFrontmatter(markdown) {
|
|
if (!markdown.startsWith('---'))
|
|
return {};
|
|
const end = markdown.indexOf('\n---', 3);
|
|
if (end === -1)
|
|
return {};
|
|
const out = {};
|
|
for (const line of markdown.slice(3, end).split('\n')) {
|
|
const match = /^\s*(name|description)\s*:\s*(.+?)\s*$/.exec(line);
|
|
if (match === null)
|
|
continue;
|
|
const value = (match[2] ?? '').replace(/^["']|["']$/g, '');
|
|
if (match[1] === 'name' && value !== '')
|
|
out.name = value;
|
|
if (match[1] === 'description' && value !== '')
|
|
out.description = value;
|
|
}
|
|
return out;
|
|
}
|
|
async function isDirectory(path) {
|
|
return stat(path).then((value) => value.isDirectory()).catch(() => false);
|
|
}
|
|
export class SkillSync {
|
|
deps;
|
|
running = false;
|
|
current = {
|
|
running: false, downloaded: [], uploaded: [], updated: [], conflicts: [], skipped: 0,
|
|
};
|
|
constructor(deps) {
|
|
this.deps = deps;
|
|
}
|
|
/** Stats of the latest pass, for the status endpoint. */
|
|
stats() {
|
|
return this.current;
|
|
}
|
|
/** The local skills root this engine syncs. */
|
|
skillsRoot() {
|
|
const dir = this.deps.settings().skillsDir;
|
|
return dir !== '' ? resolve(dir) : join(dshHome(), 'skills');
|
|
}
|
|
/** The parking area where disabled skills wait, outside the root. */
|
|
holdingRoot() {
|
|
return this.deps.holdingDir?.() ?? join(dshHome(), 'skills-disabled');
|
|
}
|
|
statePath() {
|
|
return join(dshHome(), 'cloud-memory-skills.json');
|
|
}
|
|
async loadState() {
|
|
const clean = { version: 1, uploaded: {}, delivered: {}, removed: {} };
|
|
try {
|
|
const parsed = JSON.parse(await readFile(this.statePath(), 'utf8'));
|
|
if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
const raw = parsed;
|
|
const map = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
return {
|
|
version: 1,
|
|
uploaded: map(raw.uploaded),
|
|
delivered: map(raw.delivered),
|
|
removed: map(raw.removed),
|
|
};
|
|
}
|
|
}
|
|
catch {
|
|
// Missing or corrupt sidecar: start clean; the cloud is the truth.
|
|
}
|
|
return clean;
|
|
}
|
|
async saveState(state) {
|
|
const path = this.statePath();
|
|
await mkdir(dirname(path), { recursive: true });
|
|
const temporary = `${path}.tmp-${process.pid}`;
|
|
await writeFile(temporary, JSON.stringify(state, null, 2));
|
|
await rename(temporary, path);
|
|
}
|
|
/** Read every syncable file of one skill directory (unsorted walk). */
|
|
async readSkillFiles(dir) {
|
|
const out = [];
|
|
const walk = async (sub) => {
|
|
const entries = await readdir(join(dir, sub), { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const rel = sub === '' ? entry.name : `${sub}/${entry.name}`;
|
|
if (entry.name.startsWith('.'))
|
|
continue;
|
|
if (entry.isDirectory()) {
|
|
await walk(rel);
|
|
}
|
|
else if (entry.isFile()) {
|
|
out.push({ rel, buffer: await readFile(join(dir, rel)) });
|
|
}
|
|
}
|
|
};
|
|
await walk('');
|
|
return out.sort((a, b) => (a.rel < b.rel ? -1 : 1));
|
|
}
|
|
/** True when a cloud-delivered directory still matches its manifest. */
|
|
async deliveredIntact(dir, delivery) {
|
|
const recorded = delivery.files ?? [];
|
|
let files;
|
|
try {
|
|
files = await this.readSkillFiles(dir);
|
|
}
|
|
catch {
|
|
return false;
|
|
}
|
|
const current = files.filter((file) => file.rel !== '.manifest.json');
|
|
if (current.length !== recorded.length)
|
|
return false;
|
|
const byPath = new Map(current.map((file) => [file.rel, sha256Hex(file.buffer)]));
|
|
return recorded.every((file) => byPath.get(file.path) === file.sha256);
|
|
}
|
|
/** Install or update one cloud skill into the local directory. */
|
|
async install(item, bundle, root) {
|
|
const dir = join(root, item.name);
|
|
const previous = await this.readDelivery(dir);
|
|
const keep = new Set(bundle.files.map((file) => file.path));
|
|
await mkdir(dir, { recursive: true });
|
|
const written = [];
|
|
for (const file of bundle.files) {
|
|
const buffer = Buffer.from(file.data, 'base64');
|
|
if (sha256Hex(buffer) !== file.sha256)
|
|
throw new Error(`checksum mismatch for ${file.path}`);
|
|
const target = join(dir, ...file.path.split('/'));
|
|
await mkdir(dirname(target), { recursive: true });
|
|
const temporary = `${target}.tmp-${process.pid}`;
|
|
await writeFile(temporary, buffer);
|
|
await rename(temporary, target);
|
|
written.push({ path: file.path, sha256: file.sha256, size: buffer.length });
|
|
}
|
|
for (const file of previous?.files ?? []) {
|
|
if (!keep.has(file.path))
|
|
await rm(join(dir, ...file.path.split('/')), { force: true });
|
|
}
|
|
const delivery = {
|
|
name: item.name,
|
|
revision: bundle.revision,
|
|
manifest_sha256: bundle.manifest_sha256,
|
|
files: written,
|
|
synced_at: new Date().toISOString(),
|
|
};
|
|
await writeFile(join(dir, '.manifest.json'), JSON.stringify(delivery, null, 2));
|
|
}
|
|
async readDelivery(dir) {
|
|
try {
|
|
const parsed = JSON.parse(await readFile(join(dir, '.manifest.json'), 'utf8'));
|
|
if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
return parsed;
|
|
}
|
|
}
|
|
catch {
|
|
// Not a cloud-delivered skill.
|
|
}
|
|
return undefined;
|
|
}
|
|
/** One full pass; never throws — outcomes land in the stats. */
|
|
async syncOnce() {
|
|
if (this.running)
|
|
return this.current;
|
|
const settings = this.deps.settings();
|
|
if (!settings.enabled || !settings.skillsSyncEnabled)
|
|
return this.current;
|
|
this.running = true;
|
|
const stats = {
|
|
running: true, downloaded: [], uploaded: [], updated: [], conflicts: [], skipped: 0,
|
|
};
|
|
this.current = stats;
|
|
try {
|
|
await this.pass(stats);
|
|
stats.lastRunAt = new Date().toISOString();
|
|
}
|
|
catch (error) {
|
|
stats.lastError = error instanceof Error ? error.message : String(error);
|
|
this.deps.log(`skill sync failed: ${stats.lastError}`);
|
|
}
|
|
finally {
|
|
stats.running = false;
|
|
this.running = false;
|
|
this.current = stats;
|
|
}
|
|
return this.current;
|
|
}
|
|
/** Record a conflict once per pass, however many rules trip over it. */
|
|
conflict(stats, name, reason) {
|
|
if (!stats.conflicts.some((item) => item.name === name && item.reason === reason)) {
|
|
stats.conflicts.push({ name, reason });
|
|
}
|
|
}
|
|
async pass(stats) {
|
|
const client = this.deps.client();
|
|
const listing = await client.listSkills();
|
|
if (!listing.ok)
|
|
throw new Error(`skill listing failed: ${describeError(listing.error)}`);
|
|
const items = (listing.data.items ?? []).filter((item) => typeof item.name === 'string' && CLOUD_SKILL_NAME.test(item.name));
|
|
const cloudByName = new Map(items.map((item) => [item.name, item]));
|
|
const root = this.skillsRoot();
|
|
const state = await this.loadState();
|
|
let stateDirty = false;
|
|
// ---- DOWN: cloud center -> local directory -------------------------
|
|
for (const item of items) {
|
|
if (item.enabled === false) {
|
|
stats.skipped += 1;
|
|
continue;
|
|
}
|
|
// Locally-originated skills belong to the UP pass, even though the
|
|
// center lists them too once they are published.
|
|
if (state.uploaded[item.name] !== undefined) {
|
|
stats.skipped += 1;
|
|
continue;
|
|
}
|
|
const dir = join(root, item.name);
|
|
const parkedDir = join(this.holdingRoot(), item.name);
|
|
const inRoot = await isDirectory(dir);
|
|
const parked = !inRoot && (await isDirectory(parkedDir));
|
|
const home = parked ? parkedDir : dir;
|
|
const exists = inRoot || parked;
|
|
const removed = state.removed[item.name];
|
|
if (removed !== undefined && exists) {
|
|
// The user put it back by hand; the removal is over.
|
|
delete state.removed[item.name];
|
|
stateDirty = true;
|
|
}
|
|
else if (removed !== undefined && item.manifest_sha256 === removed.manifest) {
|
|
// Deleted locally at this exact cloud version: keep it deleted.
|
|
stats.skipped += 1;
|
|
continue;
|
|
}
|
|
else if (removed !== undefined) {
|
|
// The cloud published a different version since the deletion.
|
|
delete state.removed[item.name];
|
|
stateDirty = true;
|
|
}
|
|
const delivery = await this.readDelivery(home);
|
|
if (exists && delivery !== undefined) {
|
|
// Delivered before: local integrity is judged before any download.
|
|
if (!(await this.deliveredIntact(home, delivery))) {
|
|
this.conflict(stats, item.name, 'locally_modified');
|
|
continue;
|
|
}
|
|
if (delivery.manifest_sha256 === item.manifest_sha256) {
|
|
if (state.delivered[item.name]?.manifest !== item.manifest_sha256) {
|
|
state.delivered[item.name] = { manifest: item.manifest_sha256 ?? '' };
|
|
stateDirty = true;
|
|
}
|
|
stats.skipped += 1;
|
|
continue;
|
|
}
|
|
// Intact but the cloud moved on: fall through and install the update.
|
|
}
|
|
if (exists && delivery === undefined) {
|
|
this.conflict(stats, item.name, 'name_collision');
|
|
continue;
|
|
}
|
|
if (item.script_changed === true) {
|
|
// The listing already carries the review flag; don't even fetch.
|
|
this.conflict(stats, item.name, 'script_review_required');
|
|
continue;
|
|
}
|
|
const bundle = await client.skillBundle(item.name);
|
|
if (!bundle.ok) {
|
|
this.conflict(stats, item.name, `bundle_failed: ${describeError(bundle.error)}`);
|
|
continue;
|
|
}
|
|
if (linkedSkillManifest(bundle.data.files) !== bundle.data.manifest_sha256) {
|
|
this.conflict(stats, item.name, 'manifest_mismatch');
|
|
continue;
|
|
}
|
|
if (bundle.data.script_changed === true) {
|
|
this.conflict(stats, item.name, 'script_review_required');
|
|
continue;
|
|
}
|
|
await this.install(item, bundle.data, parked ? this.holdingRoot() : root);
|
|
state.delivered[item.name] = { manifest: bundle.data.manifest_sha256 };
|
|
stateDirty = true;
|
|
(exists ? stats.updated : stats.downloaded).push(item.name);
|
|
}
|
|
// ---- UP: local originals -> cloud center ---------------------------
|
|
const folders = await readdir(root, { withFileTypes: true }).catch(() => []);
|
|
const seenFolders = new Set();
|
|
for (const entry of folders) {
|
|
if (!entry.isDirectory() || entry.name.startsWith('.'))
|
|
continue;
|
|
seenFolders.add(entry.name);
|
|
const dir = join(root, entry.name);
|
|
if ((await this.readDelivery(dir)) !== undefined)
|
|
continue; // cloud-origin: down pass owns it
|
|
const skill = await this.readLocalSkill(dir, entry.name);
|
|
if (skill === undefined)
|
|
continue;
|
|
if (typeof skill === 'string') {
|
|
this.conflict(stats, entry.name, skill);
|
|
continue;
|
|
}
|
|
const record = state.uploaded[skill.name];
|
|
const cloud = cloudByName.get(skill.name);
|
|
if (cloud !== undefined && record === undefined) {
|
|
this.conflict(stats, skill.name, 'name_collision');
|
|
continue;
|
|
}
|
|
if (record !== undefined && record.manifest === skill.manifest) {
|
|
stats.skipped += 1;
|
|
continue;
|
|
}
|
|
const files = skill.files.map((file) => {
|
|
const ext = file.rel.slice(file.rel.lastIndexOf('.')).toLowerCase();
|
|
return TEXT_EXT.has(ext)
|
|
? { path: file.rel, content: file.buffer.toString('utf8') }
|
|
: { path: file.rel, base64: file.buffer.toString('base64') };
|
|
});
|
|
const operationId = `dsh-skill-${randomUUID()}`;
|
|
if (record === undefined || cloud === undefined) {
|
|
const created = await client.createSkill({
|
|
name: skill.name,
|
|
description: skill.description,
|
|
execution_mode: skill.hasScripts ? 'local-bundle' : 'prompt-only',
|
|
files,
|
|
expected_revision: 0,
|
|
operation_id: operationId,
|
|
});
|
|
if (!created.ok) {
|
|
this.conflict(stats, skill.name, `upload_failed: ${describeError(created.error)}`);
|
|
continue;
|
|
}
|
|
state.uploaded[skill.name] = {
|
|
folder: entry.name,
|
|
revision: created.data.skill?.revision ?? created.data.revision ?? 1,
|
|
manifest: skill.manifest,
|
|
syncedAt: new Date().toISOString(),
|
|
};
|
|
stateDirty = true;
|
|
stats.uploaded.push(skill.name);
|
|
}
|
|
else {
|
|
if (cloud.revision !== undefined && cloud.revision !== record.revision) {
|
|
this.conflict(stats, skill.name, 'cloud_newer');
|
|
continue;
|
|
}
|
|
const updated = await client.updateSkill(skill.name, {
|
|
description: skill.description,
|
|
files,
|
|
expected_revision: record.revision,
|
|
operation_id: operationId,
|
|
});
|
|
if (!updated.ok) {
|
|
const reason = updated.error.code === 'revision_conflict' ? 'cloud_newer' : `upload_failed: ${describeError(updated.error)}`;
|
|
this.conflict(stats, skill.name, reason);
|
|
continue;
|
|
}
|
|
state.uploaded[skill.name] = {
|
|
folder: entry.name,
|
|
revision: updated.data.skill?.revision ?? updated.data.revision ?? record.revision + 1,
|
|
manifest: skill.manifest,
|
|
syncedAt: new Date().toISOString(),
|
|
};
|
|
stateDirty = true;
|
|
stats.updated.push(skill.name);
|
|
}
|
|
}
|
|
// Presence counts the parking area too: a parked skill is disabled,
|
|
// not deleted. Anything missing from both is a deliberate removal
|
|
// (remembered, so it is not resurrected); the cloud copy stays.
|
|
const presentFolders = new Set(seenFolders);
|
|
for (const entry of await readdir(this.holdingRoot(), { withFileTypes: true }).catch(() => [])) {
|
|
if (entry.isDirectory())
|
|
presentFolders.add(entry.name);
|
|
}
|
|
for (const [name, record] of Object.entries(state.uploaded)) {
|
|
if (!presentFolders.has(record.folder)) {
|
|
state.removed[name] = { manifest: cloudByName.get(name)?.manifest_sha256 ?? record.manifest };
|
|
delete state.uploaded[name];
|
|
stateDirty = true;
|
|
}
|
|
}
|
|
for (const [name, record] of Object.entries(state.delivered)) {
|
|
if (!presentFolders.has(name)) {
|
|
state.removed[name] = { manifest: record.manifest };
|
|
delete state.delivered[name];
|
|
stateDirty = true;
|
|
}
|
|
}
|
|
if (stateDirty)
|
|
await this.saveState(state);
|
|
const changed = stats.downloaded.length + stats.uploaded.length + stats.updated.length;
|
|
if (changed > 0 || stats.conflicts.length > 0) {
|
|
this.deps.log(`skill sync: ${stats.downloaded.length} downloaded, ${stats.uploaded.length} uploaded, ` +
|
|
`${stats.updated.length} updated, ${stats.conflicts.length} conflict(s)`);
|
|
}
|
|
}
|
|
/**
|
|
* Read one locally-originated skill for upload. Returns the skill, a
|
|
* conflict reason string, or undefined when the folder is not a skill.
|
|
*/
|
|
async readLocalSkill(dir, folder) {
|
|
let files;
|
|
try {
|
|
files = await this.readSkillFiles(dir);
|
|
}
|
|
catch {
|
|
return undefined;
|
|
}
|
|
const markdown = files.find((file) => file.rel === 'SKILL.md');
|
|
if (markdown === undefined)
|
|
return undefined;
|
|
if (files.length > MAX_FILES)
|
|
return 'too_many_files';
|
|
let total = 0;
|
|
for (const file of files) {
|
|
if (file.buffer.length > MAX_FILE_BYTES)
|
|
return 'file_too_large';
|
|
total += file.buffer.length;
|
|
if (!isCloudSkillPath(file.rel))
|
|
return 'unsupported_layout';
|
|
}
|
|
if (total > MAX_SKILL_BYTES)
|
|
return 'skill_too_large';
|
|
const frontmatter = parseSkillFrontmatter(markdown.buffer.toString('utf8'));
|
|
const name = frontmatter.name !== undefined && CLOUD_SKILL_NAME.test(frontmatter.name)
|
|
? frontmatter.name
|
|
: CLOUD_SKILL_NAME.test(folder)
|
|
? folder
|
|
: undefined;
|
|
if (name === undefined)
|
|
return 'invalid_name';
|
|
const heading = /^#\s+(.+)$/m.exec(markdown.buffer.toString('utf8'))?.[1]?.trim();
|
|
const description = (frontmatter.description ?? heading ?? name).slice(0, 500);
|
|
const manifest = linkedSkillManifest(files.map((file) => ({ path: file.rel, sha256: sha256Hex(file.buffer) })));
|
|
return {
|
|
name,
|
|
description,
|
|
files,
|
|
manifest,
|
|
hasScripts: files.some((file) => file.rel.startsWith('scripts/')),
|
|
};
|
|
}
|
|
}
|