// The memory tidy. Once a day at most, for each memory that changed, a model reads all // its entries and merges repeats, settles what contradicts, and moves details to topics. // A merge loses words, so nothing is deleted: what leaves goes to retired.jsonl beside // the entries, what a pass did is one batch the user can undo, a pinned entry is never // offered, and one pass changes at most a fifth of the entries. import { randomUUID } from 'node:crypto' import { readFileSync } from 'node:fs' import { join } from 'node:path' import { HEADING, MAIN, addLink, cleanEntry, cleanTopic, entryId, formatEntry, insertEntry, isExpired, isPinned, joinLines, parseMemory, removeLink } from './memory-entries.js' import { tidyRequest, tidySystem } from './memory-copy.js' const EVERY_MS = 30 * 60_000 // Below this there is nothing a tidy would be worth a model call for. const ENTRIES_MIN = 8 const SHARE_MAX = 5 const RETIRED = 'retired.jsonl' const NOTE_MAX = 120 const ITEMS_SHOWN = 20 function parseTidy(text) { const start = text.indexOf('{') const end = text.lastIndexOf('}') if (start === -1 || end <= start) return undefined try { const value = JSON.parse(text.slice(start, end + 1)) return Array.isArray(value?.changes) ? value.changes : undefined } catch { return undefined } } export function installTidy(rt, memory) { const { ctx, config, state } = rt const { TEAM, ownScope, groupScope, keeperOf, scopeKey, dirOf, today, queued, recoverPending, writable, settled, loadScope, publish, writeStep, removeStep, writeText, botShot, languageNow, allEntries, entryProblem, changedSince, USER } = memory // All of it sits in the memory's own folder: a private Bot's words stay in its own, // and an archive of a deleted Bot or group takes them along. const statePath = scope => join(dirOf(scope), 'tidied.json') const batchPath = (scope, batch) => join(dirOf(scope), 'tidy', `${batch}.json`) const doneOf = (scope) => { try { return JSON.parse(readFileSync(statePath(scope), 'utf8')) } catch { return undefined } } // Every memory with someone to read it for: a team tidy offers only its topics, since // what every Bot always reads is the user's to change. function jobs() { const bots = Object.values(state.bots).map(bot => ({ scope: ownScope(bot.id), keeper: bot, who: `one Bot, ${bot.name}` })) const rooms = Object.values(state.rooms).flatMap((room) => { const keeper = keeperOf(room) return keeper === undefined ? [] : [{ scope: groupScope(room.id), keeper, who: `one group chat, "${room.name}"` }] }) const main = Object.values(state.bots).find(bot => rt.isMain(bot.id)) return [...bots, ...rooms, ...(main === undefined ? [] : [{ scope: TEAM, keeper: main, who: 'the notes a team of Bots shares' }])] } const offeredOf = (job, files) => allEntries(job.scope.kind === 'team' ? files.slice(1) : files).filter(entry => !isPinned(entry)) async function tidy(job) { const key = scopeKey(job.scope) const prior = doneOf(job.scope) const day = today() const files = await settled(() => loadScope(job.scope)) const offered = offeredOf(job, files) const ended = offered.filter(entry => isExpired(entry, day)) const open = offered.filter(entry => !isExpired(entry, day)) let changes = [] if (open.length >= ENTRIES_MIN) { const max = Math.max(1, Math.floor(offered.length / SHARE_MAX)) const reply = await botShot(job.keeper, tidySystem({ who: job.who, max }), tidyRequest({ entries: open, language: languageNow() }), { sessionId: job.scope.kind === 'bot' ? rt.chatOf(job.keeper.id) : job.scope.kind === 'group' ? job.scope.roomId : rt.chatOf(job.keeper.id), kind: 'memory' }) const parsed = parseTidy(reply) if (parsed === undefined) rt.warn('ds-bot: the memory tidy of %s got no list of changes from the model', key) else changes = parsed } await queued(async () => { await recoverPending() if (rt.botOf(job.keeper.id) === undefined) return writable(job.scope, USER) await applyTidy(job, key, day, changes, open, ended, prior) }) } async function applyTidy(job, key, day, changes, numbered, ended, prior) { const files = loadScope(job.scope) const offered = offeredOf(job, files) const byId = new Map(offered.map(entry => [entryId(entry), entry])) const draft = new Map(files.map(file => [file.file, [...file.lines]])) const main = files[0] const budget = { left: Math.max(1, Math.floor(offered.length / SHARE_MAX)) } const retired = [] const items = [] const counts = { merged: 0, moved: 0, retired: 0 } const batch = randomUUID() const time = new Date().toISOString() const take = (entry) => { draft.set(entry.file.file, draft.get(entry.file.file).filter(line => line !== entry.file.lines[entry.line])) } const retire = (entry, why, extra = {}) => retired.push({ batch, time, file: entry.file.file, line: entry.file.lines[entry.line], why, ...extra }) const unchanged = (expected) => { if (expected === undefined) return undefined const current = byId.get(entryId(expected)) if (current?.file.file === expected.file.file && current.file.lines[current.line] === expected.file.lines[expected.line]) return current return undefined } const pick = at => unchanged(numbered[Number(at) - 1]) const used = new Set() for (const expected of ended) { const entry = unchanged(expected) if (entry === undefined) continue take(entry) retire(entry, 'ended') used.add(entry) counts.retired += 1 items.push({ op: 'retired', text: entry.text }) } for (const change of changes) { const op = String(change?.op ?? '') if (op === 'about') { const name = cleanTopic(change.topic) const lines = draft.get(`${name}.md`) const text = cleanEntry(change.text).slice(0, NOTE_MAX) if (name === undefined || lines === undefined || text === '' || entryProblem(text) !== undefined) continue const title = lines.findIndex(line => HEADING.test(line)) const kept = lines.filter(line => !/^>\s*\S/.test(line)) kept.splice(title + 1, 0, '', `> ${text}`) draft.set(`${name}.md`, kept) continue } if (op === 'merge') { const entries = [...new Set((Array.isArray(change.entries) ? change.entries : []).map(pick))] if (entries.some(entry => entry === undefined || used.has(entry))) continue const text = cleanEntry(change.text) if (entries.length < 2 || entries.length > budget.left || entryProblem(text) !== undefined) continue budget.left -= entries.length const [first, ...rest] = entries const line = formatEntry(text, { ...first.meta, id: entryId(first), merged: entries.map(entryId).join(' ') }) draft.set(first.file.file, draft.get(first.file.file).map(item => (item === first.file.lines[first.line] ? line : item))) rest.forEach(take) entries.forEach(entry => { retire(entry, 'merged', { into: entryId(first) }); used.add(entry) }) counts.merged += entries.length items.push({ op: 'merged', text }) continue } const entry = pick(change?.entry) if (entry === undefined || used.has(entry) || budget.left < 1) continue if (op === 'retire') { budget.left -= 1 take(entry) retire(entry, String(change.why ?? '').slice(0, NOTE_MAX) || 'retired') used.add(entry) counts.retired += 1 items.push({ op: 'retired', text: entry.text }) } else if (op === 'move' && entry.file.topic === undefined) { const name = cleanTopic(change.topic) if (name === undefined) continue budget.left -= 1 const line = entry.file.lines[entry.line] take(entry) const base = draft.get(`${name}.md`) ?? parseMemory(`# ${name}\n`).lines draft.set(`${name}.md`, insertEntry(base, line)) if (!parseMemory(joinLines(draft.get(MAIN))).links.includes(name)) draft.set(MAIN, addLink(draft.get(MAIN), name)) used.add(entry) counts.moved += 1 items.push({ op: 'moved', text: entry.text, topic: name }) } } const dir = dirOf(job.scope) const touched = [] const steps = [] const before = new Map(files.map(file => [file.file, file.content])) for (const [file, lines] of draft) { const emptied = file !== MAIN && lines.every(line => line.trim() === '' || HEADING.test(line) || /^>\s/.test(line)) const was = before.get(file) ?? null const now = emptied ? null : lines.length === 0 ? '' : joinLines(lines) if (was === now || (was === '' && lines.length === 0)) continue touched.push({ file, before: was, after: now }) } for (const item of touched.filter(entry => entry.after === null)) { const name = item.file.slice(0, -3) const lines = removeLink(draft.get(MAIN), name) draft.set(MAIN, lines) } const mainNow = joinLines(draft.get(MAIN)) const mainItem = touched.find(item => item.file === MAIN) if (mainItem !== undefined) mainItem.after = mainNow else if (main.content !== mainNow && main.lines.length > 0) touched.push({ file: MAIN, before: main.content, after: mainNow }) const record = { time, day, ...(prior?.batch === undefined ? {} : { previous: prior.batch }) } const stamp = { ...record, ...counts, ...(touched.length === 0 ? {} : { batch, items: items.slice(0, ITEMS_SHOWN) }) } if (touched.length === 0) { if (rt.readOnly() === undefined) await writeText(statePath(job.scope), JSON.stringify(stamp)) return } for (const item of touched) steps.push(item.after === null ? removeStep(join(dir, item.file)) : writeStep(join(dir, item.file), item.after)) const retiredPath = join(dir, RETIRED) let log = '' try { log = readFileSync(retiredPath, 'utf8') } catch { /* none yet */ } if (retired.length > 0) steps.push(writeStep(retiredPath, `${log}${retired.map(row => JSON.stringify(row)).join('\n')}\n`)) steps.push(writeStep(batchPath(job.scope, batch), JSON.stringify({ batch, scope: key, files: touched, retiredBefore: log }))) steps.push(writeStep(statePath(job.scope), JSON.stringify(stamp))) await publish({ op: 'tidy', scope: key, steps, row: { op: 'tidy', scope: key, batch, ...counts, actor: job.keeper.id, actorName: job.keeper.name, auto: true } }) } // A batch comes back whole, or not at all: a file changed since is left as it reads. async function undoTidy(scope, batch) { const key = scopeKey(scope) await queued(async () => { await recoverPending() writable(scope, USER) const last = doneOf(scope) if (batch === '' || last?.batch !== batch) throw new Error('That tidy is not the latest one of this memory.') const saved = JSON.parse(readFileSync(batchPath(scope, batch), 'utf8')) const dir = dirOf(scope) const read = (file) => { try { return readFileSync(join(dir, file), 'utf8') } catch { return null } } if (saved.files.some(item => read(item.file) !== item.after)) throw new Error('This memory has changed since that tidy, so it cannot be undone as a whole.') const steps = saved.files.map(item => (item.before === null ? removeStep(join(dir, item.file)) : writeStep(join(dir, item.file), item.before))) steps.push(saved.retiredBefore === '' ? removeStep(join(dir, RETIRED)) : writeStep(join(dir, RETIRED), saved.retiredBefore)) const { batch: _gone, items: _items, ...rest } = last steps.push(writeStep(statePath(scope), JSON.stringify({ ...rest, undone: batch }))) await publish({ op: 'tidy-undo', scope: key, steps, row: { op: 'tidy-undo', scope: key, batch, actor: 'user', actorName: 'user' } }) }) } const tidyView = (scopes) => { const latest = Object.entries(scopes).map(([tag, scope]) => ({ tag, ...doneOf(scope) })).filter(item => item.batch !== undefined).sort((a, b) => String(b.time).localeCompare(String(a.time)))[0] return latest === undefined ? null : { scope: latest.tag, batch: latest.batch, at: latest.time, merged: latest.merged, moved: latest.moved, retired: latest.retired, items: latest.items ?? [] } } // One memory at a time, and only one that changed since its last tidy, on a new day. const due = (job) => { const last = doneOf(job.scope) return last?.day !== today() && changedSince(scopeKey(job.scope), last?.time) } let running = false async function tick() { if (running || rt.readOnly() !== undefined) return running = true try { await rt.load() const job = jobs().find(due) if (job !== undefined) await tidy(job) } catch (error) { rt.warn('ds-bot: the memory tidy failed: %s', error?.message ?? error) } finally { running = false } } if (config.memoryTidy !== false) { ctx.effect(() => { const timer = setInterval(() => void tick(), config.memoryTidyEveryMs ?? EVERY_MS) timer.unref?.() return () => clearInterval(timer) }, 'ds-bot: memory tidy') } const tidyNow = async (key) => { await rt.load() const job = jobs().find(item => scopeKey(item.scope) === key) if (job !== undefined && doneOf(job.scope)?.day !== today()) await tidy(job) } return { tidyView, undoTidy, tidyNow } }