dsh-sansheng-liubu/ds-bot/src/host/memory-tidy.js

265 lines
13 KiB
JavaScript

// The memory tidy. Once a day at most, for each memory that changed, a model reads all
// its entries and merges repeats, settles what contradicts, and moves details to topics.
// A merge loses words, so nothing is deleted: what leaves goes to retired.jsonl beside
// the entries, what a pass did is one batch the user can undo, a pinned entry is never
// offered, and one pass changes at most a fifth of the entries.
import { randomUUID } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { HEADING, MAIN, addLink, cleanEntry, cleanTopic, entryId, formatEntry, insertEntry, isExpired, isPinned, joinLines, parseMemory, removeLink } from './memory-entries.js'
import { tidyRequest, tidySystem } from './memory-copy.js'
const EVERY_MS = 30 * 60_000
// Below this there is nothing a tidy would be worth a model call for.
const ENTRIES_MIN = 8
const SHARE_MAX = 5
const RETIRED = 'retired.jsonl'
const NOTE_MAX = 120
const ITEMS_SHOWN = 20
function parseTidy(text) {
const start = text.indexOf('{')
const end = text.lastIndexOf('}')
if (start === -1 || end <= start) return undefined
try {
const value = JSON.parse(text.slice(start, end + 1))
return Array.isArray(value?.changes) ? value.changes : undefined
} catch {
return undefined
}
}
export function installTidy(rt, memory) {
const { ctx, config, state } = rt
const { TEAM, ownScope, groupScope, keeperOf, scopeKey, dirOf, today, queued, recoverPending, writable, settled, loadScope, publish, writeStep, removeStep, writeText, botShot, languageNow, allEntries, entryProblem, changedSince, USER } = memory
// All of it sits in the memory's own folder: a private Bot's words stay in its own,
// and an archive of a deleted Bot or group takes them along.
const statePath = scope => join(dirOf(scope), 'tidied.json')
const batchPath = (scope, batch) => join(dirOf(scope), 'tidy', `${batch}.json`)
const doneOf = (scope) => {
try {
return JSON.parse(readFileSync(statePath(scope), 'utf8'))
} catch {
return undefined
}
}
// Every memory with someone to read it for: a team tidy offers only its topics, since
// what every Bot always reads is the user's to change.
function jobs() {
const bots = Object.values(state.bots).map(bot => ({ scope: ownScope(bot.id), keeper: bot, who: `one Bot, ${bot.name}` }))
const rooms = Object.values(state.rooms).flatMap((room) => {
const keeper = keeperOf(room)
return keeper === undefined ? [] : [{ scope: groupScope(room.id), keeper, who: `one group chat, "${room.name}"` }]
})
const main = Object.values(state.bots).find(bot => rt.isMain(bot.id))
return [...bots, ...rooms, ...(main === undefined ? [] : [{ scope: TEAM, keeper: main, who: 'the notes a team of Bots shares' }])]
}
const offeredOf = (job, files) => allEntries(job.scope.kind === 'team' ? files.slice(1) : files).filter(entry => !isPinned(entry))
async function tidy(job) {
const key = scopeKey(job.scope)
const prior = doneOf(job.scope)
const day = today()
const files = await settled(() => loadScope(job.scope))
const offered = offeredOf(job, files)
const ended = offered.filter(entry => isExpired(entry, day))
const open = offered.filter(entry => !isExpired(entry, day))
let changes = []
if (open.length >= ENTRIES_MIN) {
const max = Math.max(1, Math.floor(offered.length / SHARE_MAX))
const reply = await botShot(job.keeper, tidySystem({ who: job.who, max }), tidyRequest({ entries: open, language: languageNow() }), { sessionId: job.scope.kind === 'bot' ? rt.chatOf(job.keeper.id) : job.scope.kind === 'group' ? job.scope.roomId : rt.chatOf(job.keeper.id), kind: 'memory' })
const parsed = parseTidy(reply)
if (parsed === undefined) rt.warn('ds-bot: the memory tidy of %s got no list of changes from the model', key)
else changes = parsed
}
await queued(async () => {
await recoverPending()
if (rt.botOf(job.keeper.id) === undefined) return
writable(job.scope, USER)
await applyTidy(job, key, day, changes, open, ended, prior)
})
}
async function applyTidy(job, key, day, changes, numbered, ended, prior) {
const files = loadScope(job.scope)
const offered = offeredOf(job, files)
const byId = new Map(offered.map(entry => [entryId(entry), entry]))
const draft = new Map(files.map(file => [file.file, [...file.lines]]))
const main = files[0]
const budget = { left: Math.max(1, Math.floor(offered.length / SHARE_MAX)) }
const retired = []
const items = []
const counts = { merged: 0, moved: 0, retired: 0 }
const batch = randomUUID()
const time = new Date().toISOString()
const take = (entry) => { draft.set(entry.file.file, draft.get(entry.file.file).filter(line => line !== entry.file.lines[entry.line])) }
const retire = (entry, why, extra = {}) => retired.push({ batch, time, file: entry.file.file, line: entry.file.lines[entry.line], why, ...extra })
const unchanged = (expected) => {
if (expected === undefined) return undefined
const current = byId.get(entryId(expected))
if (current?.file.file === expected.file.file && current.file.lines[current.line] === expected.file.lines[expected.line]) return current
return undefined
}
const pick = at => unchanged(numbered[Number(at) - 1])
const used = new Set()
for (const expected of ended) {
const entry = unchanged(expected)
if (entry === undefined) continue
take(entry)
retire(entry, 'ended')
used.add(entry)
counts.retired += 1
items.push({ op: 'retired', text: entry.text })
}
for (const change of changes) {
const op = String(change?.op ?? '')
if (op === 'about') {
const name = cleanTopic(change.topic)
const lines = draft.get(`${name}.md`)
const text = cleanEntry(change.text).slice(0, NOTE_MAX)
if (name === undefined || lines === undefined || text === '' || entryProblem(text) !== undefined) continue
const title = lines.findIndex(line => HEADING.test(line))
const kept = lines.filter(line => !/^>\s*\S/.test(line))
kept.splice(title + 1, 0, '', `> ${text}`)
draft.set(`${name}.md`, kept)
continue
}
if (op === 'merge') {
const entries = [...new Set((Array.isArray(change.entries) ? change.entries : []).map(pick))]
if (entries.some(entry => entry === undefined || used.has(entry))) continue
const text = cleanEntry(change.text)
if (entries.length < 2 || entries.length > budget.left || entryProblem(text) !== undefined) continue
budget.left -= entries.length
const [first, ...rest] = entries
const line = formatEntry(text, { ...first.meta, id: entryId(first), merged: entries.map(entryId).join(' ') })
draft.set(first.file.file, draft.get(first.file.file).map(item => (item === first.file.lines[first.line] ? line : item)))
rest.forEach(take)
entries.forEach(entry => { retire(entry, 'merged', { into: entryId(first) }); used.add(entry) })
counts.merged += entries.length
items.push({ op: 'merged', text })
continue
}
const entry = pick(change?.entry)
if (entry === undefined || used.has(entry) || budget.left < 1) continue
if (op === 'retire') {
budget.left -= 1
take(entry)
retire(entry, String(change.why ?? '').slice(0, NOTE_MAX) || 'retired')
used.add(entry)
counts.retired += 1
items.push({ op: 'retired', text: entry.text })
} else if (op === 'move' && entry.file.topic === undefined) {
const name = cleanTopic(change.topic)
if (name === undefined) continue
budget.left -= 1
const line = entry.file.lines[entry.line]
take(entry)
const base = draft.get(`${name}.md`) ?? parseMemory(`# ${name}\n`).lines
draft.set(`${name}.md`, insertEntry(base, line))
if (!parseMemory(joinLines(draft.get(MAIN))).links.includes(name)) draft.set(MAIN, addLink(draft.get(MAIN), name))
used.add(entry)
counts.moved += 1
items.push({ op: 'moved', text: entry.text, topic: name })
}
}
const dir = dirOf(job.scope)
const touched = []
const steps = []
const before = new Map(files.map(file => [file.file, file.content]))
for (const [file, lines] of draft) {
const emptied = file !== MAIN && lines.every(line => line.trim() === '' || HEADING.test(line) || /^>\s/.test(line))
const was = before.get(file) ?? null
const now = emptied ? null : lines.length === 0 ? '' : joinLines(lines)
if (was === now || (was === '' && lines.length === 0)) continue
touched.push({ file, before: was, after: now })
}
for (const item of touched.filter(entry => entry.after === null)) {
const name = item.file.slice(0, -3)
const lines = removeLink(draft.get(MAIN), name)
draft.set(MAIN, lines)
}
const mainNow = joinLines(draft.get(MAIN))
const mainItem = touched.find(item => item.file === MAIN)
if (mainItem !== undefined) mainItem.after = mainNow
else if (main.content !== mainNow && main.lines.length > 0) touched.push({ file: MAIN, before: main.content, after: mainNow })
const record = { time, day, ...(prior?.batch === undefined ? {} : { previous: prior.batch }) }
const stamp = { ...record, ...counts, ...(touched.length === 0 ? {} : { batch, items: items.slice(0, ITEMS_SHOWN) }) }
if (touched.length === 0) {
if (rt.readOnly() === undefined) await writeText(statePath(job.scope), JSON.stringify(stamp))
return
}
for (const item of touched) steps.push(item.after === null ? removeStep(join(dir, item.file)) : writeStep(join(dir, item.file), item.after))
const retiredPath = join(dir, RETIRED)
let log = ''
try { log = readFileSync(retiredPath, 'utf8') } catch { /* none yet */ }
if (retired.length > 0) steps.push(writeStep(retiredPath, `${log}${retired.map(row => JSON.stringify(row)).join('\n')}\n`))
steps.push(writeStep(batchPath(job.scope, batch), JSON.stringify({ batch, scope: key, files: touched, retiredBefore: log })))
steps.push(writeStep(statePath(job.scope), JSON.stringify(stamp)))
await publish({ op: 'tidy', scope: key, steps, row: { op: 'tidy', scope: key, batch, ...counts, actor: job.keeper.id, actorName: job.keeper.name, auto: true } })
}
// A batch comes back whole, or not at all: a file changed since is left as it reads.
async function undoTidy(scope, batch) {
const key = scopeKey(scope)
await queued(async () => {
await recoverPending()
writable(scope, USER)
const last = doneOf(scope)
if (batch === '' || last?.batch !== batch) throw new Error('That tidy is not the latest one of this memory.')
const saved = JSON.parse(readFileSync(batchPath(scope, batch), 'utf8'))
const dir = dirOf(scope)
const read = (file) => { try { return readFileSync(join(dir, file), 'utf8') } catch { return null } }
if (saved.files.some(item => read(item.file) !== item.after)) throw new Error('This memory has changed since that tidy, so it cannot be undone as a whole.')
const steps = saved.files.map(item => (item.before === null ? removeStep(join(dir, item.file)) : writeStep(join(dir, item.file), item.before)))
steps.push(saved.retiredBefore === '' ? removeStep(join(dir, RETIRED)) : writeStep(join(dir, RETIRED), saved.retiredBefore))
const { batch: _gone, items: _items, ...rest } = last
steps.push(writeStep(statePath(scope), JSON.stringify({ ...rest, undone: batch })))
await publish({ op: 'tidy-undo', scope: key, steps, row: { op: 'tidy-undo', scope: key, batch, actor: 'user', actorName: 'user' } })
})
}
const tidyView = (scopes) => {
const latest = Object.entries(scopes).map(([tag, scope]) => ({ tag, ...doneOf(scope) })).filter(item => item.batch !== undefined).sort((a, b) => String(b.time).localeCompare(String(a.time)))[0]
return latest === undefined ? null : { scope: latest.tag, batch: latest.batch, at: latest.time, merged: latest.merged, moved: latest.moved, retired: latest.retired, items: latest.items ?? [] }
}
// One memory at a time, and only one that changed since its last tidy, on a new day.
const due = (job) => {
const last = doneOf(job.scope)
return last?.day !== today() && changedSince(scopeKey(job.scope), last?.time)
}
let running = false
async function tick() {
if (running || rt.readOnly() !== undefined) return
running = true
try {
await rt.load()
const job = jobs().find(due)
if (job !== undefined) await tidy(job)
} catch (error) {
rt.warn('ds-bot: the memory tidy failed: %s', error?.message ?? error)
} finally {
running = false
}
}
if (config.memoryTidy !== false) {
ctx.effect(() => {
const timer = setInterval(() => void tick(), config.memoryTidyEveryMs ?? EVERY_MS)
timer.unref?.()
return () => clearInterval(timer)
}, 'ds-bot: memory tidy')
}
const tidyNow = async (key) => {
await rt.load()
const job = jobs().find(item => scopeKey(item.scope) === key)
if (job !== undefined && doneOf(job.scope)?.day !== today()) await tidy(job)
}
return { tidyView, undoTidy, tidyNow }
}