dsh-sansheng-liubu/ds-bot/src/host/chat-index.js

294 lines
14 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// Reading a Bot's own chat: Session history pages, the SQLite FTS5 index, and the
// search, latest, and around reads behind read_own_chat.
//
// Chat index. read_own_chat searches a SQLite FTS5 table of every part of a Bot's own
// chat, ranked by BM25. The trigram tokenizer matches any run of three or more
// characters, so Chinese needs no word splitting; shorter words fall back to LIKE.
// The index is derived from the Session logs: a part catches up when it is read, an
// ended part is sealed once it is complete, and deleting the file only costs a
// rebuild. Without SQLite, read_own_chat scans the logs as before.
import { mkdir, unlink } from 'node:fs/promises'
import { join } from 'node:path'
import { ANCHOR_LINE_MAX, HISTORY_PAGE, HISTORY_PAGES_MAX, HISTORY_TIMEOUT_MS, OWN_CHAT_BRIEF, OWN_CHAT_LINE_MAX, OWN_CHAT_READ, OWN_CHAT_SEARCH } from './constants.js'
import { excerpt } from './text.js'
const INDEX_VERSION = 3
const AROUND_LINES = 6
const SEARCH_TERMS_MAX = 6
const ANY_TERM_HITS = 50
const INDEX_TIMEOUT_MS = 120_000
// A part still gets a few events (a title, a late reply) just after it ends.
const SEAL_AFTER_MS = 60_000
const likeOf = term => `%${term.replace(/[\\%_]/g, char => `\\${char}`)}%`
// Each term is a quoted FTS5 phrase when trigrams can match it, else a LIKE filter.
export function termQuery(terms, sessions, limit, hidden) {
const long = terms.filter(term => [...term].length >= 3)
const short = terms.filter(term => [...term].length < 3)
const where = ['session IN (SELECT value FROM json_each(?))']
const args = [JSON.stringify(sessions)]
if (hidden !== undefined) {
where.push('NOT (session = ? AND seq >= ?)')
args.push(hidden.session, hidden.from)
}
if (long.length > 0) {
where.unshift('lines MATCH ?')
args.unshift(long.map(term => `"${term.replace(/"/g, '""')}"`).join(' '))
}
for (const term of short) {
where.push("text LIKE ? ESCAPE '\\'")
args.push(likeOf(term))
}
return { sql: `SELECT text, session, seq, time, who FROM lines WHERE ${where.join(' AND ')} ORDER BY ${long.length > 0 ? 'rank' : 'time DESC'} LIMIT ${limit}`, args }
}
export function install(rt) {
const { ctx, home } = rt
const indexPath = join(home, 'chat-index.sqlite')
let chatIndex
let indexOpening
// A Session's events from newest to oldest, also for an archived part.
async function* eventsBackwards(sessionId, signal, maxPages = HISTORY_PAGES_MAX) {
const agent = await rt.resolveAgent(sessionId)
const throughSeq = Number(agent.session.seq) - 1
let beforeSeq
for (let pages = 0; throughSeq >= 0 && pages < maxPages; pages += 1) {
const page = await ctx.sessionController.page({
address: { kind: 'session', sessionId },
throughSeq,
...(beforeSeq === undefined ? {} : { beforeSeq }),
maxMessages: HISTORY_PAGE,
}, signal)
const events = page.records.map(record => record.event)
for (let index = events.length - 1; index >= 0; index -= 1) yield events[index]
if (!page.hasMore || events.length === 0) return
beforeSeq = events[0].seq
}
}
function openIndex() {
if (chatIndex !== undefined) return Promise.resolve(chatIndex)
const opening = indexOpening ??= (async () => {
let DatabaseSync
try {
({ DatabaseSync } = await import('node:sqlite'))
await mkdir(home, { recursive: true })
} catch (error) {
rt.warn('ds-bot: no chat index, read_own_chat scans the logs: %s', error)
return null
}
// A damaged file is rebuilt from the logs once.
for (const fresh of [false, true]) {
try {
if (fresh) await unlink(indexPath).catch(() => {})
return createIndex(new DatabaseSync(indexPath))
} catch (error) {
if (fresh) rt.warn('ds-bot: no chat index, read_own_chat scans the logs: %s', error)
}
}
return null
})().then((index) => {
// The plugin may have been disposed while the file opened.
if (indexOpening !== opening) {
index?.db.close()
return null
}
return chatIndex = index
})
return opening
}
function createIndex(db) {
try {
db.exec('PRAGMA journal_mode = WAL')
if (db.prepare('PRAGMA user_version').get().user_version !== INDEX_VERSION) {
db.exec(`
DROP TABLE IF EXISTS lines;
DROP TABLE IF EXISTS marks;
CREATE VIRTUAL TABLE lines USING fts5(text, session UNINDEXED, seq UNINDEXED, time UNINDEXED, who UNINDEXED, tokenize = 'trigram');
CREATE TABLE marks (session TEXT PRIMARY KEY, through INTEGER NOT NULL, keep INTEGER NOT NULL, sealed INTEGER NOT NULL, turn INTEGER NOT NULL);
PRAGMA user_version = ${INDEX_VERSION};`)
}
return {
db,
mark: db.prepare('SELECT through, keep, sealed, turn FROM marks WHERE session = ?'),
setMark: db.prepare('INSERT OR REPLACE INTO marks (session, through, keep, sealed, turn) VALUES (?, ?, ?, ?, ?)'),
insert: db.prepare('INSERT INTO lines (text, session, seq, time, who) VALUES (?, ?, ?, ?, ?)'),
before: db.prepare('SELECT text, session, seq, time, who FROM lines WHERE session = ? AND seq < ? ORDER BY seq DESC LIMIT ?'),
from: db.prepare('SELECT text, session, seq, time, who FROM lines WHERE session = ? AND seq >= ? ORDER BY seq LIMIT ?'),
}
} catch (error) {
db.close()
throw error
}
}
ctx.effect(() => () => {
chatIndex?.db.close()
chatIndex = undefined
indexOpening = undefined
}, 'ds-bot: chat index')
// Adds what a part said since its mark. Reads of one part wait for each other, so no
// line is added twice.
const catchingUp = new Map()
function catchUp(index, part) {
const running = catchingUp.get(part.sessionId)
if (running) return running
const job = (async () => {
const mark = index.mark.get(part.sessionId)
if (mark?.sealed) return
const through = mark?.through ?? -1
const events = []
for await (const event of eventsBackwards(part.sessionId, AbortSignal.timeout(INDEX_TIMEOUT_MS), Infinity)) {
if (event.seq <= through) break
events.push(event)
}
if (events.length > 0 && events.at(-1).seq !== through + 1) throw new Error(`the history of ${part.sessionId} came back with a gap`)
events.reverse()
if (index !== chatIndex) return
let keep = mark ? mark.keep === 1 : false
let turn = mark?.turn ?? -1
const seal = part.endedAt !== undefined && Date.now() - part.endedAt > SEAL_AFTER_MS
index.db.exec('BEGIN')
try {
let last = through
for (const event of events) {
last = event.seq
if (event.type === 'turn/start') turn = event.seq
const entry = rt.historyEntry(event, false)
if (entry === undefined) continue
if (entry.answer) {
if (keep) index.insert.run(entry.text, part.sessionId, event.seq, event.time ?? 0, 'you')
continue
}
keep = entry.keepAnswers
if (entry.text !== '') index.insert.run(entry.text, part.sessionId, event.seq, event.time ?? 0, entry.who)
}
index.setMark.run(part.sessionId, last, keep ? 1 : 0, seal ? 1 : 0, turn)
index.db.exec('COMMIT')
} catch (error) {
index.db.exec('ROLLBACK')
throw error
}
})().finally(() => catchingUp.delete(part.sessionId))
catchingUp.set(part.sessionId, job)
return job
}
// A message's anchor: its part (once there are several) and its seq in that part.
const anchorOf = (parts, row) => {
const part = parts.find(item => item.sessionId === row.session)
return `${parts.length > 1 ? `part ${part?.number ?? '?'} ` : ''}#${row.seq}`
}
const rowLine = (parts, row, terms = [], max = OWN_CHAT_LINE_MAX) => {
const lower = row.text.toLowerCase()
const wanted = terms.find(term => lower.includes(term)) ?? ''
const text = row.who === 'user' ? `User: ${row.text}` : row.who === 'you' ? `You: ${row.text}` : row.text
return `[${anchorOf(parts, row)} · ${rt.clock(row.time)}] ${excerpt(text, wanted, max)}`
}
// Every part, caught up. Asked from the current part, the turn that asks is left out
// (`hidden`): it is in context, and its own question would match.
async function indexedParts(botId, askingId) {
const index = await openIndex()
if (index === null) return { index }
const parts = rt.partsOf(botId)
// A part that cannot be read now is searched as far as it was indexed.
for (const part of parts) await catchUp(index, part).catch(error => rt.warn('ds-bot: indexing %s failed: %s', part.sessionId, error))
const turn = askingId !== undefined && askingId === parts.at(-1)?.sessionId ? index.mark.get(askingId)?.turn : undefined
return { index, parts, hidden: turn === undefined || turn < 0 ? undefined : { session: askingId, from: turn } }
}
// Messages with all the words, best first; with none of them having all, those with
// the most of them. Shown oldest first.
async function searchOwnChat(botId, query, { askingId, limit = OWN_CHAT_SEARCH } = {}) {
const terms = [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))].slice(0, SEARCH_TERMS_MAX)
const { index, parts, hidden } = await indexedParts(botId, askingId)
if (index === null) return { lines: await scanOwnChat(botId, query, limit), every: true }
const sessions = parts.map(part => part.sessionId)
const run = (wanted, max) => {
const { sql, args } = termQuery(wanted, sessions, max, hidden)
return index.db.prepare(sql).all(...args)
}
const byTime = (a, b) => (a.time - b.time) || (sessions.indexOf(a.session) - sessions.indexOf(b.session)) || (a.seq - b.seq)
let rows = run(terms, limit)
let every = true
if (rows.length === 0 && terms.length > 1) {
every = false
const found = new Map()
for (const term of terms) {
for (const row of run([term], ANY_TERM_HITS)) {
const key = `${row.session}#${row.seq}`
const hit = found.get(key) ?? { row, count: 0 }
hit.count += 1
found.set(key, hit)
}
}
rows = [...found.values()].sort((a, b) => (b.count - a.count) || (b.row.time - a.row.time)).slice(0, limit).map(hit => hit.row)
}
return { lines: rows.sort(byTime).map(row => rowLine(parts, row, terms)), every }
}
// The latest messages across all parts, oldest first.
async function latestOwnChat(botId, { askingId, limit = OWN_CHAT_READ } = {}) {
const { index, parts, hidden } = await indexedParts(botId, askingId)
if (index === null) return scanOwnChat(botId, '', limit)
const rows = []
for (const part of [...parts].reverse()) {
const below = hidden?.session === part.sessionId ? hidden.from : Number.MAX_SAFE_INTEGER
rows.push(...index.before.all(part.sessionId, below, limit - rows.length))
if (rows.length >= limit) break
}
return rows.reverse().map(row => rowLine(parts, row))
}
// The messages around an anchor, in its part.
async function aroundOwnChat(botId, anchor) {
const { index, parts } = await indexedParts(botId)
if (index === null) return { error: 'Anchors need the chat index, which is unavailable here. Search with words instead.' }
const found = /(?:part\s*)?(\d+)?\s*#\s*(\d+)/i.exec(anchor)
if (found === null) return { error: `"${anchor}" is not an anchor. Anchors look like "part 2 #1234", or "#1234" in the current part.` }
const number = found[1] === undefined ? parts.length : Number(found[1])
const part = parts[number - 1]
if (part === undefined) return { error: `Your own chat has no part ${number}.` }
const seq = Number(found[2])
const rows = [...index.before.all(part.sessionId, seq, AROUND_LINES).reverse(), ...index.from.all(part.sessionId, seq, AROUND_LINES + 1)]
return { part, exact: rows.some(row => row.seq === seq), lines: rows.map(row => rowLine(parts, row, [], row.seq === seq ? ANCHOR_LINE_MAX : OWN_CHAT_LINE_MAX)) }
}
// Without the index: what the user, other Bots, and the Bot itself said, across all
// parts, oldest first, by reading the logs.
async function scanOwnChat(botId, query, limit) {
const wanted = query.toLowerCase()
const parts = rt.partsOf(botId)
const signal = AbortSignal.timeout(HISTORY_TIMEOUT_MS)
const lines = []
for (const part of [...parts].reverse()) {
for await (const line of rt.partLines(part.sessionId, signal)) {
if (wanted !== '' && !line.text.toLowerCase().includes(wanted)) continue
lines.push(`[${parts.length > 1 ? `part ${part.number} · ` : ''}${rt.clock(line.time)}] ${excerpt(line.text, wanted)}`)
if (lines.length >= limit) break
}
if (lines.length >= limit) break
}
return lines.reverse()
}
// The parts of a Bot's own chat, for the model: when each ran.
const partIndex = parts => parts.map(part => `part ${part.number} (${rt.clock(part.startedAt)} – ${part.endedAt === undefined ? 'now' : rt.clock(part.endedAt)})`).join(', ')
async function ownChatBrief(member) {
let lines = []
try { lines = await latestOwnChat(member.id, { limit: OWN_CHAT_BRIEF }) } catch (error) { rt.warn('ds-bot: brief for %s failed: %s', member.name, error) }
return [
'Your own chat with the user so far (latest messages, oldest first; private, so share only what the group needs):',
lines.length > 0 ? lines.join('\n') : 'Nothing yet.',
'This brief does not change. Call read_own_chat when you need something newer or older from your own chat.',
].join('\n')
}
Object.assign(rt, { eventsBackwards, searchOwnChat, latestOwnChat, aroundOwnChat, partIndex, ownChatBrief })
}