228 lines
9.7 KiB
JavaScript
228 lines
9.7 KiB
JavaScript
// Images for text-only models. Every model may take images; before a Session request
|
|
// reaches a model that cannot read them, each image becomes text from a vision model.
|
|
// Texts are kept per attachment, so later requests repeat the same words and the
|
|
// prompt cache still hits.
|
|
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
import { join } from 'node:path'
|
|
import { BlockAssembler } from '@deepseek-ai/dsh-llm'
|
|
import { OCR_CACHE_MAX, OCR_PROMPT, OCR_RETRY_MS, OCR_TIMEOUT_MS } from './constants.js'
|
|
import { replaceFile } from './files.js'
|
|
import { splitRef } from './text.js'
|
|
|
|
export function offloadTargets(session, beforeSeq) {
|
|
const targets = []
|
|
for (const seq of session.surface.nodes) {
|
|
if (seq >= beforeSeq) continue
|
|
const event = session.eventAt(seq)
|
|
if (event?.type !== 'user/message' && event?.type !== 'tool/result') continue
|
|
const message = session.deriveEventMessage(event)
|
|
const imageIndexes = []
|
|
let imageIndex = 0
|
|
for (const block of message?.content ?? []) {
|
|
if (block?.type !== 'image') continue
|
|
if (block.offloaded !== true) imageIndexes.push(imageIndex)
|
|
imageIndex += 1
|
|
}
|
|
if (imageIndexes.length > 0) targets.push({ seq, imageIndexes })
|
|
}
|
|
return targets
|
|
}
|
|
|
|
export function install(rt) {
|
|
const { ctx, config, home } = rt
|
|
|
|
// `textOnlyModels` names models whose provider claims image input but rejects it.
|
|
const textOnlyModels = new Set(config.textOnlyModels ?? [])
|
|
const ocrPath = join(home, 'ocr.json')
|
|
let ocrCache = null
|
|
let ocrWriting = Promise.resolve()
|
|
const ocrPending = new Map()
|
|
const ocrFailed = new Map()
|
|
|
|
const liveImages = message => Array.isArray(message?.content)
|
|
&& message.content.some(block => block?.type === 'image' && !block.offloaded)
|
|
|
|
async function readsImages(provider, model) {
|
|
if (textOnlyModels.has(`${provider}/${model}`)) return false
|
|
try {
|
|
const info = await ctx.llm.resolveModelInfo(provider, model)
|
|
return info.inputModalities === undefined || info.inputModalities.includes('image')
|
|
} catch {
|
|
return true
|
|
}
|
|
}
|
|
|
|
// The configured `ocrModel` first, then served models that declare image input, the
|
|
// DSH default among them first. A provider with no key set is skipped.
|
|
async function ocrRoute() {
|
|
const models = (await rt.catalog()).filter(entry => entry.needsKey !== true && rt.isLocalAgentRef?.(entry.ref) !== true)
|
|
const served = new Set(models.map(entry => entry.ref))
|
|
const declared = models.filter(entry => entry.input?.includes('image')).map(entry => entry.ref)
|
|
const fallback = rt.defaultRef()
|
|
for (const ref of new Set([config.ocrModel, ...declared.filter(ref => ref === fallback), ...declared])) {
|
|
const route = splitRef(ref)
|
|
if (route && served.has(ref) && await readsImages(route.provider, route.model)) return { ref, ...route }
|
|
}
|
|
return undefined
|
|
}
|
|
|
|
async function loadOcr() {
|
|
if (ocrCache !== null) return
|
|
try {
|
|
ocrCache = new Map(Object.entries(JSON.parse(await readFile(ocrPath, 'utf8'))))
|
|
} catch (error) {
|
|
if (error?.code !== 'ENOENT') rt.warn('ds-bot: unreadable OCR cache %s: %s', ocrPath, error)
|
|
ocrCache = new Map()
|
|
}
|
|
}
|
|
|
|
function saveOcr() {
|
|
while (ocrCache.size > OCR_CACHE_MAX) ocrCache.delete(ocrCache.keys().next().value)
|
|
const snapshot = JSON.stringify(Object.fromEntries(ocrCache))
|
|
ocrWriting = ocrWriting.then(async () => {
|
|
await mkdir(home, { recursive: true })
|
|
const temp = `${ocrPath}.${process.pid}.tmp`
|
|
await writeFile(temp, snapshot, 'utf8')
|
|
await replaceFile(temp, ocrPath)
|
|
}).catch(error => rt.warn('ds-bot: OCR cache write failed: %s', error))
|
|
}
|
|
|
|
// `sessionId` is the Session the reading is for; its usage counts there.
|
|
async function readImage(attachment, sessionId) {
|
|
const route = await ocrRoute()
|
|
if (route === undefined) throw new Error('no vision model is available')
|
|
const assembler = new BlockAssembler()
|
|
for await (const chunk of ctx.llm.stream({
|
|
provider: route.provider,
|
|
model: route.model,
|
|
sessionId,
|
|
messages: rt.tagUsage([{ role: 'user', content: [{ type: 'image', attachment }, { type: 'text', text: OCR_PROMPT }] }], { sessionId, kind: 'image' }),
|
|
maxTokens: 8192,
|
|
signal: AbortSignal.timeout(OCR_TIMEOUT_MS),
|
|
})) assembler.push(chunk)
|
|
const finish = assembler.finish
|
|
if (finish.kind === 'error' || finish.kind === 'aborted') throw new Error(finish.failure?.message ?? finish.kind)
|
|
const text = assembler.blocks().filter(block => block.type === 'text').map(block => block.text).join('\n').trim()
|
|
if (text === '') throw new Error('the vision model returned no text')
|
|
const model = (await rt.catalog()).find(entry => entry.ref === route.ref)?.name ?? route.model
|
|
return { text, model }
|
|
}
|
|
|
|
// Resolves to the cached reading, or to null when the image cannot be read now.
|
|
async function imageReading(attachment, sessionId) {
|
|
const id = String(attachment.attachmentId)
|
|
if (ocrCache.has(id)) return ocrCache.get(id)
|
|
const failed = ocrFailed.get(id)
|
|
if (failed && Date.now() - failed.at < OCR_RETRY_MS) return null
|
|
if (!ocrPending.has(id)) {
|
|
ocrPending.set(id, readImage(attachment, sessionId).then((reading) => {
|
|
ocrCache.set(id, reading)
|
|
ocrFailed.delete(id)
|
|
saveOcr()
|
|
return reading
|
|
}, (error) => {
|
|
rt.warn('ds-bot: could not read image %s: %s', id, error)
|
|
ocrFailed.set(id, { at: Date.now() })
|
|
return null
|
|
}).finally(() => ocrPending.delete(id)))
|
|
}
|
|
return ocrPending.get(id)
|
|
}
|
|
|
|
const imageLabel = attachment => (attachment.name
|
|
? `"${attachment.name}"`
|
|
: `sha256:${String(attachment.attachmentId).slice('sha256:'.length, 'sha256:'.length + 8)}`)
|
|
|
|
function imagePath(attachment) {
|
|
try {
|
|
const hostPath = ctx.get?.('attachments')?.imageHostPath?.(attachment)
|
|
if (hostPath === undefined) return undefined
|
|
return ctx.get?.('fs')?.processPathFromHostPath?.(hostPath) ?? hostPath
|
|
} catch {
|
|
return undefined
|
|
}
|
|
}
|
|
|
|
function imageAsText(attachment, reading) {
|
|
const label = imageLabel(attachment)
|
|
const path = imagePath(attachment)
|
|
const saved = path === undefined ? '' : ` Read-only copy saved at ${JSON.stringify(path)}.`
|
|
return {
|
|
type: 'text',
|
|
text: reading === null
|
|
? `[Image ${label}: you cannot see images and it could not be read right now.${saved} Ask the user what it shows if it matters.]`
|
|
: `[Image ${label}.${saved} You cannot see images, so ${reading.model} read it for you:]\n${reading.text}\n[End of image ${label}]`,
|
|
}
|
|
}
|
|
|
|
// Members read the group as text, so a picture posted there reaches every member,
|
|
// vision model or not, as its reading; the image block itself also rides with the
|
|
// prompt for members whose model or tools can inspect it directly.
|
|
async function imagesForGroup(attachments, sessionId) {
|
|
await loadOcr()
|
|
return Promise.all(attachments.map(async (attachment) => {
|
|
const reading = await imageReading(attachment, sessionId)
|
|
const label = imageLabel(attachment)
|
|
const path = imagePath(attachment)
|
|
const saved = path === undefined ? '' : ` (read-only copy saved at ${JSON.stringify(path)})`
|
|
return reading === null
|
|
? `[Image ${label}${saved}: it could not be read right now. Ask the user what it shows if it matters.]`
|
|
: `[Image ${label}${saved}, read by ${reading.model}:]\n${reading.text}\n[End of image ${label}]`
|
|
}))
|
|
}
|
|
|
|
async function withImagesAsText(messages, sessionId) {
|
|
await loadOcr()
|
|
const images = new Map()
|
|
for (const message of messages) {
|
|
if (!liveImages(message)) continue
|
|
for (const block of message.content) {
|
|
if (block?.type === 'image' && !block.offloaded) images.set(String(block.attachment.attachmentId), block.attachment)
|
|
}
|
|
}
|
|
const readings = new Map(await Promise.all([...images].map(async ([id, attachment]) => [id, await imageReading(attachment, sessionId)])))
|
|
return messages.map(message => (liveImages(message)
|
|
? {
|
|
...message,
|
|
content: message.content.map(block => (block?.type === 'image' && !block.offloaded
|
|
? imageAsText(block.attachment, readings.get(String(block.attachment.attachmentId)))
|
|
: block)),
|
|
}
|
|
: message))
|
|
}
|
|
|
|
ctx.on('llm/stream', (options, next) => {
|
|
if (options.sessionId === undefined || rt.ownRequest(options) || !(options.messages ?? []).some(liveImages)) return next()
|
|
return (async function* imagesAsTextOrPass() {
|
|
await rt.load()
|
|
if (rt.roomOf(options.sessionId) !== undefined || await readsImages(options.provider, options.model)) {
|
|
yield* next()
|
|
return
|
|
}
|
|
// `next` cannot take new messages, so the rewritten request starts over; it
|
|
// carries no image, so this listener lets it through.
|
|
yield* ctx.llm.stream({ ...options, messages: await withImagesAsText(options.messages, options.sessionId) })
|
|
})()
|
|
})
|
|
|
|
ctx.on('agent/status', ({ agent, status }) => {
|
|
if (status !== 'idle' || !agent?.session || !rt.isBotSession(agent.id) || rt.readOnly() !== undefined) return
|
|
const beforeSeq = agent.session.seq
|
|
setImmediate(() => {
|
|
void (async () => {
|
|
try {
|
|
if (!ctx.sessions.messageProjections.some(projection => projection.type === 'image/offload')) return
|
|
const targets = offloadTargets(agent.session, beforeSeq)
|
|
if (targets.length === 0) return
|
|
agent.session.append('image/offload', { targets })
|
|
await ctx.sessions.flush(agent.session)
|
|
} catch (error) {
|
|
rt.warn('ds-bot: old images for %s not offloaded: %s', agent.id, error)
|
|
}
|
|
})()
|
|
})
|
|
})
|
|
|
|
Object.assign(rt, { imagesForGroup })
|
|
}
|