Import packages/plugin/src/codex/codex-image-codec.ts
This commit is contained in:
parent
9ed531f772
commit
44a2f70a90
|
|
@ -0,0 +1,287 @@
|
|||
type JsonRecord = Record<string, unknown>
|
||||
|
||||
const CODEX_IMAGE_MEDIA_TYPES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif'])
|
||||
const CODEX_IMAGE_ATTACHMENT_PREFIX = 'codex-image:'
|
||||
const MAX_CODEX_IMAGE_BASE64 = 288 * 1024 * 1024
|
||||
const DATA_IMAGE_URL = /^data:(image\/(?:png|jpeg|webp|gif));base64,([A-Za-z0-9+/]+={0,2})$/u
|
||||
|
||||
function string(value: unknown): string | undefined {
|
||||
return typeof value === 'string' ? value : undefined
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is JsonRecord {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function record(value: unknown): JsonRecord {
|
||||
return isRecord(value) ? value : {}
|
||||
}
|
||||
|
||||
export function toolOutputImageBlocks(item: JsonRecord): JsonRecord[] {
|
||||
const candidates = [
|
||||
record(item.result).content,
|
||||
record(item.output).content,
|
||||
item.contentItems,
|
||||
Array.isArray(item.output) ? item.output : undefined,
|
||||
]
|
||||
for (const candidate of candidates) {
|
||||
if (!Array.isArray(candidate)) continue
|
||||
const images = candidate.flatMap(value => {
|
||||
const block = record(value)
|
||||
return isImageContent(block) && parseImageBlock(block) !== undefined ? [block] : []
|
||||
})
|
||||
if (images.length > 0) return images
|
||||
}
|
||||
return []
|
||||
}
|
||||
|
||||
export function contentBlocks(value: unknown, attachmentSeed: string): JsonRecord[] {
|
||||
if (typeof value === 'string') return [{ type: 'text', text: value }]
|
||||
if (!Array.isArray(value)) return []
|
||||
const blocks: JsonRecord[] = []
|
||||
let imageIndex = 0
|
||||
for (const raw of value) {
|
||||
if (typeof raw === 'string') {
|
||||
blocks.push({ type: 'text', text: raw })
|
||||
continue
|
||||
}
|
||||
const block = record(raw)
|
||||
const image = imageContentBlock(block, attachmentSeed, imageIndex)
|
||||
if (image !== undefined) {
|
||||
imageIndex += 1
|
||||
blocks.push(image)
|
||||
continue
|
||||
}
|
||||
const text = string(block.text) ?? string(block.content)
|
||||
if (text !== undefined) blocks.push({ type: block.type === 'reasoning' ? 'reasoning' : 'text', text })
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
function imageContentBlock(block: JsonRecord, attachmentSeed: string, index: number): JsonRecord | undefined {
|
||||
if (!isImageContent(block)) return undefined
|
||||
const image = parseImageBlock(block)
|
||||
if (image === undefined) return undefined
|
||||
const dimensions = imageDimensions(image.mediaType, image.data)
|
||||
const name = string(block.name)
|
||||
const attachment = {
|
||||
attachmentId: `${CODEX_IMAGE_ATTACHMENT_PREFIX}${hashString(`${attachmentSeed}:${index}`)}:${index}`,
|
||||
mediaType: image.mediaType,
|
||||
bytes: base64ByteLength(image.data),
|
||||
width: dimensions.width,
|
||||
height: dimensions.height,
|
||||
...(name === undefined ? {} : { name }),
|
||||
}
|
||||
return {
|
||||
type: 'image',
|
||||
attachment,
|
||||
mediaType: image.mediaType,
|
||||
data: image.data,
|
||||
url: image.url,
|
||||
...(name === undefined ? {} : { name }),
|
||||
}
|
||||
}
|
||||
|
||||
function isImageContent(block: JsonRecord): boolean {
|
||||
return block.type === 'image' || block.type === 'input_image'
|
||||
}
|
||||
|
||||
function parseImageBlock(block: JsonRecord): { mediaType: string; data: string; url: string } | undefined {
|
||||
const url = string(block.url)
|
||||
?? string(block.image_url)
|
||||
?? string(record(block.image_url).url)
|
||||
const parsed = url === undefined ? undefined : parseDataImageUrl(url)
|
||||
if (parsed !== undefined) return parsed
|
||||
|
||||
const data = string(block.data)
|
||||
if (data === undefined || !isCanonicalBase64(data)) return undefined
|
||||
if (data.length > MAX_CODEX_IMAGE_BASE64) return undefined
|
||||
const mediaType = string(block.mediaType) ?? string(block.mimeType) ?? sniffImageMediaType(data)
|
||||
return mediaType !== undefined && CODEX_IMAGE_MEDIA_TYPES.has(mediaType)
|
||||
? { mediaType, data, url: `data:${mediaType};base64,${data}` }
|
||||
: undefined
|
||||
}
|
||||
|
||||
function parseDataImageUrl(value: string): { mediaType: string; data: string; url: string } | undefined {
|
||||
const match = DATA_IMAGE_URL.exec(value)
|
||||
if (match === null) return undefined
|
||||
const mediaType = match[1]!
|
||||
const data = match[2]!
|
||||
if (!CODEX_IMAGE_MEDIA_TYPES.has(mediaType) || !isCanonicalBase64(data)) return undefined
|
||||
return { mediaType, data, url: `data:${mediaType};base64,${data}` }
|
||||
}
|
||||
|
||||
export function collectImageBlocks(value: unknown, output: JsonRecord[] = []): JsonRecord[] {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) collectImageBlocks(item, output)
|
||||
return output
|
||||
}
|
||||
if (!isRecord(value)) return output
|
||||
if (value.type === 'image') output.push(value)
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (key === 'attachment') continue
|
||||
if (key === 'content' || key === 'message' || key === 'inserted' || key === 'chunk' || key === 'block') {
|
||||
collectImageBlocks(child, output)
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
function base64ByteLength(value: string): number {
|
||||
const padding = value.endsWith('==') ? 2 : value.endsWith('=') ? 1 : 0
|
||||
return Math.max(1, Math.floor(value.length * 3 / 4) - padding)
|
||||
}
|
||||
|
||||
function imageDimensions(mediaType: string, data: string): { width: number; height: number } {
|
||||
const bytes = base64PrefixBytes(data, 256 * 1024)
|
||||
const parsed = mediaType === 'image/png'
|
||||
? pngDimensions(bytes)
|
||||
: mediaType === 'image/jpeg'
|
||||
? jpegDimensions(bytes)
|
||||
: mediaType === 'image/gif'
|
||||
? gifDimensions(bytes)
|
||||
: mediaType === 'image/webp'
|
||||
? webpDimensions(bytes)
|
||||
: undefined
|
||||
return parsed ?? { width: 1, height: 1 }
|
||||
}
|
||||
|
||||
function pngDimensions(bytes: Uint8Array): { width: number; height: number } | undefined {
|
||||
if (bytes.length < 24
|
||||
|| bytes[0] !== 0x89
|
||||
|| bytes[1] !== 0x50
|
||||
|| bytes[2] !== 0x4e
|
||||
|| bytes[3] !== 0x47
|
||||
|| bytes[4] !== 0x0d
|
||||
|| bytes[5] !== 0x0a
|
||||
|| bytes[6] !== 0x1a
|
||||
|| bytes[7] !== 0x0a) return undefined
|
||||
return positiveDimensions(readUint32BE(bytes, 16), readUint32BE(bytes, 20))
|
||||
}
|
||||
|
||||
function jpegDimensions(bytes: Uint8Array): { width: number; height: number } | undefined {
|
||||
if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined
|
||||
let offset = 2
|
||||
while (offset + 9 < bytes.length) {
|
||||
if (bytes[offset] !== 0xff) {
|
||||
offset += 1
|
||||
continue
|
||||
}
|
||||
const marker = bytes[offset + 1]!
|
||||
offset += 2
|
||||
if (marker === 0xd8 || marker === 0xd9) continue
|
||||
if (offset + 2 > bytes.length) return undefined
|
||||
const length = readUint16BE(bytes, offset)
|
||||
if (length < 2 || offset + length > bytes.length) return undefined
|
||||
if (isJpegSof(marker) && length >= 7) {
|
||||
return positiveDimensions(readUint16BE(bytes, offset + 5), readUint16BE(bytes, offset + 3))
|
||||
}
|
||||
offset += length
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function gifDimensions(bytes: Uint8Array): { width: number; height: number } | undefined {
|
||||
if (bytes.length < 10
|
||||
|| bytes[0] !== 0x47
|
||||
|| bytes[1] !== 0x49
|
||||
|| bytes[2] !== 0x46
|
||||
|| bytes[3] !== 0x38
|
||||
|| (bytes[4] !== 0x37 && bytes[4] !== 0x39)
|
||||
|| bytes[5] !== 0x61) return undefined
|
||||
return positiveDimensions(readUint16LE(bytes, 6), readUint16LE(bytes, 8))
|
||||
}
|
||||
|
||||
function webpDimensions(bytes: Uint8Array): { width: number; height: number } | undefined {
|
||||
if (bytes.length < 30
|
||||
|| stringFromBytes(bytes, 0, 4) !== 'RIFF'
|
||||
|| stringFromBytes(bytes, 8, 12) !== 'WEBP') return undefined
|
||||
const chunk = stringFromBytes(bytes, 12, 16)
|
||||
if (chunk === 'VP8X') {
|
||||
return positiveDimensions(1 + readUint24LE(bytes, 24), 1 + readUint24LE(bytes, 27))
|
||||
}
|
||||
if (chunk === 'VP8L' && bytes[20] === 0x2f) {
|
||||
const b0 = bytes[21]!
|
||||
const b1 = bytes[22]!
|
||||
const b2 = bytes[23]!
|
||||
const b3 = bytes[24]!
|
||||
const width = 1 + b0 + ((b1 & 0x3f) << 8)
|
||||
const height = 1 + ((b1 & 0xc0) >> 6) + (b2 << 2) + ((b3 & 0x0f) << 10)
|
||||
return positiveDimensions(width, height)
|
||||
}
|
||||
if (chunk === 'VP8 ' && bytes[23] === 0x9d && bytes[24] === 0x01 && bytes[25] === 0x2a) {
|
||||
return positiveDimensions(readUint16LE(bytes, 26) & 0x3fff, readUint16LE(bytes, 28) & 0x3fff)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function sniffImageMediaType(data: string): string | undefined {
|
||||
const bytes = base64PrefixBytes(data, 32)
|
||||
if (pngDimensions(bytes) !== undefined) return 'image/png'
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg'
|
||||
if (gifDimensions(bytes) !== undefined) return 'image/gif'
|
||||
if (bytes.length >= 12 && stringFromBytes(bytes, 0, 4) === 'RIFF' && stringFromBytes(bytes, 8, 12) === 'WEBP') return 'image/webp'
|
||||
return undefined
|
||||
}
|
||||
|
||||
function base64PrefixBytes(value: string, maxBytes: number): Uint8Array {
|
||||
const chars = value.slice(0, Math.ceil(maxBytes / 3) * 4)
|
||||
let binary: string
|
||||
try {
|
||||
binary = atob(chars)
|
||||
} catch {
|
||||
return new Uint8Array()
|
||||
}
|
||||
const bytes = new Uint8Array(Math.min(binary.length, maxBytes))
|
||||
for (let index = 0; index < bytes.length; index += 1) bytes[index] = binary.charCodeAt(index)
|
||||
return bytes
|
||||
}
|
||||
|
||||
function readUint16BE(bytes: Uint8Array, offset: number): number {
|
||||
return (bytes[offset]! << 8) + bytes[offset + 1]!
|
||||
}
|
||||
|
||||
function readUint16LE(bytes: Uint8Array, offset: number): number {
|
||||
return bytes[offset]! + (bytes[offset + 1]! << 8)
|
||||
}
|
||||
|
||||
function readUint24LE(bytes: Uint8Array, offset: number): number {
|
||||
return bytes[offset]! + (bytes[offset + 1]! << 8) + (bytes[offset + 2]! << 16)
|
||||
}
|
||||
|
||||
function readUint32BE(bytes: Uint8Array, offset: number): number {
|
||||
return (bytes[offset]! * 0x1000000) + (bytes[offset + 1]! * 0x10000) + (bytes[offset + 2]! * 0x100) + bytes[offset + 3]!
|
||||
}
|
||||
|
||||
function positiveDimensions(width: number, height: number): { width: number; height: number } | undefined {
|
||||
return width > 0 && height > 0 ? { width, height } : undefined
|
||||
}
|
||||
|
||||
function isJpegSof(marker: number): boolean {
|
||||
return (marker >= 0xc0 && marker <= 0xc3)
|
||||
|| (marker >= 0xc5 && marker <= 0xc7)
|
||||
|| (marker >= 0xc9 && marker <= 0xcb)
|
||||
|| (marker >= 0xcd && marker <= 0xcf)
|
||||
}
|
||||
|
||||
function stringFromBytes(bytes: Uint8Array, start: number, end: number): string {
|
||||
if (bytes.length < end) return ''
|
||||
let output = ''
|
||||
for (let index = start; index < end; index += 1) output += String.fromCharCode(bytes[index]!)
|
||||
return output
|
||||
}
|
||||
|
||||
|
||||
export function isCanonicalBase64(value: string): boolean {
|
||||
return value.length >= 4 && value.length % 4 === 0
|
||||
&& /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value)
|
||||
}
|
||||
|
||||
function hashString(value: string): string {
|
||||
let hash = 2166136261
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
hash ^= value.charCodeAt(index)
|
||||
hash = Math.imul(hash, 16777619)
|
||||
}
|
||||
return (hash >>> 0).toString(16).padStart(8, '0')
|
||||
}
|
||||
Loading…
Reference in New Issue