diff --git a/packages/plugin/src/codex/codex-image-codec.ts b/packages/plugin/src/codex/codex-image-codec.ts new file mode 100644 index 0000000..d5e9e18 --- /dev/null +++ b/packages/plugin/src/codex/codex-image-codec.ts @@ -0,0 +1,287 @@ +type JsonRecord = Record + +const CODEX_IMAGE_MEDIA_TYPES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif']) +const CODEX_IMAGE_ATTACHMENT_PREFIX = 'codex-image:' +const MAX_CODEX_IMAGE_BASE64 = 288 * 1024 * 1024 +const DATA_IMAGE_URL = /^data:(image\/(?:png|jpeg|webp|gif));base64,([A-Za-z0-9+/]+={0,2})$/u + +function string(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined +} + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function record(value: unknown): JsonRecord { + return isRecord(value) ? value : {} +} + +export function toolOutputImageBlocks(item: JsonRecord): JsonRecord[] { + const candidates = [ + record(item.result).content, + record(item.output).content, + item.contentItems, + Array.isArray(item.output) ? item.output : undefined, + ] + for (const candidate of candidates) { + if (!Array.isArray(candidate)) continue + const images = candidate.flatMap(value => { + const block = record(value) + return isImageContent(block) && parseImageBlock(block) !== undefined ? [block] : [] + }) + if (images.length > 0) return images + } + return [] +} + +export function contentBlocks(value: unknown, attachmentSeed: string): JsonRecord[] { + if (typeof value === 'string') return [{ type: 'text', text: value }] + if (!Array.isArray(value)) return [] + const blocks: JsonRecord[] = [] + let imageIndex = 0 + for (const raw of value) { + if (typeof raw === 'string') { + blocks.push({ type: 'text', text: raw }) + continue + } + const block = record(raw) + const image = imageContentBlock(block, attachmentSeed, imageIndex) + if (image !== undefined) { + imageIndex += 1 + blocks.push(image) + continue + } + const text = string(block.text) ?? string(block.content) + if (text !== undefined) blocks.push({ type: block.type === 'reasoning' ? 'reasoning' : 'text', text }) + } + return blocks +} + +function imageContentBlock(block: JsonRecord, attachmentSeed: string, index: number): JsonRecord | undefined { + if (!isImageContent(block)) return undefined + const image = parseImageBlock(block) + if (image === undefined) return undefined + const dimensions = imageDimensions(image.mediaType, image.data) + const name = string(block.name) + const attachment = { + attachmentId: `${CODEX_IMAGE_ATTACHMENT_PREFIX}${hashString(`${attachmentSeed}:${index}`)}:${index}`, + mediaType: image.mediaType, + bytes: base64ByteLength(image.data), + width: dimensions.width, + height: dimensions.height, + ...(name === undefined ? {} : { name }), + } + return { + type: 'image', + attachment, + mediaType: image.mediaType, + data: image.data, + url: image.url, + ...(name === undefined ? {} : { name }), + } +} + +function isImageContent(block: JsonRecord): boolean { + return block.type === 'image' || block.type === 'input_image' +} + +function parseImageBlock(block: JsonRecord): { mediaType: string; data: string; url: string } | undefined { + const url = string(block.url) + ?? string(block.image_url) + ?? string(record(block.image_url).url) + const parsed = url === undefined ? undefined : parseDataImageUrl(url) + if (parsed !== undefined) return parsed + + const data = string(block.data) + if (data === undefined || !isCanonicalBase64(data)) return undefined + if (data.length > MAX_CODEX_IMAGE_BASE64) return undefined + const mediaType = string(block.mediaType) ?? string(block.mimeType) ?? sniffImageMediaType(data) + return mediaType !== undefined && CODEX_IMAGE_MEDIA_TYPES.has(mediaType) + ? { mediaType, data, url: `data:${mediaType};base64,${data}` } + : undefined +} + +function parseDataImageUrl(value: string): { mediaType: string; data: string; url: string } | undefined { + const match = DATA_IMAGE_URL.exec(value) + if (match === null) return undefined + const mediaType = match[1]! + const data = match[2]! + if (!CODEX_IMAGE_MEDIA_TYPES.has(mediaType) || !isCanonicalBase64(data)) return undefined + return { mediaType, data, url: `data:${mediaType};base64,${data}` } +} + +export function collectImageBlocks(value: unknown, output: JsonRecord[] = []): JsonRecord[] { + if (Array.isArray(value)) { + for (const item of value) collectImageBlocks(item, output) + return output + } + if (!isRecord(value)) return output + if (value.type === 'image') output.push(value) + for (const [key, child] of Object.entries(value)) { + if (key === 'attachment') continue + if (key === 'content' || key === 'message' || key === 'inserted' || key === 'chunk' || key === 'block') { + collectImageBlocks(child, output) + } + } + return output +} + +function base64ByteLength(value: string): number { + const padding = value.endsWith('==') ? 2 : value.endsWith('=') ? 1 : 0 + return Math.max(1, Math.floor(value.length * 3 / 4) - padding) +} + +function imageDimensions(mediaType: string, data: string): { width: number; height: number } { + const bytes = base64PrefixBytes(data, 256 * 1024) + const parsed = mediaType === 'image/png' + ? pngDimensions(bytes) + : mediaType === 'image/jpeg' + ? jpegDimensions(bytes) + : mediaType === 'image/gif' + ? gifDimensions(bytes) + : mediaType === 'image/webp' + ? webpDimensions(bytes) + : undefined + return parsed ?? { width: 1, height: 1 } +} + +function pngDimensions(bytes: Uint8Array): { width: number; height: number } | undefined { + if (bytes.length < 24 + || bytes[0] !== 0x89 + || bytes[1] !== 0x50 + || bytes[2] !== 0x4e + || bytes[3] !== 0x47 + || bytes[4] !== 0x0d + || bytes[5] !== 0x0a + || bytes[6] !== 0x1a + || bytes[7] !== 0x0a) return undefined + return positiveDimensions(readUint32BE(bytes, 16), readUint32BE(bytes, 20)) +} + +function jpegDimensions(bytes: Uint8Array): { width: number; height: number } | undefined { + if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined + let offset = 2 + while (offset + 9 < bytes.length) { + if (bytes[offset] !== 0xff) { + offset += 1 + continue + } + const marker = bytes[offset + 1]! + offset += 2 + if (marker === 0xd8 || marker === 0xd9) continue + if (offset + 2 > bytes.length) return undefined + const length = readUint16BE(bytes, offset) + if (length < 2 || offset + length > bytes.length) return undefined + if (isJpegSof(marker) && length >= 7) { + return positiveDimensions(readUint16BE(bytes, offset + 5), readUint16BE(bytes, offset + 3)) + } + offset += length + } + return undefined +} + +function gifDimensions(bytes: Uint8Array): { width: number; height: number } | undefined { + if (bytes.length < 10 + || bytes[0] !== 0x47 + || bytes[1] !== 0x49 + || bytes[2] !== 0x46 + || bytes[3] !== 0x38 + || (bytes[4] !== 0x37 && bytes[4] !== 0x39) + || bytes[5] !== 0x61) return undefined + return positiveDimensions(readUint16LE(bytes, 6), readUint16LE(bytes, 8)) +} + +function webpDimensions(bytes: Uint8Array): { width: number; height: number } | undefined { + if (bytes.length < 30 + || stringFromBytes(bytes, 0, 4) !== 'RIFF' + || stringFromBytes(bytes, 8, 12) !== 'WEBP') return undefined + const chunk = stringFromBytes(bytes, 12, 16) + if (chunk === 'VP8X') { + return positiveDimensions(1 + readUint24LE(bytes, 24), 1 + readUint24LE(bytes, 27)) + } + if (chunk === 'VP8L' && bytes[20] === 0x2f) { + const b0 = bytes[21]! + const b1 = bytes[22]! + const b2 = bytes[23]! + const b3 = bytes[24]! + const width = 1 + b0 + ((b1 & 0x3f) << 8) + const height = 1 + ((b1 & 0xc0) >> 6) + (b2 << 2) + ((b3 & 0x0f) << 10) + return positiveDimensions(width, height) + } + if (chunk === 'VP8 ' && bytes[23] === 0x9d && bytes[24] === 0x01 && bytes[25] === 0x2a) { + return positiveDimensions(readUint16LE(bytes, 26) & 0x3fff, readUint16LE(bytes, 28) & 0x3fff) + } + return undefined +} + +function sniffImageMediaType(data: string): string | undefined { + const bytes = base64PrefixBytes(data, 32) + if (pngDimensions(bytes) !== undefined) return 'image/png' + if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg' + if (gifDimensions(bytes) !== undefined) return 'image/gif' + if (bytes.length >= 12 && stringFromBytes(bytes, 0, 4) === 'RIFF' && stringFromBytes(bytes, 8, 12) === 'WEBP') return 'image/webp' + return undefined +} + +function base64PrefixBytes(value: string, maxBytes: number): Uint8Array { + const chars = value.slice(0, Math.ceil(maxBytes / 3) * 4) + let binary: string + try { + binary = atob(chars) + } catch { + return new Uint8Array() + } + const bytes = new Uint8Array(Math.min(binary.length, maxBytes)) + for (let index = 0; index < bytes.length; index += 1) bytes[index] = binary.charCodeAt(index) + return bytes +} + +function readUint16BE(bytes: Uint8Array, offset: number): number { + return (bytes[offset]! << 8) + bytes[offset + 1]! +} + +function readUint16LE(bytes: Uint8Array, offset: number): number { + return bytes[offset]! + (bytes[offset + 1]! << 8) +} + +function readUint24LE(bytes: Uint8Array, offset: number): number { + return bytes[offset]! + (bytes[offset + 1]! << 8) + (bytes[offset + 2]! << 16) +} + +function readUint32BE(bytes: Uint8Array, offset: number): number { + return (bytes[offset]! * 0x1000000) + (bytes[offset + 1]! * 0x10000) + (bytes[offset + 2]! * 0x100) + bytes[offset + 3]! +} + +function positiveDimensions(width: number, height: number): { width: number; height: number } | undefined { + return width > 0 && height > 0 ? { width, height } : undefined +} + +function isJpegSof(marker: number): boolean { + return (marker >= 0xc0 && marker <= 0xc3) + || (marker >= 0xc5 && marker <= 0xc7) + || (marker >= 0xc9 && marker <= 0xcb) + || (marker >= 0xcd && marker <= 0xcf) +} + +function stringFromBytes(bytes: Uint8Array, start: number, end: number): string { + if (bytes.length < end) return '' + let output = '' + for (let index = start; index < end; index += 1) output += String.fromCharCode(bytes[index]!) + return output +} + + +export function isCanonicalBase64(value: string): boolean { + return value.length >= 4 && value.length % 4 === 0 + && /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(value) +} + +function hashString(value: string): string { + let hash = 2166136261 + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index) + hash = Math.imul(hash, 16777619) + } + return (hash >>> 0).toString(16).padStart(8, '0') +}