feat(files): let the agent read HEIC photos

apps/sim/lib/copilot/vfs/file-reader.ts

@@ -14,7 +14,12 @@ import { recordFileRead } from '@/lib/copilot/request/metrics'
 import { markSpanForError } from '@/lib/copilot/request/otel'
 import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
 import { fetchWorkspaceFileBuffer } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
-import { isImageFileType } from '@/lib/uploads/utils/file-utils'
+import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic'
+import {
+ isImageFileType,
+ MODEL_SUPPORTED_IMAGE_MIME_TYPES,
+ resolveEffectiveMimeType,
+} from '@/lib/uploads/utils/file-utils'
 
 // Lazy tracer (same pattern as lib/copilot/request/otel.ts).
 function getVfsTracer() {
@@ -91,54 +96,82 @@ interface PreparedVisionImage {
  * dimension/quality chosen.
  */
 async function prepareImageForVision(
-buffer: Buffer,
+sourceBuffer: Buffer,
  claimedType: string
 ): Promise<PreparedVisionImage | null> {
  return getVfsTracer().startActiveSpan(
  TraceSpan.CopilotVfsPrepareImage,
  {
  attributes: {
- [TraceAttr.CopilotVfsInputBytes]: buffer.length,
+ [TraceAttr.CopilotVfsInputBytes]: sourceBuffer.length,
  [TraceAttr.CopilotVfsInputMediaTypeClaimed]: claimedType,
  },
  },
  async (span) => {
  try {
- const mediaType = detectImageMime(buffer, claimedType)
- span.setAttribute(TraceAttr.CopilotVfsInputMediaTypeDetected, mediaType)
+ const detectedType = detectImageMime(sourceBuffer, claimedType)
+ span.setAttribute(TraceAttr.CopilotVfsInputMediaTypeDetected, detectedType)
 
  let sharpModule: SharpConstructor
  try {
  sharpModule = (await import('sharp')).default
  } catch (err) {
  logger.warn('Failed to load sharp for image preparation', {
- mediaType,
+ mediaType: detectedType,
  error: toError(err).message,
  })
  span.setAttribute(TraceAttr.CopilotVfsSharpLoadFailed, true)
- const fitsWithoutSharp = buffer.length <= MAX_IMAGE_READ_BYTES
+ const fitsWithoutSharp =
+ MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(detectedType) &&
+ sourceBuffer.length <= MAX_IMAGE_READ_BYTES
  span.setAttribute(
  TraceAttr.CopilotVfsOutcome,
  fitsWithoutSharp ? 'passthrough_no_sharp' : 'rejected_no_sharp'
  )
- return fitsWithoutSharp ? { buffer, mediaType, resized: false } : null
+ return fitsWithoutSharp
+ ? { buffer: sourceBuffer, mediaType: detectedType, resized: false }
+ : null
  }
 
- let metadata: Awaited<ReturnType<ReturnType<typeof sharpModule>['metadata']>>
- try {
- metadata = await sharpModule(buffer, { limitInputPixels: false }).metadata()
- } catch (err) {
- logger.warn('Failed to read image metadata for VFS read', {
- mediaType,
- error: toError(err).message,
- })
+ const readMetadata = (candidate: Buffer) =>
+ sharpModule(candidate, { limitInputPixels: false })
+ .metadata()
+ .catch((err: unknown) => {
+ logger.warn('Failed to read image metadata for VFS read', {
+ mediaType: detectedType,
+ error: toError(err).message,
+ })
+ return null
+ })
+
+ // sharp first: its libvips reads everything we accept except HEVC-coded
+ // HEIF, and it is ~10x faster than the WASM decoder. Capability-based
+ // rather than brand-based, so AV1-coded `mif1` — which sharp handles
+ // natively — does not get sent down the slow path.
+ let buffer = sourceBuffer
+ let mediaType = detectedType
+ let metadata = await readMetadata(sourceBuffer)
+
+ if (!metadata && isHeifContainer(sourceBuffer)) {
+ const transcoded = await transcodeHeicToJpeg(sourceBuffer)
+ if (transcoded) {
+ buffer = transcoded
+ mediaType = 'image/jpeg'
+ metadata = await readMetadata(transcoded)
+ }
+ }
+
+ if (!metadata) {
  span.setAttribute(TraceAttr.CopilotVfsMetadataFailed, true)
- const fitsWithoutSharp = buffer.length <= MAX_IMAGE_READ_BYTES
+ // Bytes the model cannot decode are worse than no image: it describes
+ // them as empty rather than reporting them as broken.
+ const passthroughViable =
+ MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mediaType) && buffer.length <= MAX_IMAGE_READ_BYTES
  span.setAttribute(
  TraceAttr.CopilotVfsOutcome,
-fitsWithoutSharp ? 'passthrough_no_metadata' : 'rejected_no_metadata'
+passthroughViable ? 'passthrough_no_metadata' : 'rejected_no_metadata'
  )
- return fitsWithoutSharp ? { buffer, mediaType, resized: false } : null
+ return passthroughViable ? { buffer, mediaType, resized: false } : null
  }
 
  const width = metadata.width ?? 0
@@ -148,11 +181,15 @@ async function prepareImageForVision(
  [TraceAttr.CopilotVfsInputHeight]: height,
  })
 
- const needsResize =
+ // A format the model cannot decode has to be re-encoded even when it is
+ // already small enough — the ladder below emits JPEG or WebP, both of
+ // which it accepts.
+ const needsReencode =
+ !MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mediaType) ||
  buffer.length > MAX_IMAGE_READ_BYTES ||
  width > MAX_IMAGE_DIMENSION ||
  height > MAX_IMAGE_DIMENSION
- if (!needsResize) {
+ if (!needsReencode) {
  span.setAttributes({
  [TraceAttr.CopilotVfsResized]: false,
  [TraceAttr.CopilotVfsOutcome]: CopilotVfsOutcome.PassthroughFitsBudget,
@@ -300,14 +337,17 @@ export async function readFileRecord(record: WorkspaceFileRecord): Promise<FileR
  },
  async (span) => {
  try {
- if (isImageFileType(record.type)) {
+ // Resolve against the filename: a phone upload commonly stores as
+ // `application/octet-stream`, and matching the raw type would route a real
+ // image down the binary path where the model never sees it.
+ if (isImageFileType(resolveEffectiveMimeType(record.type, record.name))) {
  span.setAttribute(TraceAttr.CopilotVfsReadPath, CopilotVfsReadPath.Image)
  const originalBuffer = await fetchWorkspaceFileBuffer(record)
  const prepared = await prepareImageForVision(originalBuffer, record.type)
  if (!prepared) {
  span.setAttribute(TraceAttr.CopilotVfsReadOutcome, CopilotVfsReadOutcome.ImageTooLarge)
  return {
- content: `[Image too large: ${record.name} (${(record.size / 1024 / 1024).toFixed(1)}MB, limit 5MB after resize/compression)]`,
+ content: `[Image unavailable: ${record.name} (${(record.size / 1024 / 1024).toFixed(1)}MB). It could not be decoded, or still exceeded the 5MB vision limit after resizing.]`,
  totalLines: 1,
  }
  }

apps/sim/lib/uploads/server/heic.test.ts

@@ -0,0 +1,84 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it } from 'vitest'
+import { isHeifContainer, transcodeHeicToJpeg } from '@/lib/uploads/server/heic'
+
+/**
+ * An ISO-BMFF `ftyp` box: 4-byte size, the `ftyp` marker, the major brand, a
+ * 4-byte minor version, then any compatible brands.
+ */
+function ftypHeader(brand: string, compatible: string[] = []): Buffer {
+ const size = 16 + compatible.length * 4
+ const header = Buffer.alloc(size)
+ header.writeUInt32BE(size, 0)
+ header.write('ftyp', 4, 'ascii')
+ header.write(brand, 8, 'ascii')
+ compatible.forEach((entry, index) => header.write(entry, 16 + index * 4, 'ascii'))
+ return header
+}
+
+describe('isHeifContainer', () => {
+ it.each(['heic', 'heix', 'heim', 'heis', 'hevc', 'hevx', 'mif1', 'msf1'])(
+ 'detects the %s brand',
+ (brand) => {
+ expect(isHeifContainer(ftypHeader(brand))).toBe(true)
+ }
+ )
+
+ it.each(['avif', 'avis'])(
+ 'also claims the %s brand — the question is "is this HEIF", not "which codec"',
+ (brand) => {
+ expect(isHeifContainer(ftypHeader(brand))).toBe(true)
+ }
+ )
+
+ it('rejects other image formats', () => {
+ expect(isHeifContainer(Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0, 0, 0, 0, 0]))).toBe(
+ false
+ )
+ expect(isHeifContainer(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 0, 0, 0, 0, 0, 0, 0]))).toBe(
+ false
+ )
+ })
+
+ it('rejects a HEIF brand that is not behind an ftyp box', () => {
+ const riff = Buffer.alloc(16)
+ riff.write('RIFF', 0, 'ascii')
+ riff.write('heic', 8, 'ascii')
+ expect(isHeifContainer(riff)).toBe(false)
+ })
+
+ it('rejects an unknown brand in a well-formed ftyp box', () => {
+ expect(isHeifContainer(ftypHeader('qt '))).toBe(false)
+ })
+
+ it('detects a HEIF brand declared only among the compatible brands', () => {
+ // Standards-valid: a generic major brand with the HEIF brand listed after it.
+ expect(isHeifContainer(ftypHeader('isom', ['iso2', 'heic', 'mif1']))).toBe(true)
+ expect(isHeifContainer(ftypHeader('mp42', ['heix']))).toBe(true)
+ })
+
+ it('rejects a box whose compatible brands are all non-HEIF', () => {
+ expect(isHeifContainer(ftypHeader('isom', ['iso2', 'mp41', 'mp42']))).toBe(false)
+ })
+
+ it('does not read compatible brands past the declared box size', () => {
+ const truncated = ftypHeader('isom', ['heic'])
+ truncated.writeUInt32BE(16, 0)
+ expect(isHeifContainer(truncated)).toBe(false)
+ })
+
+ it('rejects buffers too short to carry a brand', () => {
+ expect(isHeifContainer(Buffer.alloc(0))).toBe(false)
+ expect(isHeifContainer(ftypHeader('heic').subarray(0, 11))).toBe(false)
+ })
+})
+
+describe('transcodeHeicToJpeg', () => {
+ it('returns null for bytes libheif cannot decode', async () => {
+ // Also proves the dynamic `heic-convert` import resolves at runtime, which no
+ // amount of type-checking establishes for a lazily loaded WebAssembly module.
+ expect(await transcodeHeicToJpeg(ftypHeader('heic'))).toBeNull()
+ })
+})

apps/sim/lib/uploads/server/heic.ts

@@ -0,0 +1,77 @@
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+
+const logger = createLogger('HeicTranscode')
+
+/**
+ * ISO-BMFF major brands in the HEIF family. The brand occupies bytes 8-11,
+ * immediately after the `ftyp` box marker at 4-7.
+ *
+ * The list is deliberately broad, `avif` included. It answers "are these bytes
+ * worth handing to a HEIF decoder", not "which codec is inside" — the brand cannot
+ * answer the latter anyway, since `mif1` is generic and carries either HEVC or AV1.
+ */
+const HEIF_BRANDS = new Set([
+ 'heic',
+ 'heix',
+ 'heim',
+ 'heis',
+ 'hevc',
+ 'hevx',
+ 'mif1',
+ 'msf1',
+ 'avif',
+ 'avis',
+])
+
+/**
+ * Whether these bytes are an ISO-BMFF container in the HEIF family.
+ *
+ * Sniffed rather than read off the declared type because the common case is a
+ * `.heic` stored as `application/octet-stream`, where the declared type says
+ * nothing at all.
+ */
+export function isHeifContainer(buffer: Buffer): boolean {
+ if (buffer.length < 12) return false
+ if (buffer.toString('ascii', 4, 8) !== 'ftyp') return false
+ if (HEIF_BRANDS.has(buffer.toString('ascii', 8, 12))) return true
+
+ // A standards-valid HEIF may carry a generic major brand such as `isom` and name
+ // the HEIF brand only among the compatible brands, which follow the 4-byte
+ // minor_version at offset 12 and run to the end of the box. A declared size of 0
+ // or 1 (the ISO-BMFF size escapes, which `ftyp` does not use) leaves `end` below
+ // the loop's start, so those simply do not scan.
+ const end = Math.min(buffer.readUInt32BE(0), buffer.length)
+ for (let offset = 16; offset + 4 <= end; offset += 4) {
+ if (HEIF_BRANDS.has(buffer.toString('ascii', offset, offset + 4))) return true
+ }
+ return false
+}
+
+/**
+ * Transcode a HEVC-coded HEIF still to JPEG.
+ *
+ * Two reasons, neither with a workaround: no vision model accepts HEIC (the Claude
+ * Messages API takes JPEG, PNG, GIF, and WebP only), and sharp's prebuilt libvips
+ * ships libheif with AV1 but not HEVC — it decodes AVIF and rejects an iPhone photo.
+ *
+ * Returns `null` when the bytes cannot be decoded; never a partial image.
+ */
+export async function transcodeHeicToJpeg(buffer: Buffer): Promise<Buffer | null> {
+ try {
+ const convert = (await import('heic-convert')).default
+ const jpeg = await convert({ buffer, format: 'JPEG' })
+ logger.info('Transcoded HEIC image', {
+ inputBytes: buffer.length,
+ outputBytes: jpeg.length,
+ })
+ return Buffer.from(jpeg)
+ } catch (error) {
+ logger.warn('Failed to transcode HEIC image', {
+ bytes: buffer.length,
+ brand: buffer.toString('ascii', 8, 12),
+ error: getErrorMessage(error),
+ })
+ return null
+ }
+}

apps/sim/package.json

@@ -168,6 +168,7 @@
  "google-auth-library": "10.5.0",
  "gray-matter": "^4.0.3",
  "groq-sdk": "^0.15.0",
+ "heic-convert": "2.1.0",
  "html-to-text": "^9.0.5",
  "http-proxy-agent": "7.0.2",
  "https-proxy-agent": "7.0.6",
@@ -248,6 +249,7 @@
  "@types/archiver": "8.0.0",
  "@types/busboy": "1.5.4",
  "@types/fluent-ffmpeg": "2.1.28",
+ "@types/heic-convert": "2.1.1",
  "@types/html-to-text": "9.0.4",
  "@types/js-yaml": "4.0.9",
  "@types/jsdom": "21.1.7",