diff --git a/.changeset/empty-image-session-poisoning.md b/.changeset/empty-image-session-poisoning.md new file mode 100644 index 0000000000..017a7a443f --- /dev/null +++ b/.changeset/empty-image-session-poisoning.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix pasted images intermittently failing to reach the model: a zero-byte image from a failed clipboard read poisoned the session, so every later image was dropped with an ambiguous placeholder and the model could hallucinate having seen it. Prompts carrying an empty image are now rejected with a clear error so you can re-paste before anything is sent, sessions already affected recover automatically, and the placeholder tells the model the attachment was removed and must not be guessed at. diff --git a/packages/agent-core-v2/src/agent/contextProjector/mediaProjection.ts b/packages/agent-core-v2/src/agent/contextProjector/mediaProjection.ts index 741465ce9b..4c4c027745 100644 --- a/packages/agent-core-v2/src/agent/contextProjector/mediaProjection.ts +++ b/packages/agent-core-v2/src/agent/contextProjector/mediaProjection.ts @@ -8,20 +8,20 @@ export const MEDIA_DEGRADE_KEEP_RECENT = 2; const MEDIA_DEGRADED_PLACEHOLDERS = { image_url: - '[image omitted: dropped to fit the provider request size limit; re-read the file to view it]', + '[An image attached to an earlier message was removed to fit the provider request size limit. You have NOT seen this image — do not describe or guess its contents. If it matters, ask the user to re-send it or to point you at the file so you can read it with ReadMediaFile.]', audio_url: - '[audio omitted: dropped to fit the provider request size limit; re-read the file to hear it]', + '[An audio clip attached to an earlier message was removed to fit the provider request size limit. You have NOT heard it — do not describe or guess its contents.]', video_url: - '[video omitted: dropped to fit the provider request size limit; re-read the file to view it]', + '[A video attached to an earlier message was removed to fit the provider request size limit. You have NOT seen it — do not describe or guess its contents.]', } as const; export const MEDIA_STRIPPED_PLACEHOLDERS = { image_url: - '[image omitted for provider compatibility; re-read the file to view it or get conversion guidance]', + '[An image attached to this message was removed before sending because the provider could not accept it (unsupported or unreadable image data). You have NOT seen this image — do not describe or guess its contents. Tell the user the image failed to reach you and suggest re-sending it as PNG or JPEG.]', audio_url: - '[audio omitted for provider compatibility; re-read the file to hear it]', + '[An audio clip attached to this message was removed before sending because the provider could not accept it. You have NOT heard it — do not describe or guess its contents.]', video_url: - '[video omitted for provider compatibility; re-read the file to view it]', + '[A video attached to this message was removed before sending because the provider could not accept it. You have NOT seen it — do not describe or guess its contents.]', } as const; type MediaPlaceholderSet = typeof MEDIA_DEGRADED_PLACEHOLDERS | typeof MEDIA_STRIPPED_PLACEHOLDERS; diff --git a/packages/agent-core-v2/src/agent/contextProjector/projection.ts b/packages/agent-core-v2/src/agent/contextProjector/projection.ts index c4cd5f726c..9cb99891bc 100644 --- a/packages/agent-core-v2/src/agent/contextProjector/projection.ts +++ b/packages/agent-core-v2/src/agent/contextProjector/projection.ts @@ -2,6 +2,7 @@ import { ErrorCodes, Error2 } from '#/errors'; import { renderToolResultForModel } from '#/agent/contextMemory/toolResultRender'; import type { ContextMessage } from '#/agent/contextMemory/types'; import { isVacuousContentPart } from '#/agent/contextMemory/vacuousContent'; +import { gateImageFormatParts } from '#/agent/media/image-format-policy'; import type { ContentPart, Message } from '#/kosong/contract/message'; export type ProjectionAnomaly = @@ -344,7 +345,11 @@ function projectedContent(source: ContextMessage, onAnomaly?: OnAnomaly): Conten note: source.note, }) : source.content; - return cleanContent(source, content, onAnomaly); + // The image format gate runs at projection time too (not only at ingestion): + // a malformed/undeliverable image already persisted in an old session's + // history is replaced by a text notice on the wire, so it can no longer + // poison every later request. Read-side only — the history keeps its parts. + return cleanContent(source, gateImageFormatParts(content), onAnomaly); } function cleanContent( diff --git a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts index 45dc08dbe6..a78f34c203 100644 --- a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts +++ b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts @@ -461,6 +461,19 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { if (signal?.aborted === true) return undefined; const raw = unwrapErrorCause(error); const media = policy?.media; + // Strip is the last-resort recovery: when it fires, the provider never saw + // any of the images, so the warn carries the rejection's status/message + // (the classification chain swallows both). + const reportMediaStripped = (message: string): void => { + const statusCode = raw instanceof APIStatusError ? raw.statusCode : undefined; + const errorMessage = (raw instanceof Error ? raw.message : String(raw)).slice(0, 300); + this.log.warn(message, { + model: request.model.name, + statusCode, + errorMessage, + ...request.logFields, + }); + }; if ( raw instanceof APIRequestTooLargeError && (media === undefined || media === 'degraded') @@ -474,23 +487,15 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { this.markRecoveryTurn(this.mediaDegradedTurns, request.source); return { ...policy, media: 'degraded' }; } - this.log.warn( + reportMediaStripped( 'provider rejected degraded-media request as too large; resending with rejected media stripped', - { - model: request.model.name, - ...request.logFields, - }, ); return { ...policy, media: captureMediaStripPolicy() }; } if (typeof media !== 'object' && isImageFormatError(raw)) { signal?.throwIfAborted(); - this.log.warn( + reportMediaStripped( 'provider rejected an image in the request; resending with rejected media stripped', - { - model: request.model.name, - ...request.logFields, - }, ); return { ...policy, media: captureMediaStripPolicy() }; } diff --git a/packages/agent-core-v2/src/agent/media/image-compress.ts b/packages/agent-core-v2/src/agent/media/image-compress.ts index 9622de6239..fd2957bc91 100644 --- a/packages/agent-core-v2/src/agent/media/image-compress.ts +++ b/packages/agent-core-v2/src/agent/media/image-compress.ts @@ -2,18 +2,14 @@ import type { ContentPart } from '#/kosong/contract/message'; import { sniffImageDimensions } from './file-type'; import { - buildMalformedImageNotice, - buildUnsupportedImageNotice, - decodeBase64Prefix, - isDataUrl, - isModelAcceptedImageMime, + gateImageFormatParts, normalizeImageMime, parseImageDataUrl, - resolveEffectiveImageMime, - unsupportedImageMimeFromUrl, } from './image-format-policy'; import { decodeWebp, isAnimatedWebp } from './webp-decode'; +export { gateImageFormatParts }; + export const MAX_IMAGE_EDGE_PX = 2000; let configuredMaxImageEdgePx: number | undefined; @@ -280,46 +276,6 @@ export interface CompressedContentParts { readonly captions: readonly string[]; } -export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] { - const out: ContentPart[] = []; - for (const part of parts) { - if (part.type === 'image_url') { - const parsed = parseImageDataUrl(part.imageUrl.url); - if (parsed === null) { - if (isDataUrl(part.imageUrl.url)) { - out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) }); - continue; - } - const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url); - if (extMime !== null) { - out.push({ - type: 'text', - text: buildUnsupportedImageNotice(extMime, part.imageUrl.url), - }); - continue; - } - out.push(part); - continue; - } - const effectiveMime = resolveEffectiveImageMime( - parsed.mimeType, - decodeBase64Prefix(parsed.base64), - ); - if (!isModelAcceptedImageMime(effectiveMime)) { - out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) }); - continue; - } - const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`; - if (part.imageUrl.url !== canonicalUrl) { - out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } }); - continue; - } - } - out.push(part); - } - return out; -} - export async function compressImageContentParts( parts: readonly ContentPart[], options: CompressImageOptions & { readonly annotate?: CompressAnnotateOptions } = {}, diff --git a/packages/agent-core-v2/src/agent/media/image-format-policy.ts b/packages/agent-core-v2/src/agent/media/image-format-policy.ts index 0d470a8970..f6574f7880 100644 --- a/packages/agent-core-v2/src/agent/media/image-format-policy.ts +++ b/packages/agent-core-v2/src/agent/media/image-format-policy.ts @@ -1,3 +1,40 @@ +/** + * `media` domain — provider-accepted image formats, the single source + * of truth. + * + * Model providers accept only PNG, JPEG, GIF, and WebP image blocks. An + * `image_url` part carrying any other MIME (AVIF, HEIC, BMP, TIFF, ICO, …) + * is rejected by the API — and because prompts and tool results persist in + * the session history, that one part makes every subsequent request fail + * too ("session poisoning"). Every ingestion point therefore refuses + * unsupported formats instead of passing the bytes through: ReadMediaFile + * refuses with a conversion command the model can run, and prompt/MCP + * ingestion replaces the image with a text notice. The same applies to an + * empty payload (`data:image/png;base64,` — a clipboard/upload failure that + * captured no bytes), which {@link gateImageFormatParts} replaces with a + * notice both at ingestion and, for histories already carrying one, at + * projection time. + * + * The policy is deliberately a closed set, not a denylist: a format is only + * ever sent when it is known to be accepted. Supporting a new format means + * adding it to {@link MODEL_ACCEPTED_IMAGE_MIMES}; tailoring the refusal + * guidance for a newly-seen unsupported format means adding one row to + * {@link UNSUPPORTED_IMAGE_FORMATS}. + * + * Inbound MIME strings are normalized for the DECISION + * ({@link normalizeImageMime}: case, whitespace, `image/jpg`), but every + * call site must forward the CANONICAL MIME into the session — strict + * provider whitelists (e.g. Anthropic's) reject the raw alias, which would + * re-create the very session poisoning this module exists to prevent. + * + * Scope: only inline `data:` images can be gated. A remote http(s) image URL + * (an MCP `resource_link`, a REST `source.kind: 'url'` part) carries no + * bytes to inspect, and providers that support URL images fetch them + * server-side; those pass through unchanged. + */ + +import type { ContentPart } from '#/kosong/contract/message'; + import { IMAGE_MIME_BY_SUFFIX, sniffMediaFromMagic } from './file-type'; export const MODEL_ACCEPTED_IMAGE_MIMES: ReadonlySet = new Set([ @@ -138,3 +175,64 @@ export function buildMalformedImageNotice(url: string): string { 'could not be parsed). Re-encode the image as PNG or JPEG and try again.]' ); } + +export function buildEmptyImageNotice(name?: string): string { + const what = name === undefined || name.length === 0 ? 'The attached image' : `"${name}"`; + return ( + `[Image omitted: ${what} contained no image data (0 bytes) — the clipboard ` + + 'or upload captured nothing. Re-paste or re-upload the image and try again.]' + ); +} + +/** + * Content-part format gate shared by every image ingestion point — and by the + * context projector, so a malformed image already sitting in an old session's + * history is replaced on the wire instead of poisoning every later request + * ("session poisoning", see the module doc). Images the provider cannot + * accept never pass through: each is replaced by a text notice that tells the + * model what happened and how to recover. + * + * A parsed data URL is still rejected when its payload is empty + * (`data:image/png;base64,` — a clipboard/upload failure captured no bytes). + */ +export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] { + const out: ContentPart[] = []; + for (const part of parts) { + if (part.type === 'image_url') { + const parsed = parseImageDataUrl(part.imageUrl.url); + if (parsed === null) { + if (isDataUrl(part.imageUrl.url)) { + out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) }); + continue; + } + const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url); + if (extMime !== null) { + out.push({ + type: 'text', + text: buildUnsupportedImageNotice(extMime, part.imageUrl.url), + }); + continue; + } + out.push(part); + continue; + } + const head = decodeBase64Prefix(parsed.base64); + if (head.length === 0) { + out.push({ type: 'text', text: buildEmptyImageNotice() }); + continue; + } + const effectiveMime = resolveEffectiveImageMime(parsed.mimeType, head); + if (!isModelAcceptedImageMime(effectiveMime)) { + out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) }); + continue; + } + const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`; + if (part.imageUrl.url !== canonicalUrl) { + out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } }); + continue; + } + } + out.push(part); + } + return out; +} diff --git a/packages/agent-core-v2/src/index.ts b/packages/agent-core-v2/src/index.ts index cf1e050fd8..1d7d82ecbc 100644 --- a/packages/agent-core-v2/src/index.ts +++ b/packages/agent-core-v2/src/index.ts @@ -571,6 +571,7 @@ export { } from '#/agent/media/image-compress'; export { MODEL_ACCEPTED_IMAGE_MIMES, + buildEmptyImageNotice, buildImageConversionGuidance, buildUnsupportedImageNotice, decodeBase64Prefix, diff --git a/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts b/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts index fc208f4374..6758668030 100644 --- a/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts +++ b/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts @@ -646,6 +646,36 @@ describe('projector tool-exchange normalization', () => { }); }); + describe('image format gate at projection time', () => { + it('replaces an empty-payload image already in history with a notice (self-heal)', () => { + const history = [ + { + role: 'user' as const, + content: [{ type: 'image_url' as const, imageUrl: { url: 'data:image/png;base64,' } }], + toolCalls: [], + origin: { kind: 'user' as const }, + }, + user('what do you see?'), + ]; + const parts = project(history).flatMap((message) => message.content); + expect(parts.some((part) => part.type === 'image_url')).toBe(false); + const texts = parts.filter((part) => part.type === 'text').map((part) => part.text); + expect(texts.some((text) => text.includes('no image data (0 bytes)'))).toBe(true); + expect(texts.some((text) => text.includes('what do you see?'))).toBe(true); + }); + + it('passes a deliverable image through untouched', () => { + const part = { + type: 'image_url' as const, + imageUrl: { url: 'data:image/png;base64,QUJD' }, + }; + const history = [ + { role: 'user' as const, content: [part], toolCalls: [], origin: { kind: 'user' as const } }, + ]; + expect(project(history).flatMap((message) => message.content)).toContainEqual(part); + }); + }); + describe('project with media: degraded policy', () => { function imageMessage(url: string): ContextMessage { return { @@ -678,7 +708,7 @@ describe('projector tool-exchange normalization', () => { .filter((part) => part.type === 'text') .map((part) => part.text); expect( - markers.filter((text) => text.includes('dropped to fit the provider request size limit')), + markers.filter((text) => text.includes('removed to fit the provider request size limit')), ).toHaveLength(2); }); @@ -737,8 +767,8 @@ describe('projector tool-exchange normalization', () => { const texts = allParts.filter((part) => part.type === 'text').map((part) => part.text); expect(texts).toContain('look at these'); expect(texts).toContain(''); - expect(texts.some((text) => text.includes('omitted for provider compatibility'))).toBe(true); - expect(texts.some((text) => text.includes('get conversion guidance'))).toBe(true); + expect(texts.some((text) => text.includes('unsupported or unreadable image data'))).toBe(true); + expect(texts.some((text) => text.includes('You have NOT seen this image'))).toBe(true); }); it('returns the projected messages untouched when there is no media', () => { diff --git a/packages/agent-core-v2/test/agent/media/image-compress.test.ts b/packages/agent-core-v2/test/agent/media/image-compress.test.ts index 37e983ee3f..e8e691b019 100644 --- a/packages/agent-core-v2/test/agent/media/image-compress.test.ts +++ b/packages/agent-core-v2/test/agent/media/image-compress.test.ts @@ -750,6 +750,15 @@ describe('gateImageFormatParts', () => { } }); + it('drops an empty-payload data URL (clipboard/upload captured nothing)', () => { + const out = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,' } }, + ]); + expect(out.some((p) => p.type === 'image_url')).toBe(false); + expect(out[0]).toMatchObject({ type: 'text' }); + expect((out[0] as { text: string }).text).toContain('no image data (0 bytes)'); + }); + it('truncates a long malformed data URL in the notice', () => { const url = `data:image/png${'x'.repeat(500)}`; const out = gateImageFormatParts([{ type: 'image_url', imageUrl: { url } }]); diff --git a/packages/kap-server/src/lib/promptMedia.ts b/packages/kap-server/src/lib/promptMedia.ts index dbcf398170..bc836b657a 100644 --- a/packages/kap-server/src/lib/promptMedia.ts +++ b/packages/kap-server/src/lib/promptMedia.ts @@ -37,6 +37,12 @@ type WireContent = PromptSubmission['content']; * the wrong media kind, e.g. a PDF submitted as a video) must reject the * request without creating the prompt agent and without touching the * session's model/thinking/permission. + * + * Zero-byte images are rejected here too (inline base64 that decodes to + * nothing, or an uploaded image file with no bytes): a clipboard/upload + * failure captured nothing, and submitting the prompt without the image the + * user meant to attach would silently waste the turn. The client keeps the + * draft and can re-paste. */ export async function assertPromptFileRefs(content: WireContent, store: IFileService): Promise { for (const part of content) { @@ -45,6 +51,21 @@ export async function assertPromptFileRefs(content: WireContent, store: IFileSer } else if ((part.type === 'image' || part.type === 'video') && part.source.kind === 'file') { const file = await store.get(part.source.file_id); assertMediaFile(file, part.type); + if (part.type === 'image' && file.meta.size === 0) { + throw new Error2( + 'validation.failed', + `"${file.meta.name}" contained no image data (0 bytes) — the clipboard or upload ` + + 'captured nothing. Re-paste or re-upload the image and try again.', + ); + } + } else if (part.type === 'image' && part.source.kind === 'base64') { + if (decodeBase64Prefix(part.source.data).length === 0) { + throw new Error2( + 'validation.failed', + 'The attached image contained no image data (0 bytes) — the clipboard or upload ' + + 'captured nothing. Re-paste or re-upload the image and try again.', + ); + } } } } diff --git a/packages/kap-server/test/prompts.test.ts b/packages/kap-server/test/prompts.test.ts index 1b41cc5a32..dc4a439abc 100644 --- a/packages/kap-server/test/prompts.test.ts +++ b/packages/kap-server/test/prompts.test.ts @@ -876,6 +876,48 @@ describe('server-v2 /api/v1 prompts', () => { expect(notice.text).toContain('photo.avif'); }); + it('rejects an inline base64 image that decodes to zero bytes without enqueuing the prompt', async () => { + // A clipboard/upload failure can produce a payload that carries no bytes + // (issue #2209). Submitting the prompt without the image the user meant + // to attach would silently waste the turn, so the route rejects — the + // client keeps the draft and can re-paste. The wire schema rejects an + // empty `data` string outright, so the reachable shape is one that + // passes min(1) but decodes to nothing. + const id = await createSession(home as string); + const session = getLiveSessionById(server!.core.accessor, id); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [ + { type: 'text', text: 'what is in this image?' }, + { type: 'image', source: { kind: 'base64', media_type: 'image/png', data: '====' } }, + ], + }); + expect(body.code).toBe(40001); + expect(body.msg).toContain('no image data (0 bytes)'); + + // The failed request must not have materialized the main agent either. + expect(session!.accessor.get(IAgentLifecycleService).findAgentHandle('main')).toBeUndefined(); + }); + + it('rejects a zero-byte uploaded image file without enqueuing the prompt', async () => { + const id = await createSession(home as string); + const session = getLiveSessionById(server!.core.accessor, id); + const uploaded = await uploadFile(Buffer.alloc(0), 'image/png', 'image.png'); + expect(uploaded.size).toBe(0); + + const { body } = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [ + { type: 'text', text: 'what is in this image?' }, + { type: 'image', source: { kind: 'file', file_id: uploaded.id } }, + ], + }); + expect(body.code).toBe(40001); + expect(body.msg).toContain('no image data (0 bytes)'); + expect(body.msg).toContain('image.png'); + + expect(session!.accessor.get(IAgentLifecycleService).findAgentHandle('main')).toBeUndefined(); + }); + it('replaces a remote image URL with an unsupported extension with a text notice', async () => { const id = await createSession(home as string); await createMainAgent(id); diff --git a/packages/klient/test/e2e/invalid-input-matrix.test.ts b/packages/klient/test/e2e/invalid-input-matrix.test.ts index 3a8cfceb67..d172f7db07 100644 --- a/packages/klient/test/e2e/invalid-input-matrix.test.ts +++ b/packages/klient/test/e2e/invalid-input-matrix.test.ts @@ -609,7 +609,9 @@ describe('image blocks with invalid data', () => { expect(secondContent.some((part) => (part as { type?: string }).type === 'image_url')).toBe( false, ); - expect(JSON.stringify(secondContent)).toContain('image omitted for provider compatibility'); + expect(JSON.stringify(secondContent)).toContain( + 'removed before sending because the provider could not accept it', + ); expect(ctx.payloads('prompt.completed')[0]?.['reason']).toBe('completed'); }, 30_000);