Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/empty-image-session-poisoning.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@moonshot-ai/kimi-code": patch
---

Fix pasted images intermittently failing to reach the model: a zero-byte image from a failed clipboard read poisoned the session, so every later image was dropped with an ambiguous placeholder and the model could hallucinate having seen it. Prompts carrying an empty image are now rejected with a clear error so you can re-paste before anything is sent, sessions already affected recover automatically, and the placeholder tells the model the attachment was removed and must not be guessed at.
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,20 @@ export const MEDIA_DEGRADE_KEEP_RECENT = 2;

const MEDIA_DEGRADED_PLACEHOLDERS = {
image_url:
'[image omitted: dropped to fit the provider request size limit; re-read the file to view it]',
'[An image attached to an earlier message was removed to fit the provider request size limit. You have NOT seen this image — do not describe or guess its contents. If it matters, ask the user to re-send it or to point you at the file so you can read it with ReadMediaFile.]',
audio_url:
'[audio omitted: dropped to fit the provider request size limit; re-read the file to hear it]',
'[An audio clip attached to an earlier message was removed to fit the provider request size limit. You have NOT heard it — do not describe or guess its contents.]',
video_url:
'[video omitted: dropped to fit the provider request size limit; re-read the file to view it]',
'[A video attached to an earlier message was removed to fit the provider request size limit. You have NOT seen it — do not describe or guess its contents.]',
} as const;

export const MEDIA_STRIPPED_PLACEHOLDERS = {
image_url:
'[image omitted for provider compatibility; re-read the file to view it or get conversion guidance]',
'[An image attached to this message was removed before sending because the provider could not accept it (unsupported or unreadable image data). You have NOT seen this image — do not describe or guess its contents. Tell the user the image failed to reach you and suggest re-sending it as PNG or JPEG.]',
audio_url:
'[audio omitted for provider compatibility; re-read the file to hear it]',
'[An audio clip attached to this message was removed before sending because the provider could not accept it. You have NOT heard it — do not describe or guess its contents.]',
video_url:
'[video omitted for provider compatibility; re-read the file to view it]',
'[A video attached to this message was removed before sending because the provider could not accept it. You have NOT seen it — do not describe or guess its contents.]',
} as const;

type MediaPlaceholderSet = typeof MEDIA_DEGRADED_PLACEHOLDERS | typeof MEDIA_STRIPPED_PLACEHOLDERS;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ErrorCodes, Error2 } from '#/errors';
import { renderToolResultForModel } from '#/agent/contextMemory/toolResultRender';
import type { ContextMessage } from '#/agent/contextMemory/types';
import { isVacuousContentPart } from '#/agent/contextMemory/vacuousContent';
import { gateImageFormatParts } from '#/agent/media/image-format-policy';
import type { ContentPart, Message } from '#/kosong/contract/message';

export type ProjectionAnomaly =
Expand Down Expand Up @@ -344,7 +345,11 @@ function projectedContent(source: ContextMessage, onAnomaly?: OnAnomaly): Conten
note: source.note,
})
: source.content;
return cleanContent(source, content, onAnomaly);
// The image format gate runs at projection time too (not only at ingestion):
// a malformed/undeliverable image already persisted in an old session's
// history is replaced by a text notice on the wire, so it can no longer
// poison every later request. Read-side only — the history keeps its parts.
return cleanContent(source, gateImageFormatParts(content), onAnomaly);
}

function cleanContent(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,19 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService {
if (signal?.aborted === true) return undefined;
const raw = unwrapErrorCause(error);
const media = policy?.media;
// Strip is the last-resort recovery: when it fires, the provider never saw
// any of the images, so the warn carries the rejection's status/message
// (the classification chain swallows both).
const reportMediaStripped = (message: string): void => {
const statusCode = raw instanceof APIStatusError ? raw.statusCode : undefined;
const errorMessage = (raw instanceof Error ? raw.message : String(raw)).slice(0, 300);
this.log.warn(message, {
model: request.model.name,
statusCode,
errorMessage,
...request.logFields,
});
};
if (
raw instanceof APIRequestTooLargeError &&
(media === undefined || media === 'degraded')
Expand All @@ -474,23 +487,15 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService {
this.markRecoveryTurn(this.mediaDegradedTurns, request.source);
return { ...policy, media: 'degraded' };
}
this.log.warn(
reportMediaStripped(
'provider rejected degraded-media request as too large; resending with rejected media stripped',
{
model: request.model.name,
...request.logFields,
},
);
return { ...policy, media: captureMediaStripPolicy() };
}
if (typeof media !== 'object' && isImageFormatError(raw)) {
signal?.throwIfAborted();
this.log.warn(
reportMediaStripped(
'provider rejected an image in the request; resending with rejected media stripped',
{
model: request.model.name,
...request.logFields,
},
);
return { ...policy, media: captureMediaStripPolicy() };
}
Expand Down
50 changes: 3 additions & 47 deletions packages/agent-core-v2/src/agent/media/image-compress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,14 @@ import type { ContentPart } from '#/kosong/contract/message';

import { sniffImageDimensions } from './file-type';
import {
buildMalformedImageNotice,
buildUnsupportedImageNotice,
decodeBase64Prefix,
isDataUrl,
isModelAcceptedImageMime,
gateImageFormatParts,
normalizeImageMime,
parseImageDataUrl,
resolveEffectiveImageMime,
unsupportedImageMimeFromUrl,
} from './image-format-policy';
import { decodeWebp, isAnimatedWebp } from './webp-decode';

export { gateImageFormatParts };

export const MAX_IMAGE_EDGE_PX = 2000;

let configuredMaxImageEdgePx: number | undefined;
Expand Down Expand Up @@ -280,46 +276,6 @@ export interface CompressedContentParts {
readonly captions: readonly string[];
}

export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] {
const out: ContentPart[] = [];
for (const part of parts) {
if (part.type === 'image_url') {
const parsed = parseImageDataUrl(part.imageUrl.url);
if (parsed === null) {
if (isDataUrl(part.imageUrl.url)) {
out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) });
continue;
}
const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url);
if (extMime !== null) {
out.push({
type: 'text',
text: buildUnsupportedImageNotice(extMime, part.imageUrl.url),
});
continue;
}
out.push(part);
continue;
}
const effectiveMime = resolveEffectiveImageMime(
parsed.mimeType,
decodeBase64Prefix(parsed.base64),
);
if (!isModelAcceptedImageMime(effectiveMime)) {
out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) });
continue;
}
const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`;
if (part.imageUrl.url !== canonicalUrl) {
out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } });
continue;
}
}
out.push(part);
}
return out;
}

export async function compressImageContentParts(
parts: readonly ContentPart[],
options: CompressImageOptions & { readonly annotate?: CompressAnnotateOptions } = {},
Expand Down
98 changes: 98 additions & 0 deletions packages/agent-core-v2/src/agent/media/image-format-policy.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,40 @@
/**
* `media` domain — provider-accepted image formats, the single source
* of truth.
*
* Model providers accept only PNG, JPEG, GIF, and WebP image blocks. An
* `image_url` part carrying any other MIME (AVIF, HEIC, BMP, TIFF, ICO, …)
* is rejected by the API — and because prompts and tool results persist in
* the session history, that one part makes every subsequent request fail
* too ("session poisoning"). Every ingestion point therefore refuses
* unsupported formats instead of passing the bytes through: ReadMediaFile
* refuses with a conversion command the model can run, and prompt/MCP
* ingestion replaces the image with a text notice. The same applies to an
* empty payload (`data:image/png;base64,` — a clipboard/upload failure that
* captured no bytes), which {@link gateImageFormatParts} replaces with a
* notice both at ingestion and, for histories already carrying one, at
* projection time.
*
* The policy is deliberately a closed set, not a denylist: a format is only
* ever sent when it is known to be accepted. Supporting a new format means
* adding it to {@link MODEL_ACCEPTED_IMAGE_MIMES}; tailoring the refusal
* guidance for a newly-seen unsupported format means adding one row to
* {@link UNSUPPORTED_IMAGE_FORMATS}.
*
* Inbound MIME strings are normalized for the DECISION
* ({@link normalizeImageMime}: case, whitespace, `image/jpg`), but every
* call site must forward the CANONICAL MIME into the session — strict
* provider whitelists (e.g. Anthropic's) reject the raw alias, which would
* re-create the very session poisoning this module exists to prevent.
*
* Scope: only inline `data:` images can be gated. A remote http(s) image URL
* (an MCP `resource_link`, a REST `source.kind: 'url'` part) carries no
* bytes to inspect, and providers that support URL images fetch them
* server-side; those pass through unchanged.
*/

import type { ContentPart } from '#/kosong/contract/message';

import { IMAGE_MIME_BY_SUFFIX, sniffMediaFromMagic } from './file-type';

export const MODEL_ACCEPTED_IMAGE_MIMES: ReadonlySet<string> = new Set([
Expand Down Expand Up @@ -138,3 +175,64 @@ export function buildMalformedImageNotice(url: string): string {
'could not be parsed). Re-encode the image as PNG or JPEG and try again.]'
);
}

export function buildEmptyImageNotice(name?: string): string {
const what = name === undefined || name.length === 0 ? 'The attached image' : `"${name}"`;
return (
`[Image omitted: ${what} contained no image data (0 bytes) — the clipboard ` +
'or upload captured nothing. Re-paste or re-upload the image and try again.]'
);
}

/**
* Content-part format gate shared by every image ingestion point — and by the
* context projector, so a malformed image already sitting in an old session's
* history is replaced on the wire instead of poisoning every later request
* ("session poisoning", see the module doc). Images the provider cannot
* accept never pass through: each is replaced by a text notice that tells the
* model what happened and how to recover.
*
* A parsed data URL is still rejected when its payload is empty
* (`data:image/png;base64,` — a clipboard/upload failure captured no bytes).
*/
export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] {
const out: ContentPart[] = [];
for (const part of parts) {
if (part.type === 'image_url') {
const parsed = parseImageDataUrl(part.imageUrl.url);
if (parsed === null) {
if (isDataUrl(part.imageUrl.url)) {
out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) });
continue;
}
const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url);
if (extMime !== null) {
out.push({
type: 'text',
text: buildUnsupportedImageNotice(extMime, part.imageUrl.url),
});
continue;
}
out.push(part);
continue;
}
const head = decodeBase64Prefix(parsed.base64);
if (head.length === 0) {
out.push({ type: 'text', text: buildEmptyImageNotice() });
continue;
}
const effectiveMime = resolveEffectiveImageMime(parsed.mimeType, head);
if (!isModelAcceptedImageMime(effectiveMime)) {
out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) });
continue;
}
const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`;
if (part.imageUrl.url !== canonicalUrl) {
out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } });
continue;
}
}
out.push(part);
}
return out;
}
1 change: 1 addition & 0 deletions packages/agent-core-v2/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,7 @@ export {
} from '#/agent/media/image-compress';
export {
MODEL_ACCEPTED_IMAGE_MIMES,
buildEmptyImageNotice,
buildImageConversionGuidance,
buildUnsupportedImageNotice,
decodeBase64Prefix,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -646,6 +646,36 @@ describe('projector tool-exchange normalization', () => {
});
});

describe('image format gate at projection time', () => {
it('replaces an empty-payload image already in history with a notice (self-heal)', () => {
const history = [
{
role: 'user' as const,
content: [{ type: 'image_url' as const, imageUrl: { url: 'data:image/png;base64,' } }],
toolCalls: [],
origin: { kind: 'user' as const },
},
user('what do you see?'),
];
const parts = project(history).flatMap((message) => message.content);
expect(parts.some((part) => part.type === 'image_url')).toBe(false);
const texts = parts.filter((part) => part.type === 'text').map((part) => part.text);
expect(texts.some((text) => text.includes('no image data (0 bytes)'))).toBe(true);
expect(texts.some((text) => text.includes('what do you see?'))).toBe(true);
});

it('passes a deliverable image through untouched', () => {
const part = {
type: 'image_url' as const,
imageUrl: { url: 'data:image/png;base64,QUJD' },
};
const history = [
{ role: 'user' as const, content: [part], toolCalls: [], origin: { kind: 'user' as const } },
];
expect(project(history).flatMap((message) => message.content)).toContainEqual(part);
});
});

describe('project with media: degraded policy', () => {
function imageMessage(url: string): ContextMessage {
return {
Expand Down Expand Up @@ -678,7 +708,7 @@ describe('projector tool-exchange normalization', () => {
.filter((part) => part.type === 'text')
.map((part) => part.text);
expect(
markers.filter((text) => text.includes('dropped to fit the provider request size limit')),
markers.filter((text) => text.includes('removed to fit the provider request size limit')),
).toHaveLength(2);
});

Expand Down Expand Up @@ -737,8 +767,8 @@ describe('projector tool-exchange normalization', () => {
const texts = allParts.filter((part) => part.type === 'text').map((part) => part.text);
expect(texts).toContain('look at these');
expect(texts).toContain('<image path="/tmp/shot.png">');
expect(texts.some((text) => text.includes('omitted for provider compatibility'))).toBe(true);
expect(texts.some((text) => text.includes('get conversion guidance'))).toBe(true);
expect(texts.some((text) => text.includes('unsupported or unreadable image data'))).toBe(true);
expect(texts.some((text) => text.includes('You have NOT seen this image'))).toBe(true);
});

it('returns the projected messages untouched when there is no media', () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -750,6 +750,15 @@ describe('gateImageFormatParts', () => {
}
});

it('drops an empty-payload data URL (clipboard/upload captured nothing)', () => {
const out = gateImageFormatParts([
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,' } },
]);
expect(out.some((p) => p.type === 'image_url')).toBe(false);
expect(out[0]).toMatchObject({ type: 'text' });
expect((out[0] as { text: string }).text).toContain('no image data (0 bytes)');
});

it('truncates a long malformed data URL in the notice', () => {
const url = `data:image/png${'x'.repeat(500)}`;
const out = gateImageFormatParts([{ type: 'image_url', imageUrl: { url } }]);
Expand Down
21 changes: 21 additions & 0 deletions packages/kap-server/src/lib/promptMedia.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@ type WireContent = PromptSubmission['content'];
* the wrong media kind, e.g. a PDF submitted as a video) must reject the
* request without creating the prompt agent and without touching the
* session's model/thinking/permission.
*
* Zero-byte images are rejected here too (inline base64 that decodes to
* nothing, or an uploaded image file with no bytes): a clipboard/upload
* failure captured nothing, and submitting the prompt without the image the
* user meant to attach would silently waste the turn. The client keeps the
* draft and can re-paste.
*/
export async function assertPromptFileRefs(content: WireContent, store: IFileService): Promise<void> {
for (const part of content) {
Expand All @@ -45,6 +51,21 @@ export async function assertPromptFileRefs(content: WireContent, store: IFileSer
} else if ((part.type === 'image' || part.type === 'video') && part.source.kind === 'file') {
const file = await store.get(part.source.file_id);
assertMediaFile(file, part.type);
if (part.type === 'image' && file.meta.size === 0) {
throw new Error2(
'validation.failed',
`"${file.meta.name}" contained no image data (0 bytes) — the clipboard or upload ` +
'captured nothing. Re-paste or re-upload the image and try again.',
);
}
} else if (part.type === 'image' && part.source.kind === 'base64') {
if (decodeBase64Prefix(part.source.data).length === 0) {
throw new Error2(
'validation.failed',
'The attached image contained no image data (0 bytes) — the clipboard or upload ' +
'captured nothing. Re-paste or re-upload the image and try again.',
);
}
}
}
}
Expand Down
Loading