From de65134b8775fd0118c18a07df190af2e786c9a9 Mon Sep 17 00:00:00 2001 From: Colin B Date: Tue, 7 Jul 2026 15:54:12 +0000 Subject: [PATCH 1/3] Preserve sibling tool calls in assistant imports --- crates/lingua/src/processing/import.rs | 136 +++++++++++++++++- ...ng-with-sibling-tool-calls.assertions.json | 14 ++ ...hinking-with-sibling-tool-calls.spans.json | 29 ++++ 3 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 payloads/import-cases/role-message-thinking-with-sibling-tool-calls.assertions.json create mode 100644 payloads/import-cases/role-message-thinking-with-sibling-tool-calls.spans.json diff --git a/crates/lingua/src/processing/import.rs b/crates/lingua/src/processing/import.rs index 6f9090c8..605555b2 100644 --- a/crates/lingua/src/processing/import.rs +++ b/crates/lingua/src/processing/import.rs @@ -196,6 +196,11 @@ fn try_parse_mixed_role_messages_for_import(data: &Value) -> Option let mut messages = Vec::new(); for item in items { + if let Some(message) = parse_lenient_assistant_message_with_tool_calls(item) { + messages.push(message); + continue; + } + let mut parsed_messages = try_parsers_in_order(item, &provider_parsers).or_else(|| { let wrapped_item = Value::Array(vec![item.clone()]); try_parsers_in_order(&wrapped_item, &provider_parsers) @@ -312,6 +317,12 @@ enum LenientAssistantContentPartCompat { #[serde(default)] encrypted_content: Option, }, + #[serde(rename = "thinking")] + Thinking { + thinking: String, + #[serde(default)] + signature: Option, + }, #[serde(rename = "tool_call", alias = "tool-call", alias = "toolCall")] ToolCall { #[serde(alias = "toolCallId")] @@ -336,6 +347,40 @@ enum LenientAssistantContentPartCompat { }, } +#[derive(Debug, Clone, Deserialize)] +struct LenientAssistantMessageCompat { + role: AssistantRoleCompat, + #[serde(default)] + content: Option, + #[serde(default)] + tool_calls: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "lowercase")] +enum AssistantRoleCompat { + Assistant, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum LenientAssistantContentCompat { + String(String), + Array(Vec), +} + +#[derive(Debug, Clone, Deserialize)] +struct LenientOpenAiToolCallCompat { + id: String, + function: LenientOpenAiFunctionCallCompat, +} + +#[derive(Debug, Clone, Deserialize)] +struct LenientOpenAiFunctionCallCompat { + name: String, + arguments: Value, +} + #[derive(Debug, Clone, Deserialize)] #[serde(tag = "type")] enum LenientToolContentPartCompat { @@ -449,7 +494,15 @@ fn parse_tool_call_arguments(value: Option) -> Option } fn try_parse_lenient_assistant_content_part(item: &Value) -> Option { - match serde_json::from_value::(item.clone()).ok()? { + lenient_assistant_content_part_to_universal( + serde_json::from_value::(item.clone()).ok()?, + ) +} + +fn lenient_assistant_content_part_to_universal( + part: LenientAssistantContentPartCompat, +) -> Option { + match part { LenientAssistantContentPartCompat::Text { text } => { Some(AssistantContentPart::Text(TextContentPart { text, @@ -465,6 +518,13 @@ fn try_parse_lenient_assistant_content_part(item: &Value) -> Option Some(AssistantContentPart::Reasoning { + text: thinking, + encrypted_content: signature, + }), LenientAssistantContentPartCompat::ToolCall { tool_call_id, tool_name, @@ -492,6 +552,80 @@ fn try_parse_lenient_assistant_content_part(item: &Value) -> Option ToolCallArguments { + match value { + Value::Object(map) => ToolCallArguments::Valid(map), + Value::String(text) => match serde_json::from_str::(&text) { + Ok(Value::Object(map)) => ToolCallArguments::Valid(map), + Ok(other) => ToolCallArguments::Invalid(other.to_string()), + Err(_) => ToolCallArguments::Invalid(text), + }, + other => ToolCallArguments::Invalid(other.to_string()), + } +} + +fn openai_tool_call_to_assistant_part( + tool_call: LenientOpenAiToolCallCompat, +) -> AssistantContentPart { + AssistantContentPart::ToolCall { + tool_call_id: tool_call.id, + tool_name: tool_call.function.name, + arguments: openai_tool_call_arguments(tool_call.function.arguments), + encrypted_content: None, + provider_options: None, + provider_executed: None, + } +} + +fn parse_lenient_assistant_message_with_tool_calls(item: &Value) -> Option { + let parsed = LenientAssistantMessageCompat::deserialize(item).ok()?; + match parsed.role { + AssistantRoleCompat::Assistant => {} + } + if parsed.tool_calls.is_empty() { + return None; + } + + let mut parts = Vec::new(); + if let Some(content) = parsed.content { + match content { + LenientAssistantContentCompat::String(text) => { + if !text.is_empty() { + parts.push(AssistantContentPart::Text(TextContentPart { + text, + encrypted_content: None, + cache_control: None, + provider_options: None, + })); + } + } + LenientAssistantContentCompat::Array(content_parts) => { + parts.extend( + content_parts + .into_iter() + .filter_map(lenient_assistant_content_part_to_universal), + ); + } + } + } + + parts.extend( + parsed + .tool_calls + .into_iter() + .map(openai_tool_call_to_assistant_part), + ); + + if parts.is_empty() { + return None; + } + + Some(Message::Assistant { + content: AssistantContent::Array(parts), + id: None, + }) +} + fn try_parse_lenient_tool_content_part(item: &Value) -> Option { match serde_json::from_value::(item.clone()).ok()? { LenientToolContentPartCompat::ToolResult { diff --git a/payloads/import-cases/role-message-thinking-with-sibling-tool-calls.assertions.json b/payloads/import-cases/role-message-thinking-with-sibling-tool-calls.assertions.json new file mode 100644 index 00000000..0413cc96 --- /dev/null +++ b/payloads/import-cases/role-message-thinking-with-sibling-tool-calls.assertions.json @@ -0,0 +1,14 @@ +{ + "expectedMessageCount": 1, + "expectedRolesInOrder": [ + "assistant" + ], + "mustContainText": [ + "Need to query structured data before answering.", + "reasoning_signature_123", + "call_structured_data_123", + "StructuredDataQueryTool", + "select count(*) from anonymized_table", + "\"limit\":10" + ] +} diff --git a/payloads/import-cases/role-message-thinking-with-sibling-tool-calls.spans.json b/payloads/import-cases/role-message-thinking-with-sibling-tool-calls.spans.json new file mode 100644 index 00000000..6caddd3f --- /dev/null +++ b/payloads/import-cases/role-message-thinking-with-sibling-tool-calls.spans.json @@ -0,0 +1,29 @@ +[ + { + "output": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Need to query structured data before answering.", + "signature": "reasoning_signature_123" + } + ], + "tool_calls": [ + { + "id": "call_structured_data_123", + "type": "function", + "function": { + "name": "StructuredDataQueryTool", + "arguments": { + "query": "select count(*) from anonymized_table", + "limit": 10 + } + } + } + ] + } + ] + } +] From d0e31fd3ef3a26e0a168cec79023e5cf2156ce59 Mon Sep 17 00:00:00 2001 From: Colin B Date: Tue, 7 Jul 2026 16:07:01 +0000 Subject: [PATCH 2/3] Defer OpenAI reasoning imports to provider parser --- crates/lingua/src/processing/import.rs | 9 +++++++- ...ng-with-sibling-tool-calls.assertions.json | 13 +++++++++++ ...asoning-with-sibling-tool-calls.spans.json | 22 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 payloads/import-cases/role-message-openai-reasoning-with-sibling-tool-calls.assertions.json create mode 100644 payloads/import-cases/role-message-openai-reasoning-with-sibling-tool-calls.spans.json diff --git a/crates/lingua/src/processing/import.rs b/crates/lingua/src/processing/import.rs index 605555b2..484a5d97 100644 --- a/crates/lingua/src/processing/import.rs +++ b/crates/lingua/src/processing/import.rs @@ -354,6 +354,10 @@ struct LenientAssistantMessageCompat { content: Option, #[serde(default)] tool_calls: Vec, + #[serde(default)] + reasoning: Option, + #[serde(default)] + reasoning_signature: Option, } #[derive(Debug, Clone, Deserialize)] @@ -582,7 +586,10 @@ fn parse_lenient_assistant_message_with_tool_calls(item: &Value) -> Option {} } - if parsed.tool_calls.is_empty() { + if parsed.tool_calls.is_empty() + || parsed.reasoning.is_some() + || parsed.reasoning_signature.is_some() + { return None; } diff --git a/payloads/import-cases/role-message-openai-reasoning-with-sibling-tool-calls.assertions.json b/payloads/import-cases/role-message-openai-reasoning-with-sibling-tool-calls.assertions.json new file mode 100644 index 00000000..3ffa114a --- /dev/null +++ b/payloads/import-cases/role-message-openai-reasoning-with-sibling-tool-calls.assertions.json @@ -0,0 +1,13 @@ +{ + "expectedMessageCount": 1, + "expectedRolesInOrder": [ + "assistant" + ], + "mustContainText": [ + "Use the structured data tool before answering.", + "top_level_reasoning_signature_123", + "call_openai_reasoning_123", + "get_summary", + "anonymized metrics" + ] +} diff --git a/payloads/import-cases/role-message-openai-reasoning-with-sibling-tool-calls.spans.json b/payloads/import-cases/role-message-openai-reasoning-with-sibling-tool-calls.spans.json new file mode 100644 index 00000000..fb85c5a7 --- /dev/null +++ b/payloads/import-cases/role-message-openai-reasoning-with-sibling-tool-calls.spans.json @@ -0,0 +1,22 @@ +[ + { + "output": [ + { + "role": "assistant", + "content": "", + "reasoning": "Use the structured data tool before answering.", + "reasoning_signature": "top_level_reasoning_signature_123", + "tool_calls": [ + { + "id": "call_openai_reasoning_123", + "type": "function", + "function": { + "name": "get_summary", + "arguments": "{\"topic\":\"anonymized metrics\"}" + } + } + ] + } + ] + } +] From 17892cf007e1e8d2c13a78d28d64e78c4025ed8c Mon Sep 17 00:00:00 2001 From: Colin B Date: Tue, 7 Jul 2026 23:23:06 +0000 Subject: [PATCH 3/3] Add import issue reproduction script --- .../show-import-thinking-tool-calls-issue.sh | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100755 scripts/show-import-thinking-tool-calls-issue.sh diff --git a/scripts/show-import-thinking-tool-calls-issue.sh b/scripts/show-import-thinking-tool-calls-issue.sh new file mode 100755 index 00000000..649561c6 --- /dev/null +++ b/scripts/show-import-thinking-tool-calls-issue.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# Show the assistant-thinking + sibling-tool-calls import behavior. +# +# On a broken importer, this prints an assistant message with reasoning only and +# then fails because the sibling tool_call fields are absent. On a fixed importer, +# it prints one assistant message containing both the reasoning part and tool_call +# part. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +TEST_FILE="$REPO_ROOT/crates/lingua/tests/show_import_thinking_tool_calls_issue.rs" + +cleanup() { + rm -f "$TEST_FILE" +} +trap cleanup EXIT + +if [ -f "$TEST_FILE" ]; then + echo "Refusing to overwrite existing test file: $TEST_FILE" >&2 + exit 1 +fi + +if ! command -v cargo >/dev/null 2>&1; then + echo "cargo is required to run this script" >&2 + exit 1 +fi + +CARGO_CMD=(cargo) +if [ -n "${CARGO_TOOLCHAIN:-}" ]; then + CARGO_CMD=(cargo "+$CARGO_TOOLCHAIN") +elif command -v rustup >/dev/null 2>&1 && rustup toolchain list | grep -q '^1\.95\.0'; then + CARGO_CMD=(cargo +1.95.0) +fi + +cat > "$TEST_FILE" <<'RS' +use lingua::processing::{import_messages_from_spans, Span}; +use lingua::serde_json; + +#[test] +fn show_import_thinking_tool_calls_issue() { + let spans_json = r#" +[ + { + "output": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Need to query structured data before answering.", + "signature": "reasoning_signature_123" + } + ], + "tool_calls": [ + { + "id": "call_structured_data_123", + "type": "function", + "function": { + "name": "StructuredDataQueryTool", + "arguments": { + "query": "select count(*) from anonymized_table", + "limit": 10 + } + } + } + ] + } + ] + } +] +"#; + + let spans: Vec = serde_json::from_str(spans_json).expect("fixture should parse"); + let messages = import_messages_from_spans(spans); + let pretty = serde_json::to_string_pretty(&messages).expect("messages should serialize"); + + println!("\nImported Lingua messages:\n{}\n", pretty); + + for expected in [ + "Need to query structured data before answering.", + "reasoning_signature_123", + "call_structured_data_123", + "StructuredDataQueryTool", + "select count(*) from anonymized_table", + "\"limit\": 10", + ] { + assert!( + pretty.contains(expected), + "missing expected text: {expected}\n\nOn the broken importer, sibling tool_calls are dropped before the UI sees them.\n\nImported messages:\n{pretty}" + ); + } +} +RS + +cd "$REPO_ROOT" +echo "Running: ${CARGO_CMD[*]} test -p lingua --test show_import_thinking_tool_calls_issue -- --nocapture" +"${CARGO_CMD[@]}" test -p lingua --test show_import_thinking_tool_calls_issue -- --nocapture