diff --git a/crates/message-format-conformance/src/harness.rs b/crates/message-format-conformance/src/harness.rs index 4bcd138..a923dd9 100644 --- a/crates/message-format-conformance/src/harness.rs +++ b/crates/message-format-conformance/src/harness.rs @@ -140,7 +140,7 @@ fn cases() -> Vec { args: Vec::new(), expected: include_str!("../../../fixtures/expected/raw_match_integer_exact.txt") .trim_end(), - host_mode: HostMode::Noop, + host_mode: HostMode::Builtin { locale: "en-US" }, }, Case { source: include_str!("../../../fixtures/mf2/raw_match_offset_selector.mf2"), @@ -148,7 +148,7 @@ fn cases() -> Vec { args: Vec::new(), expected: include_str!("../../../fixtures/expected/raw_match_offset_selector.txt") .trim_end(), - host_mode: HostMode::Noop, + host_mode: HostMode::Builtin { locale: "en-US" }, }, Case { source: include_str!("../../../fixtures/mf2/raw_match_two_locals.mf2"), diff --git a/crates/message-format-conformance/src/tr35/helpers.rs b/crates/message-format-conformance/src/tr35/helpers.rs index 9e19cc7..eea222e 100644 --- a/crates/message-format-conformance/src/tr35/helpers.rs +++ b/crates/message-format-conformance/src/tr35/helpers.rs @@ -160,19 +160,18 @@ pub(super) fn assert_format_err( } } -/// Assert that formatting returns the expected output and reports a matching error. -pub(super) fn assert_format_with_error( - source: &str, - args: &[(&str, Value)], - expected: &str, - check: impl Fn(&FormatError) -> bool, -) { - let output = format_output_with_locale(source, args, "en").expect("format failed"); - assert_eq!(strip_bidi(&output.value), expected, "source: {source}"); +/// Assert that formatting reports exactly the expected diagnostic multiset. +pub(super) fn assert_errors_multiset(actual: &[FormatError], expected: &[FormatError]) { + let mut remaining = expected.to_vec(); + for error in actual { + let Some(index) = remaining.iter().position(|candidate| candidate == error) else { + panic!("unexpected diagnostic {error:?}; expected {expected:?}"); + }; + remaining.remove(index); + } assert!( - output.errors.iter().any(check), - "expected a matching format error for: {source}: {:?}", - output.errors + remaining.is_empty(), + "missing diagnostics {remaining:?}; actual {actual:?}" ); } @@ -278,13 +277,3 @@ pub(super) fn function_error(error: MessageFunctionError) -> FormatError { pub(super) fn missing_arg(name: &str) -> FormatError { FormatError::MissingArg(name.to_string()) } - -pub(super) fn bad_selector_with_source(expected: FormatError) -> impl Fn(&FormatError) -> bool { - move |error| { - matches!( - error, - FormatError::BadSelector { source } - if source.as_deref() == Some(&expected) - ) - } -} diff --git a/crates/message-format-conformance/src/tr35/number_fn.rs b/crates/message-format-conformance/src/tr35/number_fn.rs index b9f7b7d..3a656d2 100644 --- a/crates/message-format-conformance/src/tr35/number_fn.rs +++ b/crates/message-format-conformance/src/tr35/number_fn.rs @@ -102,6 +102,130 @@ fn integer_literal() { assert_format("{42 :integer}", &[], "42"); } +/// Resolved numeric locals retain exact integer payloads beyond the f64 safe +/// integer range until the runtime renders them. +#[test] +fn numeric_local_preserves_exact_integer() { + assert_format( + ".local $n = {9007199254740993 :number useGrouping=never} {{{$n}}}", + &[], + "9007199254740993", + ); +} + +/// Re-annotating a resolved number overrides only the supplied options and +/// retains inherited options from the local declaration. +#[test] +fn numeric_local_reannotation_inherits_options() { + assert_format( + ".local $n = {4.20 :number minimumFractionDigits=2 signDisplay=always} .local $m = {$n :number minimumFractionDigits=1} {{{$m}}}", + &[], + "+4.2", + ); +} + +/// Re-annotation of an input number also preserves its resolved options. +#[test] +fn numeric_input_reannotation_inherits_options() { + assert_format( + ".input {$n :number minimumFractionDigits=2 signDisplay=always} .local $m = {$n :number minimumFractionDigits=1} {{{$m}}}", + &[("n", Value::Float(4.2))], + "+4.2", + ); +} + +/// Offset applies to the exact resolved payload while retaining inherited +/// number formatting options. +#[test] +fn numeric_local_offset_inherits_options() { + assert_format( + ".local $n = {4.2 :number minimumFractionDigits=2 signDisplay=always} .local $m = {$n :offset add=1} {{{$m}}}", + &[], + "+5.20", + ); +} + +/// Integer conversion truncates a resolved number and drops inherited digit +/// and fraction formatting options. +#[test] +fn numeric_local_integer_drops_number_options() { + assert_format( + ".local $n = {4.2 :number minimumFractionDigits=2 signDisplay=always} .local $m = {$n :integer} {{{$m}}}", + &[], + "+4", + ); +} + +/// Exact matching uses the numeric payload even when grouped display options +/// are inherited by the resolved local. +#[test] +fn numeric_local_match_uses_exact_payload() { + assert_format( + ".local $n = {1234 :number useGrouping=always} .match $n 1234 {{exact}} * {{other}}", + &[], + "exact", + ); +} + +/// Exact numeric matching takes precedence over the plural keyword arm. +#[test] +fn numeric_local_match_exact_precedes_keyword() { + assert_format( + ".local $n = {1 :number minimumFractionDigits=2} .match $n 1 {{exact}} one {{one}} * {{other}}", + &[], + "exact", + ); +} + +/// A resolved numeric local with no exact arm does not use its display text as +/// a plural category selector. +#[test] +fn numeric_local_match_keyword_uses_plural_category() { + assert_format( + ".local $n = {1 :number minimumFractionDigits=2} .match $n one {{one}} * {{other}}", + &[], + "other", + ); +} + +/// Alias chains preserve the resolved numeric call and its inherited options. +#[test] +fn numeric_local_alias_chain_preserves_resolved_value() { + assert_format( + ".local $n = {4.2 :number minimumFractionDigits=2} .local $alias = {$n} .local $m = {$alias :number signDisplay=always} {{{$m}}}", + &[], + "+4.20", + ); +} + +/// A dynamic select option stays runtime-resolved: the invalid inner number +/// annotation reports bad-option and the enclosing match reports bad-selector. +#[test] +fn numeric_local_dynamic_select_reports_both_diagnostics() { + let output = format_output( + ".local $bad = {exact} .local $n = {1 :number select=$bad} .match $n 1 {{one}} * {{other {$n}}}", + &[], + ); + assert_eq!(output.value, "other 1"); + assert_eq!( + output + .errors + .iter() + .filter(|error| is_bad_option(error)) + .count(), + 1, + "expected one bad-option diagnostic: {:?}", + output.errors + ); + assert_errors_multiset( + &output.errors, + &[ + function_error(message_format::runtime::MessageFunctionError::BadOption), + message_format::runtime::FormatError::BadSelector { source: None }, + ], + ); +} + // --------------------------------------------------------------------------- // TR35 §14 — :number selection via select= option // --------------------------------------------------------------------------- @@ -166,13 +290,21 @@ fn number_selection_ordinal() { ); } -/// TR35 §14 — dynamic numeric selector mode still prefers exact keys over keywords. +/// Dynamic selector modes remain runtime options and therefore produce a bad +/// selector instead of being rewritten to a synthesized exact selector. #[test] -fn number_selection_dynamic_mode_prefers_exact_key() { - assert_format( +fn number_selection_dynamic_mode_is_not_synthesized() { + let output = format_output( ".input { $mode }\n.input { $x :number select=$mode }\n.match $x\n1 {{EXACT}}\none {{ONE}}\n* {{OTHER}}", &[("mode", Value::Str("plural".into())), ("x", Value::Int(1))], - "EXACT", + ); + assert_eq!(output.value, "OTHER"); + assert_errors_multiset( + &output.errors, + &[ + function_error(message_format::runtime::MessageFunctionError::BadOption), + message_format::runtime::FormatError::BadSelector { source: None }, + ], ); } diff --git a/crates/message-format-conformance/src/tr35/pattern_selection.rs b/crates/message-format-conformance/src/tr35/pattern_selection.rs index e181e10..a3d73fe 100644 --- a/crates/message-format-conformance/src/tr35/pattern_selection.rs +++ b/crates/message-format-conformance/src/tr35/pattern_selection.rs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use super::helpers::*; -use message_format::runtime::{MessageFunctionError, Value}; +use message_format::runtime::{FormatError, MessageFunctionError, Value}; // --------------------------------------------------------------------------- // TR35 §12 — Pattern selection basics (SEL-2, SEL-11) @@ -139,22 +139,34 @@ fn literal_key_quoted_unquoted_equal() { /// E-7/SEL-5 — Selector with missing arg: only catch-all matches. #[test] fn selector_missing_arg_falls_to_catchall() { - assert_format_with_error( + let output = format_output( ".input { $x :string }\n.match $x\na {{A}}\n* {{CATCHALL}}", &[], // no args → selector can't resolve - "CATCHALL", - bad_selector_with_source(missing_arg("x")), + ); + assert_eq!(output.value, "CATCHALL"); + assert_errors_multiset( + &output.errors, + &[ + FormatError::MissingArg("x".into()), + FormatError::BadSelector { source: None }, + ], ); } /// E-7/SEL-5 — Invalid runtime selector options still emit `Bad Selector`. #[test] fn selector_bad_option_falls_to_catchall() { - assert_format_with_error( + let output = format_output( ".input { $mode }\n.input { $x :number select=$mode }\n.match $x\none {{ONE}}\n* {{CATCHALL}}", &[("mode", Value::Str("bogus".into())), ("x", Value::Int(1))], - "CATCHALL", - bad_selector_with_source(function_error(MessageFunctionError::BadOption)), + ); + assert_eq!(output.value, "CATCHALL"); + assert_errors_multiset( + &output.errors, + &[ + function_error(MessageFunctionError::BadOption), + FormatError::BadSelector { source: None }, + ], ); } @@ -166,22 +178,36 @@ fn selector_with_multiple_missing_inputs_reports_all_missing_args() { &[], ); assert_eq!(output.value, "CATCHALL"); - assert!( - output - .errors - .iter() - .any(bad_selector_with_source(missing_arg("x"))), - "missing primary bad-selector diagnostic: {:?}", - output.errors - ); - assert!( - output.errors.contains(&missing_arg("mode")), - "missing secondary missing-arg diagnostic: {:?}", - output.errors - ); - assert!( - output.errors.contains(&missing_arg("digits")), - "missing tertiary missing-arg diagnostic: {:?}", - output.errors + assert_errors_multiset( + &output.errors, + &[ + missing_arg("x"), + missing_arg("mode"), + missing_arg("digits"), + FormatError::BadSelector { source: None }, + ], + ); +} + +/// A fallback propagated through a local is re-annotated using the current +/// variable's fallback representation, without invoking the function host. +#[test] +fn local_fallback_reannotation_uses_current_variable_name() { + let output = format_output( + ".local $a = {$missing}\n.local $b = {$a :number}\n{{{$b}}}", + &[], + ); + assert_eq!(output.value, "{$b}"); + assert_errors_multiset(&output.errors, &[missing_arg("missing")]); +} + +/// Plain local aliases retain their own fallback identity when rendered. +#[test] +fn local_fallback_aliases_use_each_variable_name() { + let output = format_output( + ".local $a = {$missing}\n.local $b = {$a}\n{{{$a} {$b}}}", + &[], ); + assert_eq!(output.value, "{$a} {$b}"); + assert_errors_multiset(&output.errors, &[missing_arg("missing")]); } diff --git a/crates/message-format-conformance/src/tr35/string_fn.rs b/crates/message-format-conformance/src/tr35/string_fn.rs index 0f70d9f..c84115a 100644 --- a/crates/message-format-conformance/src/tr35/string_fn.rs +++ b/crates/message-format-conformance/src/tr35/string_fn.rs @@ -2,7 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use super::helpers::*; -use message_format::runtime::Value; +use message_format::compiler::CompileOptions; +use message_format::runtime::{FormatError, Value}; // --------------------------------------------------------------------------- // TR35 §13 — :string function @@ -78,6 +79,16 @@ fn string_selection_case_sensitive() { ); } +/// Resolved string selection compares raw text without numeric coercion. +#[test] +fn string_selection_does_not_match_numeric_text_canonically() { + assert_format( + ".input { $x :string }\n.match $x\n1 {{MATCH}}\n* {{OTHER}}", + &[("x", Value::Str("1.0".into()))], + "OTHER", + ); +} + // --------------------------------------------------------------------------- // TR35 §13 — :string selection: no normalization (F-6) // --------------------------------------------------------------------------- @@ -103,3 +114,50 @@ fn string_selection_no_normalization() { fn string_missing_operand() { assert_format_err("{ $x :string }", &[], is_missing_arg); } + +/// A string input is resolved once before a numeric reannotation consumes it. +#[test] +fn string_input_reannotates_to_number() { + assert_format( + ".input {$x :string} .local $y = {$x :number} {{value={$y}}}", + &[("x", Value::Int(1))], + "value=1", + ); +} + +/// A missing string input remains a recoverable missing-argument error. +#[test] +fn missing_string_input_reannotation_errors() { + let output = format_output( + ".input {$x :string} .local $y = {$x :number} {{value={$y}}}", + &[], + ); + assert_eq!(output.value, "value={$y}"); + assert_errors_multiset(&output.errors, &[FormatError::MissingArg("x".to_string())]); +} + +/// Quoted string literals can be consumed by a numeric reannotation. +#[test] +fn string_literal_reannotates_to_number() { + assert_format( + ".local $s = {|1| :string} .local $y = {$s :number} {{value={$y}}}", + &[], + "value=1", + ); +} + +/// Direction metadata is applied when the resolved string is rendered. +#[test] +fn string_explicit_direction_is_applied_at_output() { + assert_eq!( + format_with_options_raw( + "{ $x :string u:dir=rtl }", + &[("x", Value::Str("hello".into()))], + CompileOptions { + default_bidi_isolation: false, + ..CompileOptions::default() + }, + ), + "\u{2067}hello\u{2069}" + ); +} diff --git a/crates/message-format/Cargo.toml b/crates/message-format/Cargo.toml index f1f1c19..bba5905 100644 --- a/crates/message-format/Cargo.toml +++ b/crates/message-format/Cargo.toml @@ -20,6 +20,7 @@ icu4x = [ "dep:icu_plurals", "dep:icu_calendar", "dep:icu_datetime", + "dep:libm", ] compile = [ "dep:hashbrown", diff --git a/crates/message-format/src/common/text.rs b/crates/message-format/src/common/text.rs index cce0519..39d3c09 100644 --- a/crates/message-format/src/common/text.rs +++ b/crates/message-format/src/common/text.rs @@ -110,6 +110,7 @@ pub(crate) fn format_signed_string(sign_display: SignDisplay, value: String) -> } } +#[cfg(any(feature = "compile", test))] pub(crate) fn format_signed_number(sign_display: SignDisplay, value: f64) -> String { format_signed_string(sign_display, value.to_string()) } diff --git a/crates/message-format/src/compiler/compile/frontend/bindings.rs b/crates/message-format/src/compiler/compile/frontend/bindings.rs index e1ff5e9..832999e 100644 --- a/crates/message-format/src/compiler/compile/frontend/bindings.rs +++ b/crates/message-format/src/compiler/compile/frontend/bindings.rs @@ -18,6 +18,7 @@ pub(super) struct DeclarationBindings { pub(super) input_aliases: BTreeMap, pub(super) local_functions: BTreeMap, pub(super) input_functions: BTreeMap, + pub(super) slots: BTreeMap, } struct InputDeclarationBindings { @@ -37,14 +38,15 @@ pub(super) enum LocalValue { value: String, kind: OperandLiteralKind, }, - UnknownFunction(String), + Call, + UnknownFunction, } impl LocalValue { pub(super) fn as_literal(&self) -> Option<&str> { match self { Self::Literal { value, .. } => Some(value.as_str()), - Self::UnknownFunction(_) => None, + Self::Call | Self::UnknownFunction => None, } } } @@ -86,6 +88,34 @@ pub(super) fn collect_declaration_bindings( function_origin, &input_bindings.functions, )?; + let mut ordered = declarations + .inputs + .iter() + .filter(|decl| bindings_input_needs_slot(&input_bindings.functions, &decl.canonical)) + .map(|decl| (decl.expr.node.span.start, decl.canonical.as_str())) + .chain( + declarations + .locals + .iter() + .filter(|decl| { + local_analysis + .values + .get(&decl.canonical) + .is_none_or(|value| value.as_literal().is_none()) + }) + .map(|decl| (decl.expr.node.span.start, decl.canonical.as_str())), + ) + .collect::>(); + ordered.sort_by_key(|(start, _)| *start); + let slots = ordered + .into_iter() + .enumerate() + .map(|(slot, (_, name))| { + u32::try_from(slot) + .map(|slot| (name.to_owned(), slot)) + .map_err(|_| CompileError::size_overflow("declaration slots")) + }) + .collect::, _>>()?; Ok(DeclarationBindings { locals: local_analysis.values, @@ -93,9 +123,14 @@ pub(super) fn collect_declaration_bindings( input_aliases: input_bindings.aliases, local_functions: local_analysis.functions, input_functions: input_bindings.functions, + slots, }) } +fn bindings_input_needs_slot(functions: &BTreeMap, name: &str) -> bool { + functions.contains_key(name) +} + fn collect_input_declarations( source: &str, declarations: &crate::compiler::syntax::semantic::CanonicalDeclarationPrelude<'_>, @@ -140,6 +175,13 @@ fn analyze_local_declarations( if let Part::Var(alias) = &parsed { aliases.insert(name.clone(), alias.clone()); + // A local alias still denotes the already-analyzed value. Keep a + // value binding as well so later re-annotations preserve the + // structured numeric call instead of falling back to a runtime + // variable lookup for the alias name. + if let Some(value) = values.get(&canonicalize_identifier(alias)).cloned() { + values.insert(name.clone(), value); + } } if let Some(function) = normalize_declared_function_part(parsed.clone(), input_functions) { @@ -198,13 +240,30 @@ fn evaluate_local_value( return None; } let base = match operand { - Operand::Literal { value, .. } => value, - Operand::Var(var) => known_values - .get(&var) - .and_then(LocalValue::as_literal) - .map(ToOwned::to_owned)?, + Operand::Literal { value, kind } => Some(Operand::Literal { value, kind }), + Operand::Var(var) => match known_values.get(&var) { + Some(LocalValue::Literal { value, kind }) => Some(Operand::Literal { + value: value.clone(), + kind: *kind, + }), + Some(LocalValue::Call) | Some(LocalValue::UnknownFunction) | None => None, + }, + Operand::Local(_) | Operand::Call(_) => None, }; - Some(apply_literal_function(base, &func)) + // Numeric calls and calls over a non-literal declaration remain + // runtime values; folding them would lose exact payload/options. + if matches!( + func.name.as_str(), + "string" | "number" | "integer" | "offset" + ) || base.is_none() + { + Some(LocalValue::Call) + } else { + let Some(Operand::Literal { value, .. }) = base else { + return None; + }; + Some(apply_literal_function(value, &func)) + } } _ => None, } diff --git a/crates/message-format/src/compiler/compile/frontend/document.rs b/crates/message-format/src/compiler/compile/frontend/document.rs index 6603871..5a598df 100644 --- a/crates/message-format/src/compiler/compile/frontend/document.rs +++ b/crates/message-format/src/compiler/compile/frontend/document.rs @@ -12,7 +12,7 @@ use crate::compiler::syntax::span::{SourceContext, byte_to_line_col}; use super::bindings::{DeclarationBindings, collect_declaration_bindings}; use super::pattern::{FunctionOriginContext, lower_pattern_node_to_parts}; use super::raw_match::lower_raw_match_ir; -use super::rewrite::lower_parts_with_declaration_bindings; +use super::rewrite::{lower_declaration_prelude, lower_parts_with_declaration_bindings}; struct AnalyzedSingleMessage<'a> { declarations: crate::compiler::syntax::semantic::CanonicalDeclarationPrelude<'a>, @@ -34,19 +34,31 @@ pub(super) fn parse_single_message( ctx, )? { + let mut parts = lower_raw_match_ir( + source, + ctx, + &analyzed.bindings, + match_prelude, + options, + source_id.map(|source_id| FunctionOriginContext { + source_id, + base_byte: 0, + }), + )?; + let mut declarations = lower_declaration_prelude( + source, + &analyzed.declarations, + &analyzed.bindings, + ctx, + source_id.map(|source_id| FunctionOriginContext { + source_id, + base_byte: 0, + }), + )?; + declarations.append(&mut parts); return Ok(Message { id: String::from("main"), - parts: lower_raw_match_ir( - source, - ctx, - &analyzed.bindings, - match_prelude, - options, - source_id.map(|source_id| FunctionOriginContext { - source_id, - base_byte: 0, - }), - )?, + parts: declarations, origin: source_id.map(|source_id| SourceSpan { source_id, byte_start: 0, @@ -146,7 +158,18 @@ fn preprocess_single_message_parts( }), )?; lower_parts_with_declaration_bindings(&mut parts, &analyzed.bindings, has_declarations)?; - Ok(parts) + let mut declarations = lower_declaration_prelude( + source, + &analyzed.declarations, + &analyzed.bindings, + ctx, + source_id.map(|source_id| FunctionOriginContext { + source_id, + base_byte: 0, + }), + )?; + declarations.append(&mut parts); + Ok(declarations) } fn pattern_context_for_subslice(source: &str, pattern: &str, ctx: SourceContext) -> SourceContext { diff --git a/crates/message-format/src/compiler/compile/frontend/local_eval.rs b/crates/message-format/src/compiler/compile/frontend/local_eval.rs index af60971..07b6bd5 100644 --- a/crates/message-format/src/compiler/compile/frontend/local_eval.rs +++ b/crates/message-format/src/compiler/compile/frontend/local_eval.rs @@ -14,6 +14,9 @@ use crate::common::text::{ }; pub(super) fn local_function_may_fail_select(function_spec: &FunctionSpec) -> bool { + if function_spec_has_dynamic_select_option(function_spec) { + return true; + } match function_spec.name.as_str() { "test:format" => true, "test:select" => function_spec.options.iter().any(|option| { @@ -40,43 +43,24 @@ pub(super) fn normalize_local_function_expression( expression: DeclFunction, input_functions: &BTreeMap, ) -> DeclFunction { - let Some((operand, local_name, _local_has_options)) = function_signature(&expression) else { + let Operand::Var(operand) = &expression.operand else { return expression; }; - let Some(input_expression) = input_functions.get(&operand) else { + let Some(input_expression) = input_functions.get(operand) else { return expression; }; - let Some((_input_operand, input_name, input_has_options)) = - function_signature(input_expression) - else { - return expression; - }; - if local_name == input_name && input_has_options { - input_expression.clone() - } else { - expression - } -} - -pub(super) fn selector_chain_is_unstable( - name: &str, - local_functions: &BTreeMap, -) -> bool { - let Some(expr) = local_functions.get(name) else { - return false; - }; - if function_has_dynamic_select_option(expr) { - return true; + // Keep the local annotation as the outer call while making the input + // declaration's resolved value and options its structured operand. This + // preserves option override semantics when both annotations use the same + // builtin (an earlier implementation discarded the outer options). + DeclFunction { + operand: Operand::Call(Box::new(CallExpr { + operand: input_expression.operand.clone(), + func: input_expression.func.clone(), + fallback: None, + })), + func: expression.func, } - let Some((operand, func_name, _has_options)) = function_signature(expr) else { - return false; - }; - if func_name != "number" && func_name != "integer" { - return false; - } - local_functions - .get(&operand) - .is_some_and(function_has_select_option) } pub(super) fn resolve_alias( @@ -93,31 +77,16 @@ pub(super) fn resolve_alias( Err(CompileError::alias_resolution_overflow(name)) } -fn function_signature(function: &DeclFunction) -> Option<(String, String, bool)> { - let Operand::Var(var) = &function.operand else { - return None; - }; - Some(( - canonicalize_identifier(var), - function.func.name.clone(), - !function.func.options.is_empty(), - )) -} - -fn function_has_select_option(function: &DeclFunction) -> bool { - function - .func - .options - .iter() - .any(|option| option.key == "select") -} - -fn function_has_dynamic_select_option(function: &DeclFunction) -> bool { - function - .func - .options - .iter() - .any(|option| option.key == "select" && matches!(option.value, FunctionOptionValue::Var(_))) +fn function_spec_has_dynamic_select_option(function: &FunctionSpec) -> bool { + function.options.iter().any(|option| { + option.key == "select" + && matches!( + option.value, + FunctionOptionValue::Var(_) + | FunctionOptionValue::ResolvedVar { .. } + | FunctionOptionValue::LocalVar { .. } + ) + }) } pub(super) fn apply_literal_function( @@ -192,7 +161,7 @@ pub(super) fn apply_literal_function( } } _ => { - return LocalValue::UnknownFunction(function_spec.name.clone()); + return LocalValue::UnknownFunction; } } diff --git a/crates/message-format/src/compiler/compile/frontend/matching.rs b/crates/message-format/src/compiler/compile/frontend/matching.rs index ddb3d04..b182cac 100644 --- a/crates/message-format/src/compiler/compile/frontend/matching.rs +++ b/crates/message-format/src/compiler/compile/frontend/matching.rs @@ -143,7 +143,14 @@ pub(super) fn build_nested_match_ir( }; if exact_keys.is_empty() { - return Ok(default); + // A selector is still evaluated when every arm is a default arm. Its + // function may report a diagnostic even though dispatch cannot change + // the selected pattern. + return Ok(vec![Part::Select(SelectExpr { + selector: selectors[level].clone(), + arms: Vec::new(), + default, + })]); } let mut lowered_arms = Vec::new(); @@ -205,7 +212,13 @@ fn build_numeric_match_ir( } if exact_keys.is_empty() && keyword_keys.is_empty() { - return Ok(default); + // Keep selector evaluation observable for a catch-all match. In + // particular, an invalid selector must still produce BadSelector. + return Ok(vec![Part::Select(SelectExpr { + selector: selectors[level].clone(), + arms: Vec::new(), + default, + })]); } let keyword_fallback = if keyword_keys.is_empty() { @@ -232,8 +245,14 @@ fn build_numeric_match_ir( let mut exact_arms = Vec::with_capacity(exact_keys.len()); for key in exact_keys { let branch = exact_match_candidates(arms, candidates, level, &key.key); - let parts = - build_nested_match_ir(selectors, arms, level + 1, &branch, Some(&default), line)?; + let parts = build_nested_match_ir( + selectors, + arms, + level + 1, + &branch, + Some(&keyword_fallback), + line, + )?; exact_arms.push(select_arm_from_key(key, parts)); } @@ -277,7 +296,11 @@ fn build_leaf_match_ir( .ok_or(CompileError::invalid_expr(line))? }; if exact_keys.is_empty() { - return Ok(default); + return Ok(vec![Part::Select(SelectExpr { + selector: selectors[level].clone(), + arms: Vec::new(), + default, + })]); } let mut lowered_arms = Vec::with_capacity(exact_keys.len()); @@ -342,10 +365,19 @@ pub(super) enum BuiltinNumericSelectorMode { pub(super) fn builtin_numeric_selector_mode( selector: &SelectorExpr, ) -> Option { - let SelectorExpr::Call { func, .. } = selector else { - return None; - }; - builtin_numeric_selector_mode_for_func(func) + match selector { + SelectorExpr::Call { func, .. } => builtin_numeric_selector_mode_for_func(func), + SelectorExpr::Local { + func: Some(func), .. + } + | SelectorExpr::CheckedLocal { + func: Some(func), .. + } => builtin_numeric_selector_mode_for_func(func), + SelectorExpr::Var(_) + | SelectorExpr::Local { func: None, .. } + | SelectorExpr::CheckedLocal { func: None, .. } + | SelectorExpr::Literal(_) => None, + } } pub(super) fn builtin_selector_variant_key_expectation( @@ -377,6 +409,9 @@ pub(super) fn builtin_selector_accepts_variant_key(selector: &SelectorExpr, key: fn builtin_numeric_selector_mode_for_func( func: &FunctionSpec, ) -> Option { + if func.name == "offset" { + return Some(BuiltinNumericSelectorMode::Plural); + } if !matches!(func.name.as_str(), "number" | "integer") { return None; } @@ -390,7 +425,11 @@ fn builtin_numeric_selector_mode_for_func( has_select_option = true; match &option.value { FunctionOptionValue::Literal(value) => select_literal = Some(value.as_str()), - FunctionOptionValue::Var(_) => return Some(BuiltinNumericSelectorMode::Dynamic), + FunctionOptionValue::Var(_) + | FunctionOptionValue::ResolvedVar { .. } + | FunctionOptionValue::LocalVar { .. } => { + return Some(BuiltinNumericSelectorMode::Dynamic); + } } } @@ -423,26 +462,66 @@ struct NumericSelectorLoweringPlan { } fn numeric_selector_lowering_plan(selector: &SelectorExpr) -> Option { - let SelectorExpr::Call { operand, func } = selector else { - return None; - }; - match builtin_numeric_selector_mode_for_func(func)? { - BuiltinNumericSelectorMode::Exact => None, - BuiltinNumericSelectorMode::Plural => Some(NumericSelectorLoweringPlan { - exact_selector: clone_numeric_selector_with_select(operand.clone(), func, "exact"), - keyword_selector: clone_numeric_selector_with_select(operand.clone(), func, "plural"), - }), - BuiltinNumericSelectorMode::Ordinal => Some(NumericSelectorLoweringPlan { - exact_selector: clone_numeric_selector_with_select(operand.clone(), func, "exact"), - keyword_selector: clone_numeric_selector_with_select(operand.clone(), func, "ordinal"), - }), - BuiltinNumericSelectorMode::Dynamic => Some(NumericSelectorLoweringPlan { - exact_selector: clone_numeric_selector_with_select(operand.clone(), func, "exact"), - keyword_selector: SelectorExpr::Call { - operand: operand.clone(), - func: func.clone(), - }, - }), + match selector { + SelectorExpr::Call { operand, func } + if matches!(func.name.as_str(), "number" | "integer") => + { + match builtin_numeric_selector_mode_for_func(func)? { + BuiltinNumericSelectorMode::Exact => None, + BuiltinNumericSelectorMode::Plural => Some(NumericSelectorLoweringPlan { + exact_selector: clone_numeric_selector_with_select( + operand.clone(), + func, + "exact", + ), + keyword_selector: clone_numeric_selector_with_select( + operand.clone(), + func, + "plural", + ), + }), + BuiltinNumericSelectorMode::Ordinal => Some(NumericSelectorLoweringPlan { + exact_selector: clone_numeric_selector_with_select( + operand.clone(), + func, + "exact", + ), + keyword_selector: clone_numeric_selector_with_select( + operand.clone(), + func, + "ordinal", + ), + }), + // A variable (or otherwise unknown) select mode must remain part of + // the runtime selector call. Synthesizing `select=exact` here would + // bypass an invalid dynamic option and could select an exact arm even + // though the original selector is required to report BadSelector. + BuiltinNumericSelectorMode::Dynamic => None, + } + } + SelectorExpr::Call { .. } => None, + // A stored numeric value already contains both its exact text and + // plural/ordinal category. Reuse the one loaded local for both passes; + // the nested IR controls whether numeric keys or category keys are + // considered, without re-running the function call. + SelectorExpr::Local { + func: Some(func), .. + } + | SelectorExpr::CheckedLocal { + func: Some(func), .. + } => match builtin_numeric_selector_mode_for_func(func)? { + BuiltinNumericSelectorMode::Plural | BuiltinNumericSelectorMode::Ordinal => { + Some(NumericSelectorLoweringPlan { + exact_selector: selector.clone(), + keyword_selector: selector.clone(), + }) + } + BuiltinNumericSelectorMode::Exact | BuiltinNumericSelectorMode::Dynamic => None, + }, + SelectorExpr::Var(_) + | SelectorExpr::Local { func: None, .. } + | SelectorExpr::CheckedLocal { func: None, .. } + | SelectorExpr::Literal(_) => None, } } diff --git a/crates/message-format/src/compiler/compile/frontend/raw_match.rs b/crates/message-format/src/compiler/compile/frontend/raw_match.rs index ce9e6f8..0a5a316 100644 --- a/crates/message-format/src/compiler/compile/frontend/raw_match.rs +++ b/crates/message-format/src/compiler/compile/frontend/raw_match.rs @@ -8,18 +8,17 @@ use crate::compiler::semantic::SelectorExpr; use crate::compiler::syntax::span::byte_to_line_col; use super::bindings::{DeclFunction, DeclarationBindings, LocalValue}; -use super::local_eval::{resolve_alias, selector_chain_is_unstable}; +use super::local_eval::resolve_alias; use super::matching::{ LoweredMatchArm, MatchArm, build_nested_match_ir, builtin_selector_accepts_variant_key, builtin_selector_variant_key_expectation, lower_match_declaration_prelude, }; use super::pattern::{FunctionOriginContext, lower_pattern_node_to_parts}; -use super::rewrite::lower_parts_with_declaration_bindings; +use super::rewrite::{lower_parts_with_declaration_bindings, rewrite_selector_expr_from_locals}; struct AnalyzedSelectors { parts: Vec, compile_time_values: Option>, - has_unstable_chain: bool, } pub(super) fn lower_raw_match_ir( @@ -55,17 +54,27 @@ pub(super) fn lower_raw_match_ir( &parsed_match.duplicate_keys, )?; - if selectors.has_unstable_chain { - return default_arm_parts(source, ctx, &arms); - } - if let Some(selector_values) = selectors.compile_time_values.as_ref() { return resolve_compile_time_arm(source, ctx, selector_values, &arms); } let (line, _) = ctx.location(source, 0); let candidates = (0..arms.len()).collect::>(); - build_nested_match_ir(&selectors.parts, &arms, 0, &candidates, None, line) + let mut dispatch_selectors = selectors.parts; + let mut checks = Vec::new(); + for selector in &mut dispatch_selectors { + let SelectorExpr::Local { slot, func } = selector else { + continue; + }; + let slot = *slot; + let func = func.clone(); + checks.push(Part::CheckSelector(slot)); + *selector = SelectorExpr::CheckedLocal { slot, func }; + } + let mut dispatch = + build_nested_match_ir(&dispatch_selectors, &arms, 0, &candidates, None, line)?; + checks.append(&mut dispatch); + Ok(checks) } fn annotate_match_key_origins( @@ -100,11 +109,9 @@ fn analyze_selectors( let mut parts = Vec::with_capacity(selectors.len()); let mut compile_time_values = Vec::with_capacity(selectors.len()); let mut all_compile_time = true; - let mut has_unstable_chain = false; for selector in selectors { let analyzed = analyze_selector(source, ctx, selector, bindings)?; - has_unstable_chain |= analyzed.is_unstable; all_compile_time &= analyzed.compile_time_value.is_some(); compile_time_values.push(analyzed.compile_time_value.unwrap_or_default()); parts.push(analyzed.part); @@ -113,14 +120,12 @@ fn analyze_selectors( Ok(AnalyzedSelectors { parts, compile_time_values: all_compile_time.then_some(compile_time_values), - has_unstable_chain, }) } struct AnalyzedSelector { part: SelectorExpr, compile_time_value: Option, - is_unstable: bool, } fn analyze_selector( @@ -135,13 +140,34 @@ fn analyze_selector( .get(&name) .and_then(LocalValue::as_literal) .map(ToOwned::to_owned); - let part = bindings + let mut part = bindings .local_functions .get(&name) .cloned() .or_else(|| bindings.input_functions.get(&name).cloned()) .map(selector_expr_from_decl_function) .unwrap_or_else(|| SelectorExpr::Var(name.clone())); + if let Some(slot) = bindings.slots.get(&name).copied() + && bindings + .locals + .get(&name) + .is_none_or(|value| value.as_literal().is_none()) + { + if let Some(function) = bindings + .local_functions + .get(&name) + .or_else(|| bindings.input_functions.get(&name)) + { + part = SelectorExpr::Local { + slot, + func: Some(function.func.clone()), + }; + } else { + part = SelectorExpr::Local { slot, func: None }; + } + } else { + rewrite_selector_expr_from_locals(&mut part, &bindings.locals, &bindings.slots); + } if matches!(part, SelectorExpr::Var(_)) { let (line, col) = ctx.location(source, 0); return Err(CompileError::missing_selector_annotation_detail( @@ -153,7 +179,6 @@ fn analyze_selector( } Ok(AnalyzedSelector { compile_time_value: literal.filter(|value| value.is_ascii()), - is_unstable: selector_chain_is_unstable(&name, &bindings.local_functions), part, }) } @@ -168,6 +193,11 @@ fn selector_expr_from_decl_function(function: DeclFunction) -> SelectorExpr { operand: Operand::Literal { value, kind }, func: function.func, }, + Operand::Call(call) => SelectorExpr::Call { + operand: Operand::Call(call), + func: function.func, + }, + Operand::Local(slot) => SelectorExpr::Local { slot, func: None }, } } @@ -239,20 +269,6 @@ fn validate_builtin_selector_variant_keys( Ok(()) } -fn default_arm_parts( - source: &str, - ctx: SourceContext, - arms: &[LoweredMatchArm], -) -> Result, CompileError> { - arms.iter() - .find(|arm| arm.is_default()) - .map(|arm| arm.parts.clone()) - .ok_or_else(|| { - let (line, _) = ctx.location(source, 0); - CompileError::missing_default_arm(line) - }) -} - fn resolve_compile_time_arm( source: &str, ctx: SourceContext, diff --git a/crates/message-format/src/compiler/compile/frontend/rewrite.rs b/crates/message-format/src/compiler/compile/frontend/rewrite.rs index 1c05a5e..e8bdcba 100644 --- a/crates/message-format/src/compiler/compile/frontend/rewrite.rs +++ b/crates/message-format/src/compiler/compile/frontend/rewrite.rs @@ -10,7 +10,7 @@ use alloc::{ use super::*; use crate::compiler::semantic::SelectorExpr; -use super::bindings::{DeclarationBindings, LocalValue}; +use super::bindings::{DeclFunction, DeclarationBindings, LocalValue}; use super::local_eval::resolve_alias; pub(super) fn lower_parts_with_declaration_bindings( @@ -19,37 +19,105 @@ pub(super) fn lower_parts_with_declaration_bindings( repeat_local_pass_after_alias: bool, ) -> Result<(), CompileError> { walk_parts_mut(parts, &bindings.locals, &mut |part| { - lower_part_with_bindings(part, bindings, repeat_local_pass_after_alias) + lower_part_with_bindings(part, bindings, repeat_local_pass_after_alias, None) }) } +pub(super) fn lower_declaration_prelude( + source: &str, + declarations: &crate::compiler::syntax::semantic::CanonicalDeclarationPrelude<'_>, + bindings: &DeclarationBindings, + ctx: SourceContext, + function_origin: Option, +) -> Result, CompileError> { + let mut ordered = declarations + .inputs + .iter() + .map(|decl| { + ( + decl.expr.node.span.start, + decl.canonical.as_str(), + &decl.expr.node, + ) + }) + .chain(declarations.locals.iter().map(|decl| { + ( + decl.expr.node.span.start, + decl.canonical.as_str(), + &decl.expr.node, + ) + })) + .collect::>(); + ordered.sort_by_key(|(start, _, _)| *start); + + let mut parts = Vec::with_capacity(ordered.len()); + for (_, name, expression) in ordered { + let Some(slot) = bindings.slots.get(name).copied() else { + continue; + }; + let mut value = lower_expression_node_to_part(source, expression, ctx, function_origin)?; + if let Part::Call(call) = &mut value { + call.fallback = Some(format!("{{${name}}}")); + } + lower_part_with_bindings(&mut value, bindings, false, Some(name))?; + parts.push(Part::Bind { + slot, + fallback: format!("{{${name}}}"), + value: Box::new(value), + }); + } + Ok(parts) +} + fn lower_part_with_bindings( part: &mut Part, bindings: &DeclarationBindings, repeat_local_pass_after_alias: bool, + excluded: Option<&str>, ) -> Result<(), CompileError> { - rewrite_dynamic_option_vars_from_locals(part, &bindings.locals); + rewrite_dynamic_option_vars_from_locals(part, &bindings.locals, &bindings.slots); match part { Part::Var(var) => { let canonical = canonicalize_identifier(var); - if let Some(local) = bindings.locals.get(&canonical) { - *part = local_value_to_var_part(local, &canonical); + let _ = resolve_alias(&canonical, &bindings.aliases)?; + if excluded == Some(canonical.as_str()) { + return Ok(()); + } + if excluded != Some(canonical.as_str()) + && slot_is_runtime(bindings, &canonical) + && let Some(slot) = bindings.slots.get(&canonical) + { + *part = Part::Local(*slot); + return Ok(()); + } + if let Some(LocalValue::Literal { value, .. }) = bindings.locals.get(&canonical) { + *part = Part::Literal(value.clone()); return Ok(()); } let aliased = resolve_alias(&canonical, &bindings.aliases)?; - if repeat_local_pass_after_alias && let Some(local) = bindings.locals.get(&aliased) { - *part = local_value_to_var_part(local, &aliased); + if excluded != Some(aliased.as_str()) + && slot_is_runtime(bindings, &aliased) + && let Some(slot) = bindings.slots.get(&aliased) + { + *part = Part::Local(*slot); return Ok(()); } - if let Some(function) = bindings - .local_functions - .get(&aliased) - .cloned() - .or_else(|| bindings.input_functions.get(&aliased).cloned()) + if repeat_local_pass_after_alias + && let Some(LocalValue::Literal { value, .. }) = bindings.locals.get(&aliased) { + *part = Part::Literal(value.clone()); + return Ok(()); + } + if let Some(function) = bindings.input_functions.get(&aliased).cloned() { let fb = format!("{{${aliased}}}"); - *part = function.into_part(Some(fb)); + let mut lowered = function.into_part(Some(fb)); + rewrite_dynamic_option_vars_from_locals( + &mut lowered, + &bindings.locals, + &bindings.slots, + ); + *part = lowered; return Ok(()); } if let Some(declared) = bindings.input_aliases.get(&aliased) { @@ -64,14 +132,70 @@ fn lower_part_with_bindings( .. }) => { let canonical = canonicalize_identifier(var); - if let Some(local) = bindings.locals.get(&canonical) { - *part = local_value_to_call_part(local, func, &canonical); + let _ = resolve_alias(&canonical, &bindings.aliases)?; + if excluded == Some(canonical.as_str()) { + return Ok(()); + } + if excluded != Some(canonical.as_str()) + && slot_is_runtime(bindings, &canonical) + && let Some(slot) = bindings.slots.get(&canonical) + { + *part = Part::Call(CallExpr { + operand: Operand::Local(*slot), + func: func.clone(), + fallback: Some(format!("{{${canonical}}}")), + }); + return Ok(()); + } + if let Some(LocalValue::Literal { value, kind }) = bindings.locals.get(&canonical) { + *part = Part::Call(CallExpr { + operand: Operand::Literal { + value: value.clone(), + kind: *kind, + }, + func: func.clone(), + fallback: Some(format!("{{${canonical}}}")), + }); return Ok(()); } let aliased = resolve_alias(&canonical, &bindings.aliases)?; - if repeat_local_pass_after_alias && let Some(local) = bindings.locals.get(&aliased) { - *part = local_value_to_call_part(local, func, &aliased); + if excluded != Some(aliased.as_str()) + && slot_is_runtime(bindings, &aliased) + && let Some(slot) = bindings.slots.get(&aliased) + { + *part = Part::Call(CallExpr { + operand: Operand::Local(*slot), + func: func.clone(), + fallback: Some(format!("{{${aliased}}}")), + }); + return Ok(()); + } + if repeat_local_pass_after_alias + && let Some(LocalValue::Literal { value, kind }) = bindings.locals.get(&aliased) + { + *part = Part::Call(CallExpr { + operand: Operand::Literal { + value: value.clone(), + kind: *kind, + }, + func: func.clone(), + fallback: Some(format!("{{${aliased}}}")), + }); + return Ok(()); + } + if let Some(function) = bindings.input_functions.get(&aliased).cloned() { + let mut lowered = Part::Call(CallExpr { + operand: input_function_operand(function), + func: func.clone(), + fallback: Some(format!("{{${aliased}}}")), + }); + rewrite_dynamic_option_vars_from_locals( + &mut lowered, + &bindings.locals, + &bindings.slots, + ); + *part = lowered; return Ok(()); } if let Some(declared) = bindings.input_aliases.get(&aliased) { @@ -85,69 +209,80 @@ fn lower_part_with_bindings( Ok(()) } -fn local_value_to_var_part(value: &LocalValue, var_name: &str) -> Part { - match value { - LocalValue::Literal { value, .. } => Part::Literal(value.clone()), - LocalValue::UnknownFunction(name) => Part::Call(CallExpr { - operand: Operand::literal(String::new()), - func: FunctionSpec { - name: name.clone(), - options: Vec::new(), - origin: None, - }, - fallback: Some(format!("{{${var_name}}}")), - }), - } +fn slot_is_runtime(bindings: &DeclarationBindings, name: &str) -> bool { + bindings + .locals + .get(name) + .is_none_or(|value| value.as_literal().is_none()) } -fn local_value_to_call_part(value: &LocalValue, func: &FunctionSpec, var_name: &str) -> Part { - match value { - LocalValue::Literal { value, kind } => Part::Call(CallExpr { - operand: Operand::Literal { - value: value.clone(), - kind: *kind, - }, - func: func.clone(), - fallback: Some(format!("{{${var_name}}}")), - }), - LocalValue::UnknownFunction(name) => Part::Call(CallExpr { - operand: Operand::literal(String::new()), - func: FunctionSpec { - name: name.clone(), - options: func.options.clone(), - origin: func.origin.clone(), - }, - fallback: Some(format!("{{${var_name}}}")), - }), - } +fn input_function_operand(function: DeclFunction) -> Operand { + Operand::Call(Box::new(CallExpr { + operand: function.operand, + func: function.func, + fallback: None, + })) } -fn rewrite_dynamic_option_vars_from_locals(part: &mut Part, locals: &BTreeMap) { - let func = match part { - Part::Call(CallExpr { func, .. }) => func, - _ => return, - }; +fn rewrite_dynamic_option_vars_from_locals( + part: &mut Part, + locals: &BTreeMap, + slots: &BTreeMap, +) { + let Part::Call(call) = part else { return }; + rewrite_call_options_from_locals(call, locals, slots); +} + +fn rewrite_call_options_from_locals( + call: &mut CallExpr, + locals: &BTreeMap, + slots: &BTreeMap, +) { + let func = &mut call.func; for option in &mut func.options { let FunctionOptionValue::Var(var) = &option.value else { continue; }; let Some(local_value) = locals.get(var).and_then(LocalValue::as_literal) else { + if let Some(slot) = slots.get(var) { + option.value = FunctionOptionValue::LocalVar { + name: var.clone(), + slot: *slot, + }; + } continue; }; + if option.key == "select" { + // Keep the option dynamic so invalid values are diagnosed by the + // builtin at runtime, while carrying the local payload because a + // local declaration is not a message argument. + option.value = FunctionOptionValue::ResolvedVar { + name: var.clone(), + value: local_value.to_string(), + }; + continue; + } // Function option literals are stored as decoded semantic text, not raw // source tokens, so local substitution can stay allocation-light and // avoid reintroducing a quoted-literal protocol here. option.value = FunctionOptionValue::Literal(local_value.to_string()); } + if let Operand::Call(nested) = &mut call.operand { + rewrite_call_options_from_locals(nested, locals, slots); + } } -fn rewrite_selector_expr_from_locals( +pub(super) fn rewrite_selector_expr_from_locals( selector: &mut SelectorExpr, locals: &BTreeMap, + slots: &BTreeMap, ) { - let func = match selector { - SelectorExpr::Call { func, .. } => func, - SelectorExpr::Var(_) | SelectorExpr::Literal(_) => return, + let (operand, func) = match selector { + SelectorExpr::Call { operand, func } => (operand, func), + SelectorExpr::Var(_) + | SelectorExpr::Local { .. } + | SelectorExpr::CheckedLocal { .. } + | SelectorExpr::Literal(_) => return, }; for option in &mut func.options { let FunctionOptionValue::Var(var) = &option.value else { @@ -156,8 +291,18 @@ fn rewrite_selector_expr_from_locals( let Some(local_value) = locals.get(var).and_then(LocalValue::as_literal) else { continue; }; + if option.key == "select" { + option.value = FunctionOptionValue::ResolvedVar { + name: var.clone(), + value: local_value.to_string(), + }; + continue; + } option.value = FunctionOptionValue::Literal(local_value.to_string()); } + if let Operand::Call(nested) = operand { + rewrite_call_options_from_locals(nested, locals, slots); + } } fn walk_parts_mut( @@ -173,7 +318,7 @@ fn walk_parts_mut( default, }) = part { - rewrite_selector_expr_from_locals(selector, locals); + rewrite_selector_expr_from_locals(selector, locals, &BTreeMap::new()); for arm in arms { walk_parts_mut(&mut arm.parts, locals, f)?; } diff --git a/crates/message-format/src/compiler/compile/interning.rs b/crates/message-format/src/compiler/compile/interning.rs index 40dca67..651546b 100644 --- a/crates/message-format/src/compiler/compile/interning.rs +++ b/crates/message-format/src/compiler/compile/interning.rs @@ -49,7 +49,16 @@ pub(super) fn collect_strings(messages: &[Message], out: &mut BTreeSet) fn collect_parts_strings(parts: &[Part], out: &mut BTreeSet) { for part in parts { match part { - Part::Text(_) | Part::Literal(_) => {} + Part::Text(_) | Part::Local(_) | Part::CheckSelector(_) => {} + Part::Literal(value) => { + out.insert(value.clone()); + } + Part::Bind { + fallback, value, .. + } => { + out.insert(fallback.clone()); + collect_parts_strings(core::slice::from_ref(value), out); + } Part::Var(name) => { out.insert(name.clone()); } @@ -89,6 +98,13 @@ fn collect_parts_strings(parts: &[Part], out: &mut BTreeSet) { FunctionOptionValue::Var(var) => { out.insert(var.clone()); } + FunctionOptionValue::ResolvedVar { name, value } => { + out.insert(name.clone()); + out.insert(value.clone()); + } + FunctionOptionValue::LocalVar { name, .. } => { + out.insert(name.clone()); + } } } } @@ -109,6 +125,13 @@ fn collect_function_strings(func: &FunctionSpec, out: &mut BTreeSet) { FunctionOptionValue::Var(var) => { out.insert(var.clone()); } + FunctionOptionValue::ResolvedVar { name, value } => { + out.insert(name.clone()); + out.insert(value.clone()); + } + FunctionOptionValue::LocalVar { name, .. } => { + out.insert(name.clone()); + } } } } @@ -135,9 +158,13 @@ fn collect_parts_functions( ) -> Result<(), CompileError> { for part in parts { match part { - Part::Call(CallExpr { func, .. }) => { + Part::Call(CallExpr { operand, func, .. }) => { + collect_operand_functions(operand, func_map, entries)?; register_function(func, func_map, entries)?; } + Part::Bind { value, .. } => { + collect_parts_functions(core::slice::from_ref(value), func_map, entries)?; + } Part::Select(SelectExpr { selector, arms, @@ -160,6 +187,7 @@ fn collect_selector_strings(selector: &SelectorExpr, out: &mut BTreeSet) SelectorExpr::Var(name) => { out.insert(name.clone()); } + SelectorExpr::Local { .. } | SelectorExpr::CheckedLocal { .. } => {} SelectorExpr::Call { operand, func } => { collect_operand_strings(operand, out); collect_function_strings(func, out); @@ -176,8 +204,14 @@ fn collect_selector_functions( entries: &mut Vec, ) -> Result<(), CompileError> { match selector { - SelectorExpr::Call { func, .. } => register_function(func, func_map, entries), - SelectorExpr::Var(_) | SelectorExpr::Literal(_) => Ok(()), + SelectorExpr::Call { operand, func } => { + collect_operand_functions(operand, func_map, entries)?; + register_function(func, func_map, entries) + } + SelectorExpr::Var(_) + | SelectorExpr::Local { .. } + | SelectorExpr::CheckedLocal { .. } + | SelectorExpr::Literal(_) => Ok(()), } } @@ -186,17 +220,44 @@ fn collect_operand_strings(operand: &Operand, out: &mut BTreeSet) { Operand::Var(value) | Operand::Literal { value, .. } => { out.insert(value.clone()); } + Operand::Local(_) => {} + Operand::Call(call) => { + collect_operand_strings(&call.operand, out); + collect_function_strings(&call.func, out); + if let Some(fallback) = &call.fallback { + out.insert(fallback.clone()); + } else { + out.insert(render_call_fallback(&call.operand, &call.func)); + } + } } } fn render_call_fallback(operand: &Operand, func: &FunctionSpec) -> String { match operand { Operand::Var(var) => format!("{{${var}}}"), + Operand::Local(slot) => format!("{{}}"), Operand::Literal { value, .. } if value.is_empty() => format!("{{:{}}}", func.name), Operand::Literal { value, .. } => format!("{{|{}|}}", escape_fallback_literal(value)), + Operand::Call(call) => call + .fallback + .clone() + .unwrap_or_else(|| render_call_fallback(&call.operand, &call.func)), } } +fn collect_operand_functions( + operand: &Operand, + func_map: &mut BTreeMap, + entries: &mut Vec, +) -> Result<(), CompileError> { + if let Operand::Call(call) = operand { + collect_operand_functions(&call.operand, func_map, entries)?; + register_function(&call.func, func_map, entries)?; + } + Ok(()) +} + fn register_function( func: &FunctionSpec, func_map: &mut BTreeMap, @@ -214,7 +275,9 @@ fn register_function( .iter() .filter_map(|opt| match &opt.value { FunctionOptionValue::Literal(value) => Some((opt.key.clone(), value.clone())), - FunctionOptionValue::Var(_) => None, + FunctionOptionValue::Var(_) + | FunctionOptionValue::ResolvedVar { .. } + | FunctionOptionValue::LocalVar { .. } => None, }) .collect(); @@ -238,6 +301,12 @@ pub(crate) fn function_catalog_key(func: &FunctionSpec) -> FunctionCatalogKey { FunctionCatalogOptionValue::Literal(value.clone()) } FunctionOptionValue::Var(var) => FunctionCatalogOptionValue::Var(var.clone()), + FunctionOptionValue::ResolvedVar { name, .. } => { + FunctionCatalogOptionValue::Var(name.clone()) + } + FunctionOptionValue::LocalVar { name, .. } => { + FunctionCatalogOptionValue::Var(name.clone()) + } }, }) .collect(), @@ -248,11 +317,27 @@ pub(super) fn escape_fallback_literal(value: &str) -> String { value.replace('\\', "\\\\").replace('|', "\\|") } -pub(super) fn function_dynamic_options(func: &FunctionSpec) -> Vec<(&str, &str)> { +pub(super) fn function_dynamic_options( + func: &FunctionSpec, +) -> Vec<(&str, &str, Option, Option<&str>)> { let mut out = Vec::new(); for option in &func.options { - if let FunctionOptionValue::Var(var) = &option.value { - out.push((option.key.as_str(), var.as_str())); + match &option.value { + FunctionOptionValue::Var(var) => { + out.push((option.key.as_str(), var.as_str(), None, None)); + } + FunctionOptionValue::ResolvedVar { name, value } => { + out.push(( + option.key.as_str(), + name.as_str(), + None, + Some(value.as_str()), + )); + } + FunctionOptionValue::LocalVar { name, slot } => { + out.push((option.key.as_str(), name.as_str(), Some(*slot), None)); + } + FunctionOptionValue::Literal(_) => {} } } out diff --git a/crates/message-format/src/compiler/compile/lowering.rs b/crates/message-format/src/compiler/compile/lowering.rs index dabbe8d..f2f7158 100644 --- a/crates/message-format/src/compiler/compile/lowering.rs +++ b/crates/message-format/src/compiler/compile/lowering.rs @@ -130,44 +130,65 @@ fn hash_str(builder: &DefaultHashBuilder, value: &str) -> u64 { builder.hash_one(value) } -/// Compute the fallback string for a call part. -fn compute_fallback(part: &Part) -> String { - match part { - Part::Call(CallExpr { - operand, - func, - fallback, - }) => fallback - .clone() - .unwrap_or_else(|| render_operand_fallback(operand, func)), - _ => String::new(), - } -} - -/// Emit an `ExprFallback` instruction before a call in the output path. -fn emit_expr_fallback( - part: &Part, +pub(super) fn lower_parts( + parts: &[Part], string_map: &BTreeMap, + func_map: &BTreeMap, + literals: &mut LiteralPool, code: &mut Vec, ) -> Result<(), CompileError> { - let fb = compute_fallback(part); - let fb_str_id = *string_map - .get(&fb) - .ok_or(CompileError::internal("missing interned string"))?; - code.push(schema::Opcode::ExprFallback as u8); - code.extend_from_slice(&fb_str_id.to_le_bytes()); - Ok(()) + let mut state = LoweringState::default(); + lower_parts_inner(parts, string_map, func_map, literals, code, &mut state).map(|_| ()) } -pub(super) fn lower_parts( - parts: &[Part], +struct DefaultContinuation<'a> { + parts: &'a [Part], + depth: usize, + jump_sites: Vec, +} + +#[derive(Default)] +struct LoweringState<'a> { + defaults: Vec>, + depth: usize, +} + +fn lower_parts_inner<'a>( + parts: &'a [Part], string_map: &BTreeMap, func_map: &BTreeMap, literals: &mut LiteralPool, code: &mut Vec, -) -> Result<(), CompileError> { + state: &mut LoweringState<'a>, +) -> Result { + if let Some((default_index, continuation)) = state + .defaults + .iter() + .enumerate() + .rev() + .find(|(_, continuation)| continuation.parts == parts) + { + let unwind = state + .depth + .checked_sub(continuation.depth) + .ok_or(CompileError::internal( + "default continuation outside selector scope", + ))?; + for _ in 0..unwind { + code.push(schema::Opcode::SelectEnd as u8); + } + code.push(schema::Opcode::Jmp as u8); + let rel_pos = code.len(); + code.extend_from_slice(&0_i32.to_le_bytes()); + state.defaults[default_index].jump_sites.push(rel_pos); + return Ok(true); + } for part in parts { match part { + Part::CheckSelector(slot) => { + code.push(schema::Opcode::CheckSelector as u8); + code.extend_from_slice(&slot.to_le_bytes()); + } Part::Text(value) => { let (off, len) = literals.intern(value)?; code.push(schema::Opcode::OutSlice as u8); @@ -187,34 +208,13 @@ pub(super) fn lower_parts( code.push(schema::Opcode::OutArg as u8); code.extend_from_slice(&str_id.to_le_bytes()); } - Part::Call(CallExpr { operand, func, .. }) => { - let func_key = function_catalog_key(func); - let fn_id = *func_map - .get(&func_key) - .ok_or(CompileError::internal("missing function entry"))?; - let dynamic_options = function_dynamic_options(func); - - emit_operand(operand, string_map, code)?; - for (key, value) in dynamic_options.iter().copied() { - let key_str_id = *string_map - .get(key) - .ok_or(CompileError::internal("missing interned string"))?; - code.push(schema::Opcode::PushConst as u8); - code.extend_from_slice(&key_str_id.to_le_bytes()); - let var_str_id = *string_map - .get(value) - .ok_or(CompileError::internal("missing interned variable"))?; - code.push(schema::Opcode::LoadArg as u8); - code.extend_from_slice(&var_str_id.to_le_bytes()); - } - emit_expr_fallback(part, string_map, code)?; - code.push(schema::Opcode::CallFunc as u8); - code.extend_from_slice(&fn_id.to_le_bytes()); - code.push(1); - code.push( - u8::try_from(dynamic_options.len()) - .map_err(|_| CompileError::size_overflow("option count"))?, - ); + Part::Local(slot) => { + code.push(schema::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + code.push(schema::Opcode::OutVal as u8); + } + Part::Call(call) => { + emit_call(call, string_map, func_map, code)?; code.push(schema::Opcode::OutVal as u8); } Part::MarkupOpen { name, options } => { @@ -242,11 +242,65 @@ pub(super) fn lower_parts( ); } Part::Select(select) => { - lower_select(select, string_map, func_map, literals, code)?; + lower_select(select, string_map, func_map, literals, code, state)?; + } + Part::Bind { + slot, + fallback, + value, + } => { + emit_value_part(value, string_map, func_map, literals, code)?; + let fallback_id = *string_map + .get(fallback) + .ok_or(CompileError::internal("missing declaration fallback"))?; + code.push(schema::Opcode::ExprFallback as u8); + code.extend_from_slice(&fallback_id.to_le_bytes()); + code.push(schema::Opcode::StoreLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); } } } + Ok(false) +} + +fn emit_value_part( + part: &Part, + string_map: &BTreeMap, + func_map: &BTreeMap, + literals: &mut LiteralPool, + code: &mut Vec, +) -> Result<(), CompileError> { + match part { + Part::Literal(value) => { + let value_id = *string_map.get(value).ok_or(CompileError::internal( + "missing interned declaration literal", + ))?; + code.push(schema::Opcode::PushConst as u8); + code.extend_from_slice(&value_id.to_le_bytes()); + } + Part::Var(name) => { + let value_id = *string_map.get(name).ok_or(CompileError::internal( + "missing interned declaration variable", + ))?; + code.push(schema::Opcode::LoadArg as u8); + code.extend_from_slice(&value_id.to_le_bytes()); + } + Part::Local(slot) => { + code.push(schema::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + } + Part::Call(call) => emit_call(call, string_map, func_map, code)?, + Part::Bind { .. } + | Part::CheckSelector(_) + | Part::Text(_) + | Part::Select(_) + | Part::MarkupOpen { .. } + | Part::MarkupClose { .. } => { + return Err(CompileError::internal("non-scalar declaration expression")); + } + } + let _ = literals; Ok(()) } @@ -276,17 +330,29 @@ fn emit_markup_options( code.push(schema::Opcode::LoadArg as u8); code.extend_from_slice(&var_str_id.to_le_bytes()); } + FunctionOptionValue::ResolvedVar { value, .. } => { + let value_str_id = *string_map + .get(value) + .ok_or(CompileError::internal("missing interned string"))?; + code.push(schema::Opcode::PushConst as u8); + code.extend_from_slice(&value_str_id.to_le_bytes()); + } + FunctionOptionValue::LocalVar { slot, .. } => { + code.push(schema::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + } } } Ok(()) } -fn lower_select( - select: &SelectExpr, +fn lower_select<'a>( + select: &'a SelectExpr, string_map: &BTreeMap, func_map: &BTreeMap, literals: &mut LiteralPool, code: &mut Vec, + state: &mut LoweringState<'a>, ) -> Result<(), CompileError> { emit_selector_start(&select.selector, string_map, func_map, code)?; @@ -306,25 +372,41 @@ fn lower_select( let default_rel_pos = code.len(); code.extend_from_slice(&0_i32.to_le_bytes()); + state.defaults.push(DefaultContinuation { + parts: &select.default, + depth: state.depth + 1, + jump_sites: Vec::new(), + }); + state.depth += 1; let mut arm_starts = vec![0_u32; select.arms.len()]; let mut end_jump_patch_positions = Vec::new(); for (arm_idx, arm) in select.arms.iter().enumerate() { arm_starts[arm_idx] = u32::try_from(code.len()) .map_err(|_| CompileError::size_overflow("bytecode program counter"))?; - lower_parts(&arm.parts, string_map, func_map, literals, code)?; - code.push(schema::Opcode::Jmp as u8); - let rel_pos = code.len(); - code.extend_from_slice(&0_i32.to_le_bytes()); - end_jump_patch_positions.push(rel_pos); + let forwarded = lower_parts_inner(&arm.parts, string_map, func_map, literals, code, state)?; + if !forwarded { + code.push(schema::Opcode::Jmp as u8); + let rel_pos = code.len(); + code.extend_from_slice(&0_i32.to_le_bytes()); + end_jump_patch_positions.push(rel_pos); + } } let default_start = u32::try_from(code.len()) .map_err(|_| CompileError::size_overflow("bytecode program counter"))?; - lower_parts(&select.default, string_map, func_map, literals, code)?; + let current_default = state + .defaults + .pop() + .expect("select continuation is registered before its arms"); + for rel_pos in current_default.jump_sites { + patch_rel32(code, rel_pos, default_start)?; + } + lower_parts_inner(&select.default, string_map, func_map, literals, code, state)?; let end_pc = u32::try_from(code.len()) .map_err(|_| CompileError::size_overflow("bytecode program counter"))?; code.push(schema::Opcode::SelectEnd as u8); + state.depth -= 1; for (rel_pos, arm_idx) in dispatch_patches { patch_rel32(code, rel_pos, arm_starts[arm_idx])?; @@ -344,6 +426,17 @@ fn emit_selector_start( code: &mut Vec, ) -> Result<(), CompileError> { match selector { + SelectorExpr::Local { slot, .. } => { + code.push(schema::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + code.push(schema::Opcode::SelectBegin as u8); + Ok(()) + } + SelectorExpr::CheckedLocal { slot, .. } => { + code.push(schema::Opcode::SelectLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + Ok(()) + } SelectorExpr::Var(name) => { let str_id = *string_map .get(name) @@ -378,13 +471,21 @@ fn lower_selector( code: &mut Vec, ) -> Result<(), CompileError> { match selector { + SelectorExpr::Local { slot, .. } => { + code.push(schema::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + Ok(()) + } + SelectorExpr::CheckedLocal { .. } => Err(CompileError::internal( + "checked local selector requires selector-start lowering", + )), SelectorExpr::Var(name) => { - emit_operand(&Operand::Var(name.clone()), string_map, code)?; + emit_operand(&Operand::Var(name.clone()), string_map, func_map, code)?; Ok(()) } SelectorExpr::Call { operand, func } => { if func.name == "string" && func.options.is_empty() { - emit_operand(operand, string_map, code)?; + emit_operand(operand, string_map, func_map, code)?; return Ok(()); } let func_key = function_catalog_key(func); @@ -392,18 +493,27 @@ fn lower_selector( .get(&func_key) .ok_or(CompileError::internal("missing function entry"))?; let dynamic_options = function_dynamic_options(func); - emit_operand(operand, string_map, code)?; - for (key, value) in dynamic_options.iter().copied() { + emit_operand(operand, string_map, func_map, code)?; + for (key, value, local, resolved) in dynamic_options.iter().copied() { let key_str_id = *string_map .get(key) .ok_or(CompileError::internal("missing interned string"))?; code.push(schema::Opcode::PushConst as u8); code.extend_from_slice(&key_str_id.to_le_bytes()); - let value_var_str_id = *string_map - .get(value) - .ok_or(CompileError::internal("missing interned variable"))?; - code.push(schema::Opcode::LoadArg as u8); - code.extend_from_slice(&value_var_str_id.to_le_bytes()); + if let Some(slot) = local { + code.push(schema::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + } else { + let value_str_id = *string_map + .get(resolved.unwrap_or(value)) + .ok_or(CompileError::internal("missing interned variable"))?; + code.push(if resolved.is_some() { + schema::Opcode::PushConst as u8 + } else { + schema::Opcode::LoadArg as u8 + }); + code.extend_from_slice(&value_str_id.to_le_bytes()); + } } // No ExprFallback for selectors — errors abort. code.push(schema::Opcode::CallSelect as u8); @@ -429,6 +539,7 @@ fn lower_selector( fn emit_operand( operand: &Operand, string_map: &BTreeMap, + func_map: &BTreeMap, code: &mut Vec, ) -> Result<(), CompileError> { match operand { @@ -439,6 +550,10 @@ fn emit_operand( code.push(schema::Opcode::LoadArg as u8); code.extend_from_slice(&var_str_id.to_le_bytes()); } + Operand::Local(slot) => { + code.push(schema::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + } Operand::Literal { value, .. } => { let value_str_id = *string_map .get(value) @@ -446,15 +561,75 @@ fn emit_operand( code.push(schema::Opcode::PushConst as u8); code.extend_from_slice(&value_str_id.to_le_bytes()); } + Operand::Call(call) => emit_call(call, string_map, func_map, code)?, } Ok(()) } +fn emit_call( + call: &CallExpr, + string_map: &BTreeMap, + func_map: &BTreeMap, + code: &mut Vec, +) -> Result<(), CompileError> { + let func_key = function_catalog_key(&call.func); + let fn_id = *func_map + .get(&func_key) + .ok_or(CompileError::internal("missing function entry"))?; + let dynamic_options = function_dynamic_options(&call.func); + emit_operand(&call.operand, string_map, func_map, code)?; + for (key, value, local, resolved) in dynamic_options.iter().copied() { + let key_str_id = *string_map + .get(key) + .ok_or(CompileError::internal("missing interned string"))?; + code.push(schema::Opcode::PushConst as u8); + code.extend_from_slice(&key_str_id.to_le_bytes()); + if let Some(slot) = local { + code.push(schema::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + } else { + let value_str_id = *string_map + .get(resolved.unwrap_or(value)) + .ok_or(CompileError::internal("missing interned variable"))?; + code.push(if resolved.is_some() { + schema::Opcode::PushConst as u8 + } else { + schema::Opcode::LoadArg as u8 + }); + code.extend_from_slice(&value_str_id.to_le_bytes()); + } + } + // Fallback is needed for the outer expression; nested calls still leave + // their resolved value on the stack and use the same VM fallback path. + let fb = call + .fallback + .clone() + .unwrap_or_else(|| render_operand_fallback(&call.operand, &call.func)); + let fb_str_id = *string_map + .get(&fb) + .ok_or(CompileError::internal("missing interned string"))?; + code.push(schema::Opcode::ExprFallback as u8); + code.extend_from_slice(&fb_str_id.to_le_bytes()); + code.push(schema::Opcode::CallFunc as u8); + code.extend_from_slice(&fn_id.to_le_bytes()); + code.push(1); + code.push( + u8::try_from(dynamic_options.len()) + .map_err(|_| CompileError::size_overflow("option count"))?, + ); + Ok(()) +} + fn render_operand_fallback(operand: &Operand, func: &FunctionSpec) -> String { match operand { Operand::Var(var) => format!("{{${var}}}"), + Operand::Local(slot) => format!("{{}}"), Operand::Literal { value, .. } if value.is_empty() => format!("{{:{}}}", func.name), Operand::Literal { value, .. } => format!("{{|{}|}}", escape_fallback_literal(value)), + Operand::Call(call) => call + .fallback + .clone() + .unwrap_or_else(|| render_operand_fallback(&call.operand, &call.func)), } } @@ -471,3 +646,97 @@ fn patch_rel32(code: &mut [u8], rel_pos: usize, target_pc: u32) -> Result<(), Co code[rel_pos..rel_pos + 4].copy_from_slice(&rel_i32.to_le_bytes()); Ok(()) } + +#[cfg(all(test, feature = "icu4x"))] +mod tests { + use super::*; + use crate::compiler::compile_str; + use crate::runtime::{BuiltinHost, Catalog, Formatter, Value}; + + fn opcode_count(catalog: &Catalog) -> usize { + let mut pc = 0; + let mut count = 0; + while pc < catalog.code().len() { + pc += schema::Opcode::try_from(catalog.code()[pc]) + .expect("valid opcode") + .bytes(); + count += 1; + } + count + } + + fn numeric_match(exacts: usize, keywords: usize) -> Catalog { + let mut source = String::from(".input {$a :number}\n.input {$b :number}\n.match $a $b\n"); + for n in 0..exacts { + source.push_str(&format!("{n} {n} {{{{exact{n}}}}}\n")); + } + for key in ["one", "two", "few"].into_iter().take(keywords) { + source.push_str(&format!("{key} {key} {{{{{key}}}}}\n")); + } + source.push_str("* * {{fallback}}"); + let bytes = compile_str(&source).expect("compiled"); + Catalog::from_bytes(&bytes).expect("shared defaults verify") + } + + fn format_numbers(catalog: &Catalog, args: &[(&str, i64)]) -> String { + let locale = "ru".parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + let mut formatter = Formatter::new(catalog, host).expect("formatter"); + let handle = formatter.resolve("main").expect("handle"); + let args = args + .iter() + .map(|(name, value)| { + ( + catalog.string_id(name).expect("argument id"), + Value::Int(*value), + ) + }) + .collect::>(); + let mut output = String::new(); + let mut errors = Vec::new(); + formatter + .format_to(handle, &args, &mut output, Some(&mut errors)) + .expect("formatted"); + assert!(errors.is_empty(), "{errors:?}"); + output + } + + #[test] + fn shared_defaults_keep_two_selector_bytecode_linear() { + for keywords in [1, 3] { + let counts = [3, 6, 12].map(|exacts| opcode_count(&numeric_match(exacts, keywords))); + // Previously 134/246 instructions for just three exact keys. + assert!(counts[0] <= 100, "{keywords} keyword arms: {counts:?}"); + assert!(counts[1] - counts[0] <= 36, "{counts:?}"); + assert!(counts[2] - counts[1] <= 72, "{counts:?}"); + let catalog = numeric_match(3, keywords); + assert_eq!(format_numbers(&catalog, &[("a", 1), ("b", 1)]), "exact1"); + // The second selector rejects the exact branch; retry the first + // selector's plural category instead of going straight to '*'. + assert_eq!(format_numbers(&catalog, &[("a", 1), ("b", 21)]), "one"); + assert_eq!(format_numbers(&catalog, &[("a", 1), ("b", 2)]), "fallback"); + if keywords == 3 { + assert_eq!(format_numbers(&catalog, &[("a", 2), ("b", 22)]), "few"); + } + } + } + + #[test] + fn shared_defaults_unwind_three_selector_levels() { + let source = ".input {$a :number}\n.input {$b :number}\n.input {$c :number}\n.match $a $b $c\n1 1 0 {{exact}}\none one one {{category}}\n* * * {{fallback}}"; + let bytes = compile_str(source).expect("compiled"); + let catalog = Catalog::from_bytes(&bytes).expect("shared defaults verify"); + assert_eq!( + format_numbers(&catalog, &[("a", 1), ("b", 1), ("c", 0)]), + "exact" + ); + assert_eq!( + format_numbers(&catalog, &[("a", 1), ("b", 21), ("c", 31)]), + "category" + ); + assert_eq!( + format_numbers(&catalog, &[("a", 1), ("b", 21), ("c", 2)]), + "fallback" + ); + } +} diff --git a/crates/message-format/src/compiler/compile/mod.rs b/crates/message-format/src/compiler/compile/mod.rs index b2effa9..3721ec5 100644 --- a/crates/message-format/src/compiler/compile/mod.rs +++ b/crates/message-format/src/compiler/compile/mod.rs @@ -984,17 +984,20 @@ fn collect_builtin_validation_errors(messages: &[Message]) -> Vec fn collect_builtin_part_errors(parts: &[Part], message: &Message, errors: &mut Vec) { for part in parts { match part { - Part::Call(CallExpr { - operand: _, func, .. - }) => { + Part::Call(CallExpr { operand, func, .. }) => { + collect_builtin_operand_errors(operand, message, errors); collect_builtin_function_errors(func, message, errors); } + Part::Bind { value, .. } => { + collect_builtin_part_errors(core::slice::from_ref(value), message, errors); + } Part::Select(SelectExpr { selector, arms, default, }) => { - if let crate::compiler::semantic::SelectorExpr::Call { func, .. } = selector { + if let crate::compiler::semantic::SelectorExpr::Call { operand, func } = selector { + collect_builtin_operand_errors(operand, message, errors); collect_builtin_function_errors(func, message, errors); } for arm in arms { @@ -1006,11 +1009,25 @@ fn collect_builtin_part_errors(parts: &[Part], message: &Message, errors: &mut V | Part::MarkupClose { .. } | Part::Text(_) | Part::Literal(_) - | Part::Var(_) => {} + | Part::Var(_) + | Part::Local(_) + | Part::CheckSelector(_) => {} } } } +fn collect_builtin_operand_errors( + operand: &Operand, + message: &Message, + errors: &mut Vec, +) { + let Operand::Call(call) = operand else { + return; + }; + collect_builtin_operand_errors(&call.operand, message, errors); + collect_builtin_function_errors(&call.func, message, errors); +} + fn collect_builtin_function_errors( func: &FunctionSpec, message: &Message, @@ -1101,6 +1118,7 @@ fn collect_manifest_part_errors( for part in parts { match part { Part::Call(CallExpr { operand, func, .. }) => { + collect_operand_manifest_errors(operand, manifest, message, errors); collect_function_spec_errors_into( func, Some(operand), @@ -1110,6 +1128,9 @@ fn collect_manifest_part_errors( errors, ); } + Part::Bind { value, .. } => { + collect_manifest_value_errors(value, manifest, message, errors); + } Part::Select(SelectExpr { selector, arms, @@ -1121,7 +1142,11 @@ fn collect_manifest_part_errors( } collect_manifest_part_errors(default, manifest, message, errors); } - Part::Text(_) | Part::Literal(_) | Part::Var(_) => {} + Part::Text(_) + | Part::Literal(_) + | Part::Var(_) + | Part::Local(_) + | Part::CheckSelector(_) => {} Part::MarkupOpen { name, options } | Part::MarkupClose { name, options } => { collect_markup_manifest_errors_into(name, options, manifest, message, errors); } @@ -1129,6 +1154,25 @@ fn collect_manifest_part_errors( } } +fn collect_manifest_value_errors( + value: &Part, + manifest: &FunctionManifest, + message: &Message, + errors: &mut Vec, +) { + if let Part::Call(CallExpr { operand, func, .. }) = value { + collect_operand_manifest_errors(operand, manifest, message, errors); + collect_function_spec_errors_into( + func, + Some(operand), + FunctionUse::Select, + manifest, + message, + errors, + ); + } +} + fn collect_markup_manifest_errors_into( name: &str, options: &[FunctionOption], @@ -1181,7 +1225,9 @@ fn validate_markup_option( }; let accepts = match option.value { FunctionOptionValue::Literal(_) => option_schema.value_kind.accepts_literal(), - FunctionOptionValue::Var(_) => option_schema.value_kind.accepts_variable(), + FunctionOptionValue::Var(_) + | FunctionOptionValue::ResolvedVar { .. } + | FunctionOptionValue::LocalVar { .. } => option_schema.value_kind.accepts_variable(), }; if accepts { return Ok(()); @@ -1221,10 +1267,25 @@ fn collect_selector_manifest_errors( message: &Message, errors: &mut Vec, ) { - if let crate::compiler::semantic::SelectorExpr::Call { operand, func } = selector { + let (operand, func) = match selector { + crate::compiler::semantic::SelectorExpr::Call { operand, func } => { + (Some(operand), Some(func)) + } + crate::compiler::semantic::SelectorExpr::Local { + func: Some(func), .. + } => (None, Some(func)), + crate::compiler::semantic::SelectorExpr::CheckedLocal { + func: Some(func), .. + } => (None, Some(func)), + _ => (None, None), + }; + if let Some(func) = func { + if let Some(operand) = operand { + collect_operand_manifest_errors(operand, manifest, message, errors); + } collect_function_spec_errors_into( func, - Some(operand), + operand, FunctionUse::Select, manifest, message, @@ -1241,6 +1302,26 @@ fn collect_selector_manifest_errors( } } +fn collect_operand_manifest_errors( + operand: &Operand, + manifest: &FunctionManifest, + message: &Message, + errors: &mut Vec, +) { + let Operand::Call(call) = operand else { + return; + }; + collect_operand_manifest_errors(&call.operand, manifest, message, errors); + collect_function_spec_errors_into( + &call.func, + Some(&call.operand), + FunctionUse::Format, + manifest, + message, + errors, + ); +} + #[derive(Clone, Copy)] enum FunctionUse { Format, @@ -1328,7 +1409,9 @@ fn validate_function_option( }; let accepts = match option.value { FunctionOptionValue::Literal(_) => option_schema.value_kind.accepts_literal(), - FunctionOptionValue::Var(_) => option_schema.value_kind.accepts_variable(), + FunctionOptionValue::Var(_) + | FunctionOptionValue::ResolvedVar { .. } + | FunctionOptionValue::LocalVar { .. } => option_schema.value_kind.accepts_variable(), }; if accepts { return Ok(()); @@ -1387,7 +1470,7 @@ fn validate_function_operand( )) }; } - Some(Operand::Var(_)) | None => None, + Some(Operand::Var(_) | Operand::Local(_) | Operand::Call(_)) | None => None, }, FunctionOperandKind::Number => match operand { Some(Operand::Literal { value, kind }) => { @@ -1403,7 +1486,7 @@ fn validate_function_operand( )) }; } - Some(Operand::Var(_)) | None => None, + Some(Operand::Var(_) | Operand::Local(_) | Operand::Call(_)) | None => None, }, }; let Some(operand) = operand else { diff --git a/crates/message-format/src/compiler/compile/tests.rs b/crates/message-format/src/compiler/compile/tests.rs index e988701..7c62751 100644 --- a/crates/message-format/src/compiler/compile/tests.rs +++ b/crates/message-format/src/compiler/compile/tests.rs @@ -1,8 +1,12 @@ // Copyright 2026 the Message Format Authors // SPDX-License-Identifier: Apache-2.0 OR MIT +#[cfg(feature = "icu4x")] +use crate::runtime::BuiltinHost; use crate::runtime::schema; use crate::runtime::{Catalog, Formatter, HostFn, NoopHost, Value}; +use alloc::rc::Rc; +use core::cell::Cell; use crate::compiler::manifest::{ FunctionManifest, FunctionOperandKind, FunctionOptionValueKind, FunctionSchema, @@ -34,6 +38,10 @@ fn expect_errors(report: CompileReport) -> Vec { report.diagnostics } +fn passthrough_host() -> impl crate::runtime::Host { + HostFn(|_, args: &[Value], _: &[(u32, Value)]| Ok(args.first().cloned().unwrap_or(Value::Null))) +} + fn arg_id(catalog: &Catalog, name: &str) -> u32 { catalog.string_id(name).expect("arg id") } @@ -42,6 +50,17 @@ fn arg(catalog: &Catalog, name: &str, value: Value) -> (u32, Value) { (arg_id(catalog, name), value) } +fn opcodes(catalog: &Catalog) -> Vec { + let mut result = Vec::new(); + let mut pc = 0; + while pc < catalog.code().len() { + let opcode = schema::Opcode::try_from(catalog.code()[pc]).expect("valid opcode"); + result.push(opcode); + pc += opcode.bytes(); + } + result +} + fn chunk_len(bytes: &[u8], tag: [u8; 4]) -> u32 { let chunk_count = u32::from_le_bytes(bytes[16..20].try_into().expect("chunk count")); let chunk_table_offset = @@ -112,17 +131,26 @@ fn bare_interpolation_lowers_to_out_arg() { } #[test] -fn simple_select_lowers_to_select_arg() { +fn string_input_resolves_once_before_selection() { let bytes = compile_str(".input { $kind :string }\n.match $kind\nformal {{Good evening}}\n* {{Hi}}") .expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - assert_eq!(catalog.code()[0], schema::Opcode::SelectArg as u8); - assert!( - !catalog - .code() - .contains(&(schema::Opcode::SelectBegin as u8)) - ); + let code = opcodes(&catalog); + for opcode in [ + schema::Opcode::CallFunc, + schema::Opcode::StoreLocal, + schema::Opcode::CheckSelector, + schema::Opcode::SelectLocal, + ] { + assert_eq!( + code.iter() + .filter(|candidate| **candidate == opcode) + .count(), + 1 + ); + } + assert!(!code.contains(&schema::Opcode::SelectArg)); } #[test] @@ -183,7 +211,7 @@ fn exact_match_beats_default_even_if_default_appears_first() { compile_str(".input { $kind :string }\n.match $kind\n* {{Hi}}\nformal {{Good evening}}") .expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let mut formatter = Formatter::new(&catalog, passthrough_host()).expect("formatter"); let out = formatter .format_by_id_for_test( @@ -666,6 +694,18 @@ fn compile_str_rejects_invalid_builtin_number_use_grouping_literal() { } } +#[test] +fn compile_str_rejects_invalid_builtin_option_in_nested_local_call() { + let err = + compile_str(".local $n = {1 :number signDisplay=bogus} .local $m = {$n :number} {{{$m}}}") + .expect_err("must fail"); + assert!(matches!( + err, + CompileError::InvalidBuiltinOptionValue { function, option, .. } + if function == "number" && option == "signDisplay" + )); +} + #[test] fn compile_str_rejects_invalid_plural_variant_key() { let err = @@ -1435,6 +1475,30 @@ fn compile_inputs_with_manifest_reports_precise_source() { } } +#[test] +fn compile_inputs_with_manifest_reports_declaration_function_origin() { + let source = ".input { $value :custom:unknown }\n{{Hello}}"; + let err = expect_first_error(compile_inputs_with_manifest( + [CompileInput { + name: "bad.mf2", + message_id: "main", + source, + kind: SourceKind::MessageFormat, + }], + CompileOptions::default(), + &FunctionManifest::new(), + )); + + match *err.error { + CompileError::UnknownFunction { origin, .. } => { + let origin = origin.expect("function origin"); + assert_eq!(origin.byte_start, source.find(":custom").expect("function")); + assert_eq!(origin.byte_end, source.find(" }").expect("function end")); + } + other => panic!("unexpected error: {other:?}"), + } +} + #[test] fn render_with_source_highlights_unknown_function_annotation() { let err = compile_with_manifest( @@ -1611,6 +1675,30 @@ fn compiles_and_formats_call() { ); } +#[test] +fn local_declaration_is_evaluated_once_when_interpolated_twice() { + let bytes = compile_str( + ".local $n = {seed :test:format} .match $n called {{hit {$n} {$n}}} * {{miss}}", + ) + .expect("compiled"); + let catalog = Catalog::from_bytes(&bytes).expect("catalog"); + let calls = Rc::new(Cell::new(0)); + let observed = Rc::clone(&calls); + let mut formatter = Formatter::new( + &catalog, + HostFn(move |_fn_id, _args, _opts| { + observed.set(observed.get() + 1); + Ok(Value::Str("called".to_string())) + }), + ) + .expect("formatter"); + let output = formatter + .format_by_id_for_test("main", &Vec::<(u32, Value)>::new()) + .expect("formatted"); + assert_eq!(output, "hit called called"); + assert_eq!(calls.get(), 1); +} + #[test] fn invalid_expr_fails() { let err = compile_str("{}").expect_err("must fail"); @@ -1899,7 +1987,7 @@ fn raw_match_statement_is_rewritten_and_formatted() { ".input {$kind :string} .match $kind formal {{Good evening}} casual {{Hi}} * {{Hello}}"; let bytes = compile_str(source).expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let mut formatter = Formatter::new(&catalog, passthrough_host()).expect("formatter"); let args1 = vec![arg(&catalog, "kind", Value::Str("formal".to_string()))]; assert_eq!( @@ -1927,11 +2015,11 @@ fn raw_match_statement_is_rewritten_and_formatted() { } #[test] -fn raw_match_with_local_literal_selector_resolves_at_compile_time() { +fn raw_match_with_local_literal_string_resolves_through_host() { let source = ".local $kind = {formal :string} .match $kind formal {{Good evening}} * {{Hello}}"; let bytes = compile_str(source).expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let mut formatter = Formatter::new(&catalog, passthrough_host()).expect("formatter"); let out = formatter .format_by_id_for_test("main", &Vec::<(u32, Value)>::new()) .expect("formatted"); @@ -1944,7 +2032,7 @@ fn raw_match_with_local_alias_selector_uses_input() { ".input {$kind :string} .local $k = {$kind} .match $k formal {{Good evening}} * {{Hello}}"; let bytes = compile_str(source).expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let mut formatter = Formatter::new(&catalog, passthrough_host()).expect("formatter"); let args = vec![arg(&catalog, "kind", Value::Str("formal".to_string()))]; let out = formatter .format_by_id_for_test("main", &args) @@ -1952,6 +2040,90 @@ fn raw_match_with_local_alias_selector_uses_input() { assert_eq!(out, "Good evening"); } +#[cfg(feature = "icu4x")] +#[test] +fn raw_match_with_numeric_local_selector_falls_back_to_plural_category() { + for arms in [ + "1 a {{exact-a}} one b {{plural-b}} * * {{fallback}}", + "* * {{fallback}} one b {{plural-b}} 1 a {{exact-a}}", + "* * {{fallback}} 1 a {{exact-a}} one b {{plural-b}}", + ] { + let source = + alloc::format!(".input {{$n :number}} .input {{$s :string}} .match $n $s {arms}"); + let bytes = compile_str(&source).expect("compiled"); + let catalog = Catalog::from_bytes(&bytes).expect("catalog"); + let locale = "en".parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + let mut formatter = Formatter::new(&catalog, host).expect("formatter"); + let args = vec![ + arg(&catalog, "n", Value::Int(1)), + arg(&catalog, "s", Value::Str("b".to_string())), + ]; + assert_eq!( + formatter + .format_by_id_for_test("main", &args) + .expect("formatted"), + "plural-b" + ); + + let exact_args = vec![ + arg(&catalog, "n", Value::Int(1)), + arg(&catalog, "s", Value::Str("a".to_string())), + ]; + assert_eq!( + formatter + .format_by_id_for_test("main", &exact_args) + .expect("formatted"), + "exact-a" + ); + } +} + +#[cfg(feature = "icu4x")] +#[test] +fn raw_match_rechecks_each_source_local_selector_once() { + let source = + ".input {$n :number} .match $n $n 1 1 {{exact}} one one {{category}} * * {{fallback}}"; + let bytes = compile_str(source).expect("compiled"); + let catalog = Catalog::from_bytes(&bytes).expect("catalog"); + let locale = "en".parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + let mut formatter = Formatter::new(&catalog, host).expect("formatter"); + let message = formatter.resolve("main").expect("message"); + let mut sink = String::new(); + let mut diagnostics = Vec::new(); + formatter + .format_to(message, &[], &mut sink, Some(&mut diagnostics)) + .expect("formatted"); + assert_eq!(sink, "fallback"); + assert_eq!( + diagnostics, + vec![ + crate::runtime::FormatError::MissingArg("n".to_string()), + crate::runtime::FormatError::BadSelector { source: None }, + crate::runtime::FormatError::BadSelector { source: None }, + ] + ); +} + +#[cfg(feature = "icu4x")] +#[test] +fn raw_match_with_offset_numeric_local_selector_falls_back_to_plural_category() { + let source = ".local $n = {2 :number} .local $m = {$n :offset subtract=1} .input {$s :string} .match $m $s 1 a {{exact-a}} one b {{plural-b}} * * {{fallback}}"; + let bytes = compile_str(source).expect("compiled"); + let catalog = Catalog::from_bytes(&bytes).expect("catalog"); + let locale = "en".parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + let mut formatter = Formatter::new(&catalog, host).expect("formatter"); + let args = vec![arg(&catalog, "s", Value::Str("b".to_string()))]; + assert_eq!( + formatter + .format_by_id_for_test("main", &args) + .expect("formatted"), + "plural-b" + ); +} + #[test] fn local_alias_is_substituted() { let bytes = compile_str(".local $a = {$name} {{Hello {$a}!}}").expect("compiled"); @@ -1998,11 +2170,14 @@ fn alias_cycle_reports_resolution_overflow() { assert!(matches!(err, CompileError::AliasResolutionOverflow { .. })); } +#[cfg(feature = "icu4x")] #[test] fn local_integer_function_is_evaluated() { let bytes = compile_str(".local $x = {4.2 :integer} {{X={$x}}}").expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let locale = "en".parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + let mut formatter = Formatter::new(&catalog, host).expect("formatter"); let out = formatter .format_by_id_for_test("main", &Vec::<(u32, Value)>::new()) .expect("formatted"); @@ -2027,6 +2202,10 @@ fn raw_match_with_dynamic_select_option_uses_default_arm() { ".input {$mode} .local $x = {1 :test:select select=$mode} .match $x 1 {{A}} * {{B}}"; let bytes = compile_str(source).expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); + let code = opcodes(&catalog); + assert!(code.contains(&schema::Opcode::StoreLocal)); + assert!(code.contains(&schema::Opcode::CheckSelector)); + assert!(code.contains(&schema::Opcode::SelectLocal)); let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); let out = formatter .format_by_id_for_test("main", &Vec::<(u32, Value)>::new()) @@ -2063,7 +2242,7 @@ fn raw_match_with_two_runtime_selectors_is_rewritten() { let source = ".input {$x :string} .input {$y :string} .match $x $y 1 1 {{1,1}} * * {{*,*}}"; let bytes = compile_str(source).expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let mut formatter = Formatter::new(&catalog, passthrough_host()).expect("formatter"); let args_11 = vec![ arg(&catalog, "x", Value::Int(1)), arg(&catalog, "y", Value::Int(1)), @@ -2089,7 +2268,7 @@ fn raw_match_with_escaped_quoted_key_selects_expected_arm() { let source = ".input {$kind :string} .match $kind |a\\|b| {{pipe}} * {{other}}"; let bytes = compile_str(source).expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let mut formatter = Formatter::new(&catalog, passthrough_host()).expect("formatter"); let args_hit = vec![arg(&catalog, "kind", Value::Str(String::from("a|b")))]; let args_default = vec![arg(&catalog, "kind", Value::Str(String::from("x")))]; @@ -2104,30 +2283,51 @@ fn raw_match_with_escaped_quoted_key_selects_expected_arm() { assert_eq!(out_default, "other"); } +#[cfg(feature = "icu4x")] #[test] fn raw_match_with_integer_select_exact_formats_literal_arm() { let source = ".local $sel = {1 :integer select=exact} .match $sel 1 {{literal select {$sel}}} * {{OTHER}}"; let bytes = compile_str(source).expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let locale = "en".parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + let mut formatter = Formatter::new(&catalog, host).expect("formatter"); let out = formatter .format_by_id_for_test("main", &Vec::<(u32, Value)>::new()) .expect("formatted"); assert_eq!(out, "literal select 1"); } +#[cfg(feature = "icu4x")] #[test] fn local_offset_result_can_be_used_in_match_selector() { let source = ".local $x = {10 :integer} .local $y = {$x :offset subtract=6} .match $y 10 {{=10}} 4 {{=4}} * {{other}}"; let bytes = compile_str(source).expect("compiled"); let catalog = Catalog::from_bytes(&bytes).expect("catalog"); - let mut formatter = Formatter::new(&catalog, NoopHost).expect("formatter"); + let locale = "en".parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + let mut formatter = Formatter::new(&catalog, host).expect("formatter"); let out = formatter .format_by_id_for_test("main", &Vec::<(u32, Value)>::new()) .expect("formatted"); assert_eq!(out, "=4"); } +#[cfg(feature = "icu4x")] +#[test] +fn raw_offset_result_defaults_to_plural_selection() { + let source = ".local $n = {2 :offset subtract=1} .match $n one {{one}} * {{other}}"; + let bytes = compile_str(source).expect("compiled"); + let catalog = Catalog::from_bytes(&bytes).expect("catalog"); + let locale = "en".parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + let mut formatter = Formatter::new(&catalog, host).expect("formatter"); + let out = formatter + .format_by_id_for_test("main", &Vec::<(u32, Value)>::new()) + .expect("formatted"); + assert_eq!(out, "one"); +} + #[test] fn missing_default_match_fails() { let err = compile_str(".input { $x :string }\n.match $x\na {{A}}").expect_err("must fail"); @@ -2280,6 +2480,30 @@ fn local_literal_option_values_are_substituted_as_semantic_text() { assert!(catalog.string_id("|fast path|").is_none()); } +#[test] +fn custom_host_evaluates_string_input_once_for_repeated_uses() { + let bytes = compile_str(".input {$x :string} {{A={$x} B={$x}}}").expect("compiled"); + let catalog = Catalog::from_bytes(&bytes).expect("catalog"); + let calls = Rc::new(Cell::new(0)); + let observed = Rc::clone(&calls); + let mut formatter = Formatter::new( + &catalog, + HostFn(move |_, _, _| { + observed.set(observed.get() + 1); + Ok(Value::Str("resolved".to_string())) + }), + ) + .expect("formatter"); + let args = vec![arg(&catalog, "x", Value::Int(1))]; + assert_eq!( + formatter + .format_by_id_for_test("main", &args) + .expect("formatted"), + "A=resolved B=resolved" + ); + assert_eq!(calls.get(), 1); +} + // ─── Attribute values (mf-1m8f) ────────────────────────────────────── #[test] diff --git a/crates/message-format/src/compiler/semantic.rs b/crates/message-format/src/compiler/semantic.rs index 21d4917..383082e 100644 --- a/crates/message-format/src/compiler/semantic.rs +++ b/crates/message-format/src/compiler/semantic.rs @@ -9,7 +9,7 @@ //! data, but should treat it as a low-level compiler input boundary rather than //! a long-term calm public IR. -use alloc::{string::String, vec::Vec}; +use alloc::{boxed::Box, string::String, vec::Vec}; /// Parsed catalog message definition. #[derive(Debug, Clone, PartialEq, Eq)] @@ -176,10 +176,23 @@ pub enum Part { Literal(String), /// Variable interpolation from caller args. Var(String), + /// Previously evaluated declaration value. + Local(u32), /// Function call on one operand. Call(CallExpr), /// Select expression with case arms and a required default arm. Select(SelectExpr), + /// Validate one local selector before generated match dispatch. + CheckSelector(u32), + /// Evaluate and store one declaration value in a local slot. + Bind { + /// Dense declaration slot. + slot: u32, + /// Source expression used when declaration evaluation fails. + fallback: String, + /// Declaration expression. + value: Box, + }, /// Markup open tag, e.g. `{#bold}` or `{#link href=$url}`. MarkupOpen { /// Tag name. @@ -251,6 +264,21 @@ impl Part { pub enum SelectorExpr { /// Variable selector. Var(String), + /// Previously evaluated declaration value. + Local { + /// Dense declaration slot. + slot: u32, + /// Original selector annotation for compile-time validation. + func: Option, + }, + /// Previously evaluated local selector whose validity was checked once + /// before generated dispatch. + CheckedLocal { + /// Dense declaration slot. + slot: u32, + /// Original selector annotation for variant validation. + func: Option, + }, /// Function call selector. Call { /// Selector operand. @@ -287,6 +315,8 @@ impl SelectorExpr { pub enum Operand { /// Variable operand. Var(String), + /// Previously evaluated declaration value. + Local(u32), /// Literal operand. Literal { /// Decoded literal text. @@ -294,6 +324,12 @@ pub enum Operand { /// Source-level literal classification. kind: OperandLiteralKind, }, + /// Nested function call operand produced by declaration lowering. + /// + /// Keeping the call structured lets runtime hosts receive the resolved + /// value and options of an earlier declaration call before applying the + /// outer annotation. + Call(Box), } /// Literal classification for function/select operands. @@ -335,7 +371,7 @@ impl Operand { pub fn literal_value(&self) -> Option<&str> { match self { Self::Literal { value, .. } => Some(value.as_str()), - Self::Var(_) => None, + Self::Var(_) | Self::Local(_) | Self::Call(_) => None, } } @@ -344,7 +380,7 @@ impl Operand { pub fn literal_kind(&self) -> Option { match self { Self::Literal { kind, .. } => Some(*kind), - Self::Var(_) => None, + Self::Var(_) | Self::Local(_) | Self::Call(_) => None, } } } @@ -527,6 +563,25 @@ pub enum FunctionOptionValue { Literal(String), /// Dynamic variable option value. Var(String), + /// Dynamic variable option value with a compile-time-resolved payload. + /// + /// The variable name remains part of the semantic value so the option is + /// still lowered and validated as dynamic. The resolved payload lets the + /// compiler preserve a local declaration's value without requiring that + /// declaration to be exposed as a message input. + ResolvedVar { + /// Original dynamic variable name. + name: String, + /// Value resolved from the declaration during compilation. + value: String, + }, + /// Dynamic variable option value loaded from a declaration slot. + LocalVar { + /// Original dynamic variable name. + name: String, + /// Dense declaration slot. + slot: u32, + }, } /// One select case arm. diff --git a/crates/message-format/src/lib.rs b/crates/message-format/src/lib.rs index e10bf10..b4c517f 100644 --- a/crates/message-format/src/lib.rs +++ b/crates/message-format/src/lib.rs @@ -467,7 +467,7 @@ mod tests { let args = MessageArgs::new(); assert_eq!( compile_and_format("{-1e23 :offset subtract=1 signDisplay=never}", &args), - "100000000000000000000000" + "100000000000000000000001" ); } diff --git a/crates/message-format/src/runtime/builtin.rs b/crates/message-format/src/runtime/builtin.rs index ab96054..3465fa2 100644 --- a/crates/message-format/src/runtime/builtin.rs +++ b/crates/message-format/src/runtime/builtin.rs @@ -3,13 +3,16 @@ //! ICU4X-backed built-in function host. +#[cfg(test)] +use alloc::vec; use alloc::{ - borrow::Cow, collections::BTreeMap, format, string::String, string::ToString, vec::Vec, + borrow::Cow, boxed::Box, collections::BTreeMap, format, string::String, string::ToString, + vec::Vec, }; use core::array; use core::str::FromStr; -use fixed_decimal::Decimal; +use fixed_decimal::{Decimal, SignedRoundingMode, UnsignedRoundingMode}; use icu_calendar::Date; use icu_datetime::fieldsets; use icu_datetime::input::{DateTime, Time}; @@ -19,8 +22,7 @@ use icu_locale_core::Locale; use icu_plurals::{PluralCategory, PluralRules}; use crate::common::text::{ - SignDisplay, format_signed_number, format_signed_string, parse_number_literal, - strip_bidi_controls, + SignDisplay, format_signed_string, parse_number_literal, strip_bidi_controls, }; use crate::runtime::{ catalog::Catalog, @@ -28,7 +30,10 @@ use crate::runtime::{ FormatError, HostCallError, ImplementationFailure, MessageFunctionError, Trap, UnsupportedOperation, }, - value::Value, + value::{ + NumberFormatOptions, NumberGrouping, NumberSelection, NumberSignDisplay, NumberValue, + ResolvedNumber, ResolvedSelect, ResolvedString, StringDirection, Value, + }, vm::Host, }; @@ -335,6 +340,7 @@ impl BuiltinHost { entry: &BuiltinEntry, args: &[Value], opts: &[(u32, Value)], + on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { let Some(raw_arg) = args.first() else { return Err(bad_operand()); @@ -345,39 +351,47 @@ impl BuiltinHost { validate_builtin_option_values(entry.func, &options)?; match entry.func { - BuiltinFn::String => Ok(Value::Str(format_string(catalog, raw_arg, &options))), + BuiltinFn::String => Ok(Value::String(format_string(catalog, raw_arg, &options))), BuiltinFn::Number | BuiltinFn::Integer => { let integer_only = entry.func == BuiltinFn::Integer; - match options.get(BuiltinOptionKey::Select).as_deref() { - Some("plural") => Ok(Value::Str(format_plural( - raw_arg, - catalog, - cardinal_rules, - &options, - )?)), - Some("ordinal") => Ok(Value::Str(format_plural( - raw_arg, - catalog, - ordinal_rules, - &options, - )?)), - _ if options.get(BuiltinOptionKey::Style).as_deref() == Some("percent") => { - Ok(Value::Str(format_percent(raw_arg, catalog, &options)?)) - } - _ => Ok(Value::Str(format_number( - catalog, - raw_arg, - integer_only, - &options, - )?)), + if opts.iter().any(|(key_id, _)| { + index.option_keys_by_str_id.get(key_id) == Some(&BuiltinOptionKey::Select) + }) { + on_error(MessageFunctionError::BadOption); + } + if options.get(BuiltinOptionKey::Style).as_deref() == Some("percent") { + return Ok(Value::Str(format_percent(raw_arg, catalog, &options)?)); } + let minimum_fraction_digits = parse_minimum_fraction_digits(&options)?; + let maximum_fraction_digits = parse_maximum_fraction_digits(&options)?; + let _ = parse_minimum_integer_digits(&options)?; + validate_digit_range_relationship( + minimum_fraction_digits, + maximum_fraction_digits, + )?; + let resolved = resolve_number( + raw_arg, + catalog, + integer_only, + &options, + cardinal_rules, + ordinal_rules, + on_error, + )?; + Ok(Value::Number(Box::new(resolved))) } BuiltinFn::Percent => Ok(Value::Str(format_percent(raw_arg, catalog, &options)?)), BuiltinFn::Currency => Ok(Value::Str(format_currency(raw_arg, catalog, &options)?)), - BuiltinFn::Offset => Ok(Value::Str(format_offset(raw_arg, catalog, &options)?)), - BuiltinFn::TestSelect => { - Ok(Value::Str(format_test_select(raw_arg, catalog, &options)?)) - } + BuiltinFn::Offset => Ok(Value::Number(Box::new(resolve_offset( + raw_arg, + catalog, + &options, + cardinal_rules, + ordinal_rules, + )?))), + BuiltinFn::TestSelect => Ok(Value::ResolvedSelect(Box::new(ResolvedSelect::new( + format_test_select(raw_arg, catalog, &options)?, + )))), BuiltinFn::TestFunction => format_test_function(raw_arg, catalog, &options), BuiltinFn::TestFormat => Err(implementation_failure(ImplementationFailure::TestFormat)), BuiltinFn::Date => { @@ -427,31 +441,34 @@ impl BuiltinHost { catalog: &Catalog, index: &BuiltinHostCatalogIndex, entry: &BuiltinEntry, + args: &[Value], opts: &[(u32, Value)], ) -> Option<&PluralRules> { - if entry - .options - .iter() - .enumerate() - .any(|(index, value)| value.is_some() && index != BuiltinOptionKey::Select.index()) - { + if !matches!(entry.func, BuiltinFn::Number | BuiltinFn::Integer) { return None; } + if let Some(Value::Number(number)) = args.first() { + match number.selection { + NumberSelection::Plural => return Some(&self.cardinal_rules), + NumberSelection::Ordinal => return Some(&self.ordinal_rules), + NumberSelection::Exact | NumberSelection::None | NumberSelection::Invalid => {} + } + } if opts.is_empty() { return match entry.select_mode { BuiltinSelectMode::Plural => Some(&self.cardinal_rules), BuiltinSelectMode::Ordinal => Some(&self.ordinal_rules), - BuiltinSelectMode::None => None, + // Number selectors use cardinal plural selection by default. + // The function entry has no explicit mode when the annotation + // omits `select`, but a selector still needs a category. + BuiltinSelectMode::None => Some(&self.cardinal_rules), }; } - if !matches!(entry.func, BuiltinFn::Number | BuiltinFn::Integer) { - return None; - } if opts.iter().any(|(key_id, _)| { index .option_keys_by_str_id .get(key_id) - .is_none_or(|key| *key != BuiltinOptionKey::Select) + .is_some_and(|key| *key == BuiltinOptionKey::Select) }) { return None; } @@ -505,6 +522,7 @@ impl Host for BuiltinHost { fn_id: u16, args: &[Value], opts: &[(u32, Value)], + on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { let Some(entry) = index.by_id.get(&fn_id) else { return Err(HostCallError::UnknownFunction { fn_id }); @@ -519,6 +537,7 @@ impl Host for BuiltinHost { entry, args, opts, + on_error, ) .map_err(into_host_call_error) } @@ -530,13 +549,32 @@ impl Host for BuiltinHost { fn_id: u16, args: &[Value], opts: &[(u32, Value)], + on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { let Some(entry) = index.by_id.get(&fn_id) else { return Err(HostCallError::UnknownFunction { fn_id }); }; + if opts.iter().any(|(key_id, _)| { + index.option_keys_by_str_id.get(key_id) == Some(&BuiltinOptionKey::Select) + }) { + on_error(MessageFunctionError::BadOption); + return Ok(Value::Null); + } + if matches!(entry.func, BuiltinFn::Number | BuiltinFn::Integer) + && args.first().is_some_and(|value| { + matches!(value, Value::Number(number) if number.selection == NumberSelection::Exact) + }) + { + return Ok(args.first().cloned().expect("checked above")); + } // For number/integer with select=plural|ordinal, compute category and // return a StrRef into the string pool instead of allocating. - if let Some(rules) = self.plural_rules_for(catalog, index, entry, opts) { + if args.first().is_some_and(|value| { + matches!(value, Value::Number(number) if number.selection == NumberSelection::Invalid) + }) { + return Ok(Value::Null); + } + if let Some(rules) = self.plural_rules_for(catalog, index, entry, args, opts) { let raw_arg = args .first() .ok_or_else(bad_operand) @@ -561,18 +599,28 @@ impl Host for BuiltinHost { entry, args, opts, + on_error, ) .map_err(into_host_call_error) } fn format_default( &mut self, - _catalog: &Catalog, + catalog: &Catalog, _index: &BuiltinHostCatalogIndex, value: &Value, ) -> Option { match value { Value::Float(v) => Some(format_number_default_locale(*v, &self.locale)), + Value::Number(number) => render_resolved_number(catalog, number), + Value::String(value) => Some(apply_bidi_dir( + Cow::Borrowed(value.text()), + Some(match value.direction { + StringDirection::Auto => "auto", + StringDirection::Ltr => "ltr", + StringDirection::Rtl => "rtl", + }), + )), _ => None, } } @@ -587,14 +635,21 @@ fn plain_text<'a>(catalog: &'a Catalog, value: &'a Value) -> Cow<'a, str> { Value::Int(v) => Cow::Owned(v.to_string()), Value::Float(v) => Cow::Owned(v.to_string()), Value::Str(v) => Cow::Borrowed(v.as_str()), + Value::String(v) => Cow::Borrowed(v.text()), Value::StrRef(id) => catalog .pool_string_opt(*id) .map(Cow::Borrowed) .unwrap_or_else(|| Cow::Owned(id.to_string())), + Value::Fallback(id) => catalog + .pool_string_opt(*id) + .map(Cow::Borrowed) + .unwrap_or_else(|| Cow::Owned(id.to_string())), Value::LitRef { off, len } => catalog .literal_opt(*off, *len) .map(Cow::Borrowed) .unwrap_or_else(|| Cow::Owned(format!("{off}:{len}"))), + Value::Number(number) => Cow::Owned(number.text()), + Value::ResolvedSelect(value) => Cow::Borrowed(value.text()), } } @@ -612,9 +667,22 @@ fn parse_static_select_mode( } } -fn format_string(catalog: &Catalog, value: &Value, options: &EffectiveOptions<'_>) -> String { +fn format_string( + catalog: &Catalog, + value: &Value, + options: &EffectiveOptions<'_>, +) -> ResolvedString { let dir = options.get(BuiltinOptionKey::UDir); - apply_bidi_dir(plain_text(catalog, value), dir.as_deref()) + let text = plain_text(catalog, value).into_owned(); + let direction = match dir.as_deref() { + Some("ltr") => StringDirection::Ltr, + Some("rtl") => StringDirection::Rtl, + _ => StringDirection::Auto, + }; + ResolvedString { + text: text.into_boxed_str(), + direction, + } } fn value_text<'a>(catalog: &'a Catalog, value: &'a Value) -> Option<&'a str> { @@ -622,90 +690,332 @@ fn value_text<'a>(catalog: &'a Catalog, value: &'a Value) -> Option<&'a str> { Value::Str(value) => Some(value), Value::StrRef(id) => catalog.pool_string_opt(*id), Value::LitRef { off, len } => catalog.literal_opt(*off, *len), + Value::ResolvedSelect(value) => Some(value.text()), + Value::String(value) => Some(value.text()), _ => None, } } -fn format_number( - catalog: &Catalog, +fn resolve_number( value: &Value, + catalog: &Catalog, integer_only: bool, options: &EffectiveOptions<'_>, -) -> Result { - let notation = options.get(BuiltinOptionKey::Notation); - if notation.is_some_and(|n| n == "scientific") { - let num = numeric_operand(value, catalog)?; - return Ok(format_scientific(num)); - } - - let minimum_fraction_digits = parse_minimum_fraction_digits(options)?; - let maximum_fraction_digits = parse_maximum_fraction_digits(options)?; - let minimum_integer_digits = parse_minimum_integer_digits(options)?; - validate_digit_range_relationship(minimum_fraction_digits, maximum_fraction_digits)?; - let sign_display = parse_sign_display(options); - let use_grouping = parse_use_grouping(options); - - let result = match value { - Value::Null | Value::Bool(_) => Err(bad_operand()), - Value::Int(v) => { - if integer_only { - Ok(format_signed_string(sign_display, v.to_string())) - } else if let Some(min) = minimum_fraction_digits { - // Keep integer operands exact here instead of routing them - // through f64 formatting, which would silently round large i64s. - Ok(format_signed_string( - sign_display, - format_int_with_min_fraction_digits(*v, min), - )) - } else { - Ok(format_signed_string(sign_display, v.to_string())) + cardinal_rules: &PluralRules, + ordinal_rules: &PluralRules, + on_error: &mut dyn FnMut(MessageFunctionError), +) -> Result { + let (mut number, inherited_format, inherited_selection, inherited_select) = match value { + Value::Number(number) => ( + number.value.clone(), + number.format, + number.selection, + number.has_explicit_select, + ), + _ => ( + parse_number_value(value, catalog)?, + NumberFormatOptions::DEFAULT, + NumberSelection::None, + false, + ), + }; + if integer_only { + // Integer annotations intentionally discard precision inherited from a + // preceding number annotation. The integer function itself emits no + // fraction digits either. + let integer_text = match &number { + NumberValue::Integer(value) => value.to_string(), + NumberValue::Decimal(value) => { + truncate_decimal_text(&value.to_string()).ok_or_else(bad_operand)? } + NumberValue::NonFinite(value) => value.to_string(), + }; + number = if matches!(&number, NumberValue::NonFinite(_)) { + NumberValue::NonFinite(integer_text.parse().map_err(|_| bad_operand())?) + } else if let Ok(value) = integer_text.parse::() { + NumberValue::Integer(value) + } else { + NumberValue::Decimal(Decimal::from_str(&integer_text).map_err(|_| bad_operand())?) + }; + } + let format = resolve_number_format_options(inherited_format, options, integer_only)?; + let has_explicit_select = inherited_select || options.get(BuiltinOptionKey::Select).is_some(); + let selection = if inherited_select { + if !options.has_runtime(BuiltinOptionKey::Select) { + on_error(MessageFunctionError::BadOption); } - Value::Str(v) => { - let parsed = parse_number_literal(v).ok_or_else(bad_operand)?; - if integer_only { - Ok(format_signed_string(sign_display, format_trunc(parsed))) - } else if let Some(min) = minimum_fraction_digits { - Ok(format_signed_string( - sign_display, - format!("{parsed:.min$}"), - )) - } else { - Ok(format_signed_number(sign_display, parsed)) - } + NumberSelection::Invalid + } else if options.has_runtime(BuiltinOptionKey::Select) { + // The caller reports this option error before resolving the value. + NumberSelection::Invalid + } else if let Some(select) = options.get(BuiltinOptionKey::Select) { + match select.as_ref() { + "plural" => NumberSelection::Plural, + "ordinal" => NumberSelection::Ordinal, + "exact" => NumberSelection::Exact, + _ => NumberSelection::Invalid, } - Value::Float(v) => { - if integer_only { - Ok(format_signed_string(sign_display, format_trunc(*v))) - } else if let Some(min) = minimum_fraction_digits { - Ok(format_signed_string( - sign_display, - format_float_with_min_fraction_digits(*v, min), - )) - } else { - Ok(format_signed_number(sign_display, *v)) - } + } else { + // A number selector uses cardinal plural rules when no explicit + // selection mode is present. Retain that mode on the stored value so + // direct local/input matching can use the resolved category. + match inherited_selection { + NumberSelection::None => NumberSelection::Plural, + selection => selection, } - _ => { - let text = value_text(catalog, value).ok_or_else(bad_operand)?; - let parsed = parse_number_literal(text).ok_or_else(bad_operand)?; - if integer_only { - Ok(format_signed_string(sign_display, format_trunc(parsed))) - } else if let Some(min) = minimum_fraction_digits { - Ok(format_signed_string( - sign_display, - format!("{parsed:.min$}"), - )) - } else { - Ok(format_signed_number(sign_display, parsed)) - } + }; + let mut resolved = ResolvedNumber::new(number, format, selection, has_explicit_select); + if matches!( + selection, + NumberSelection::Plural | NumberSelection::Ordinal + ) && !matches!(resolved.value, NumberValue::NonFinite(_)) + { + let rules = if selection == NumberSelection::Ordinal { + ordinal_rules + } else { + cardinal_rules + }; + resolved.set_selection_category(Some(resolved_plural_category(&resolved, rules)?)); + } + Ok(resolved) +} + +fn parse_number_value(value: &Value, catalog: &Catalog) -> Result { + if let Value::Float(value) = value + && value.is_sign_negative() + && *value == 0.0 + { + return Ok(NumberValue::Decimal( + Decimal::from_str("-0").map_err(|_| bad_operand())?, + )); + } + if let Value::Float(value) = value + && !value.is_finite() + { + return Ok(NumberValue::NonFinite(*value)); + } + if let Value::Float(value) = value + && let Some(value) = exact_integral_float(*value) + { + return Ok(NumberValue::Integer(value)); + } + let text = match value { + Value::Int(value) => return Ok(NumberValue::Integer(*value)), + Value::Float(value) => value.to_string(), + _ => value_text(catalog, value) + .ok_or_else(bad_operand)? + .to_string(), + }; + // Parsing integers before any floating-point conversion is essential: an + // i64 such as 9007199254740993 must remain exact. + if text == "-0" { + return Ok(NumberValue::Decimal( + Decimal::from_str(&text).map_err(|_| bad_operand())?, + )); + } + if let Ok(value) = text.parse::() { + if parse_number_literal(&text).is_none() { + return Err(bad_operand()); + } + return Ok(NumberValue::Integer(value)); + } + if parse_number_literal(&text).is_none() { + return Err(bad_operand()); + } + let decimal = match Decimal::from_str(&text) { + Ok(decimal) => decimal, + Err(_) => { + let value = text.parse::().map_err(|_| bad_operand())?; + Decimal::from_str(&value.to_string()).map_err(|_| bad_operand())? } - }?; + }; + Ok(NumberValue::Decimal(trim_decimal_end(decimal))) +} + +/// Convert an integral float to an integer only while every integer in its +/// range is exactly representable by `f64`. Larger integral floats retain the +/// existing decimal parsing path so their shortest decimal representation is +/// not changed by a narrowing conversion. +fn exact_integral_float(value: f64) -> Option { + let limit = MAX_EXACT_I64_IN_F64 as f64; + if libm::trunc(value) == value && (-limit..=limit).contains(&value) { + // The range check above makes this cast exact and within i64 bounds. + #[allow( + clippy::cast_possible_truncation, + reason = "the preceding exact range check proves this conversion is lossless" + )] + Some(value as i64) + } else { + None + } +} - let result = apply_maximum_fraction_digits(result, maximum_fraction_digits); - let result = apply_minimum_integer_digits(result, minimum_integer_digits); - let result = apply_grouping_strategy(result, use_grouping); - Ok(result) +fn resolve_number_format_options( + inherited: NumberFormatOptions, + options: &EffectiveOptions<'_>, + integer_only: bool, +) -> Result { + let minimum_fraction_digits = if integer_only { + None + } else { + parse_digit_option_or_inherited( + options, + BuiltinOptionKey::MinimumFractionDigits, + inherited.minimum_fraction_digits, + MAX_FRACTION_DIGITS, + )? + }; + let maximum_fraction_digits = if integer_only { + None + } else { + parse_digit_option_or_inherited( + options, + BuiltinOptionKey::MaximumFractionDigits, + inherited.maximum_fraction_digits, + MAX_FRACTION_DIGITS, + )? + }; + validate_digit_range_relationship(minimum_fraction_digits, maximum_fraction_digits)?; + let minimum_integer_digits = parse_digit_option_or_inherited( + options, + BuiltinOptionKey::MinimumIntegerDigits, + inherited.minimum_integer_digits, + MAX_INTEGER_DIGITS, + )?; + let sign_display = match options.get(BuiltinOptionKey::SignDisplay).as_deref() { + None => inherited.sign_display, + Some("auto") => NumberSignDisplay::Auto, + Some("always") => NumberSignDisplay::Always, + Some("never") => NumberSignDisplay::Never, + Some(_) => return Err(bad_option()), + }; + let notation_scientific = match options.get(BuiltinOptionKey::Notation).as_deref() { + None => inherited.notation_scientific, + Some("scientific") => true, + Some(_) => return Err(bad_option()), + }; + let grouping = match options.get(BuiltinOptionKey::UseGrouping).as_deref() { + None => inherited.grouping, + Some("auto") => NumberGrouping::Auto, + Some("always") => NumberGrouping::Always, + Some("never") => NumberGrouping::Never, + Some("min2") => NumberGrouping::Min2, + Some(_) => return Err(bad_option()), + }; + Ok(NumberFormatOptions { + minimum_fraction_digits, + maximum_fraction_digits, + minimum_integer_digits, + sign_display, + notation_scientific, + grouping, + }) +} + +fn parse_digit_option_or_inherited( + options: &EffectiveOptions<'_>, + key: BuiltinOptionKey, + inherited: Option, + max: usize, +) -> Result, FormatError> { + if let Some(value) = options.get(key) { + let value = value.parse::().map_err(|_| bad_option())?; + if value > max { + return Err(bad_option()); + } + return Ok(Some(value)); + } + Ok(inherited) +} + +fn render_resolved_number(_catalog: &Catalog, number: &ResolvedNumber) -> Option { + let format = number.format; + if let NumberValue::NonFinite(value) = number.value { + return Some(format_signed_string( + match format.sign_display { + NumberSignDisplay::Auto => SignDisplay::Auto, + NumberSignDisplay::Always => SignDisplay::Always, + NumberSignDisplay::Never => SignDisplay::Never, + }, + value.to_string(), + )); + } + if format.notation_scientific { + return Some(format_signed_string( + match format.sign_display { + NumberSignDisplay::Auto => SignDisplay::Auto, + NumberSignDisplay::Always => SignDisplay::Always, + NumberSignDisplay::Never => SignDisplay::Never, + }, + format_scientific_text(&number.text()), + )); + } + let text = format_int_or_decimal_with_min_fraction_digits( + number.text(), + format.minimum_fraction_digits.unwrap_or(0), + ); + let text = apply_maximum_fraction_digits(text, format.maximum_fraction_digits); + let text = format_signed_string( + match format.sign_display { + NumberSignDisplay::Auto => SignDisplay::Auto, + NumberSignDisplay::Always => SignDisplay::Always, + NumberSignDisplay::Never => SignDisplay::Never, + }, + text, + ); + let text = apply_minimum_integer_digits(text, format.minimum_integer_digits); + Some(apply_grouping_strategy( + text, + match format.grouping { + NumberGrouping::Auto => BuiltinGrouping::Auto, + NumberGrouping::Always => BuiltinGrouping::Always, + NumberGrouping::Never => BuiltinGrouping::Never, + NumberGrouping::Min2 => BuiltinGrouping::Min2, + }, + )) +} + +fn format_scientific_text(value: &str) -> String { + let (sign, unsigned) = if let Some(value) = value.strip_prefix('-') { + ("-", value) + } else if let Some(value) = value.strip_prefix('+') { + ("+", value) + } else { + ("", value) + }; + let (integer, fraction) = unsigned + .split_once('.') + .map_or((unsigned, ""), |parts| parts); + let digits = format!("{integer}{fraction}"); + let leading = digits.len() - digits.trim_start_matches('0').len(); + if leading == digits.len() { + return format!("{sign}0E0"); + } + let significant = &digits[leading..]; + let exponent = integer.len().cast_signed() - leading.cast_signed() - 1; + let mut mantissa = significant[..1].to_string(); + let rest = significant[1..].trim_end_matches('0'); + if !rest.is_empty() { + mantissa.push('.'); + mantissa.push_str(rest); + } + format!("{sign}{mantissa}E{exponent}") +} + +fn format_int_or_decimal_with_min_fraction_digits(mut text: String, minimum: usize) -> String { + let current = text + .split_once('.') + .map_or(0, |(_, fraction)| fraction.len()); + if current >= minimum { + return text; + } + if current == 0 { + text.push('.'); + } + for _ in current..minimum { + text.push('0'); + } + text } fn plural_category( @@ -714,8 +1024,8 @@ fn plural_category( rules: &PluralRules, options: &EffectiveOptions<'_>, ) -> Result { - let minimum_fraction_digits = parse_minimum_fraction_digits(options)?; - let maximum_fraction_digits = parse_maximum_fraction_digits(options)?; + let minimum_fraction_digits = parse_number_digit_option(value, options, true)?; + let maximum_fraction_digits = parse_number_digit_option(value, options, false)?; validate_digit_range_relationship(minimum_fraction_digits, maximum_fraction_digits)?; if minimum_fraction_digits.is_none() && maximum_fraction_digits.is_none() { @@ -725,6 +1035,11 @@ fn plural_category( let decimal = Decimal::from_str(&v.to_string()).map_err(|_| bad_operand())?; Ok(rules.category_for(&decimal)) } + Value::Number(number) => match &number.value { + NumberValue::Integer(value) => Ok(rules.category_for(*value)), + NumberValue::Decimal(value) => Ok(rules.category_for(value)), + NumberValue::NonFinite(_) => Err(bad_operand()), + }, _ => { let text = value_text(catalog, value).ok_or_else(bad_operand)?; let decimal = Decimal::from_str(text).map_err(|_| bad_operand())?; @@ -743,6 +1058,59 @@ fn plural_category( Ok(rules.category_for(&decimal)) } +/// Compute the category for a resolved number using the precision that was +/// validated and retained on the value. This is intentionally separate from +/// `plural_category`: stored values must be selectable without reapplying +/// their function or resolving options a second time. +fn resolved_plural_category( + number: &ResolvedNumber, + rules: &PluralRules, +) -> Result { + let minimum_fraction_digits = number.format.minimum_fraction_digits; + let maximum_fraction_digits = number.format.maximum_fraction_digits; + validate_digit_range_relationship(minimum_fraction_digits, maximum_fraction_digits)?; + if minimum_fraction_digits.is_none() && maximum_fraction_digits.is_none() { + return match &number.value { + NumberValue::Integer(value) => Ok(rules.category_for(*value)), + NumberValue::Decimal(value) => Ok(rules.category_for(value)), + NumberValue::NonFinite(_) => Err(bad_operand()), + }; + } + let mut decimal = match &number.value { + NumberValue::Integer(value) => Decimal::from(*value), + NumberValue::Decimal(value) => value.clone(), + NumberValue::NonFinite(_) => return Err(bad_operand()), + }; + + if let Some(minimum) = minimum_fraction_digits { + // The supported option range is small, but keep the conversion + // checked because fixed-decimal positions are i16. + let minimum = i16::try_from(minimum) + .map_err(|_| bad_option())? + .checked_neg() + .ok_or_else(bad_option)?; + if *decimal.magnitude_range().start() > minimum { + decimal.pad_end(minimum); + } + } + if let Some(maximum) = maximum_fraction_digits { + let maximum = i16::try_from(maximum).map_err(|_| bad_option())?; + let current_fraction_digits = decimal + .magnitude_range() + .start() + .checked_neg() + .unwrap_or(i16::MAX) + .max(0); + if current_fraction_digits > maximum { + decimal.round_with_mode( + -maximum, + SignedRoundingMode::Unsigned(UnsignedRoundingMode::HalfExpand), + ); + } + } + Ok(rules.category_for(&decimal)) +} + fn format_plural_operand( value: &Value, catalog: &Catalog, @@ -750,6 +1118,13 @@ fn format_plural_operand( maximum_fraction_digits: Option, ) -> Result { match value { + Value::Number(number) => Ok(apply_maximum_fraction_digits( + minimum_fraction_digits.map_or_else( + || number.text(), + |minimum| format_int_or_decimal_with_min_fraction_digits(number.text(), minimum), + ), + maximum_fraction_digits, + )), Value::Int(v) => { let rendered = if let Some(min) = minimum_fraction_digits { format_int_with_min_fraction_digits(*v, min) @@ -791,6 +1166,33 @@ fn format_plural_operand( } } +fn parse_number_digit_option( + value: &Value, + options: &EffectiveOptions<'_>, + minimum: bool, +) -> Result, FormatError> { + let key = if minimum { + BuiltinOptionKey::MinimumFractionDigits + } else { + BuiltinOptionKey::MaximumFractionDigits + }; + if let Some(found) = options.get(key) { + let value = found.parse::().map_err(|_| bad_option())?; + if value > MAX_FRACTION_DIGITS { + return Err(bad_option()); + } + return Ok(Some(value)); + } + if let Value::Number(number) = value { + return Ok(if minimum { + number.format.minimum_fraction_digits + } else { + number.format.maximum_fraction_digits + }); + } + Ok(None) +} + const CATEGORY_NAMES: [&str; 6] = ["zero", "one", "two", "few", "many", "other"]; const MAX_FRACTION_DIGITS: usize = 20; const MAX_INTEGER_DIGITS: usize = 21; @@ -810,15 +1212,6 @@ fn category_name(category: PluralCategory) -> &'static str { CATEGORY_NAMES[category_index(category)] } -fn format_plural( - value: &Value, - catalog: &Catalog, - rules: &PluralRules, - options: &EffectiveOptions<'_>, -) -> Result { - plural_category(value, catalog, rules, options).map(|c| category_name(c).to_string()) -} - fn parse_builtin_option_key(value: &str) -> Option { Some(match value { "u:dir" => BuiltinOptionKey::UDir, @@ -932,6 +1325,9 @@ fn validate_enum_option( key: BuiltinOptionKey, allowed: &[&str], ) -> Result<(), FormatError> { + if key == BuiltinOptionKey::Select && options.has_runtime(key) { + return Ok(()); + } let Some(value) = options.get(key) else { return Ok(()); }; @@ -1092,6 +1488,7 @@ fn parse_sign_display(options: &EffectiveOptions<'_>) -> SignDisplay { } } +#[cfg(test)] fn format_scientific(value: f64) -> String { if !value.is_finite() { return value.to_string(); @@ -1196,25 +1593,11 @@ fn apply_grouping_min2(value: String) -> String { apply_grouping(value) } -fn parse_use_grouping(options: &EffectiveOptions<'_>) -> BuiltinGrouping { - match options.get(BuiltinOptionKey::UseGrouping).as_deref() { - Some("always") => BuiltinGrouping::Always, - Some("never") => BuiltinGrouping::Never, - Some("min2") => BuiltinGrouping::Min2, - Some("auto") | None => BuiltinGrouping::Auto, - Some(other) => unreachable!("unexpected validated useGrouping value: {other}"), - } -} - -fn format_trunc(value: f64) -> String { - let text = value.to_string(); - truncate_decimal_text(&text).unwrap_or(text) -} - fn numeric_operand(value: &Value, catalog: &Catalog) -> Result { match value { Value::Int(v) => exact_i64_to_f64(*v), Value::Float(v) => Ok(*v), + Value::Number(number) => number.text().parse::().map_err(|_| bad_operand()), _ => value_text(catalog, value) .and_then(parse_number_literal) .ok_or_else(bad_operand), @@ -1226,6 +1609,25 @@ fn format_percent( catalog: &Catalog, options: &EffectiveOptions<'_>, ) -> Result { + if let Value::Number(number) = value { + if let NumberValue::NonFinite(value) = number.value { + return Ok(format_signed_string( + parse_sign_display(options), + format!("{}%", value), + )); + } + let rendered = multiply_decimal_by_100(&number.text())?; + let minimum = parse_minimum_fraction_digits(options)?; + let rendered = if let Some(minimum) = minimum { + format_int_or_decimal_with_min_fraction_digits(rendered, minimum) + } else { + rendered + }; + return Ok(format!( + "{}%", + format_signed_string(parse_sign_display(options), rendered) + )); + } let mut number = numeric_operand(value, catalog)? * 100.0; if number == -0.0 { number = 0.0; @@ -1239,6 +1641,49 @@ fn format_percent( Ok(format!("{rendered}%")) } +fn multiply_decimal_by_100(value: &str) -> Result { + let (negative, value) = if let Some(value) = value.strip_prefix('-') { + (true, value) + } else { + (false, value.strip_prefix('+').unwrap_or(value)) + }; + let (integer, fraction) = value.split_once('.').map_or((value, ""), |parts| parts); + if integer.is_empty() + || !integer.chars().all(|ch| ch.is_ascii_digit()) + || !fraction.chars().all(|ch| ch.is_ascii_digit()) + { + return Err(bad_operand()); + } + let digits = format!("{integer}{fraction}"); + let decimal_pos = integer.len().checked_add(2).ok_or_else(bad_operand)?; + let mut out = if decimal_pos >= digits.len() { + format!("{digits}{}", "0".repeat(decimal_pos - digits.len())) + } else { + let split = decimal_pos; + format!("{}.{digits}", &digits[..split]) + }; + while out.contains('.') && out.ends_with('0') { + out.pop(); + } + if out.ends_with('.') { + out.pop(); + } + let integer_end = out.find('.').unwrap_or(out.len()); + let leading = out[..integer_end] + .bytes() + .take_while(|digit| *digit == b'0') + .count(); + if leading >= integer_end { + out.replace_range(..integer_end, "0"); + } else if leading > 0 { + out.replace_range(..leading, ""); + } + if negative && out != "0" { + out.insert(0, '-'); + } + Ok(out) +} + fn format_currency( value: &Value, catalog: &Catalog, @@ -1273,19 +1718,35 @@ fn looks_like_currency_literal(value: &str) -> bool { parse_number_literal(number).is_some() } -fn format_offset( +fn resolve_offset( value: &Value, catalog: &Catalog, options: &EffectiveOptions<'_>, -) -> Result { - let number = parse_offset_operand(value, catalog)?; - let preserve_plus = value_text(catalog, value).is_some_and(|raw| raw.starts_with('+')); + cardinal_rules: &PluralRules, + ordinal_rules: &PluralRules, +) -> Result { + let (mut number, inherited_format, selection, has_explicit_select) = match value { + Value::Number(number) => ( + number.value.clone(), + number.format, + number.selection, + number.has_explicit_select, + ), + _ => ( + parse_number_value(value, catalog)?, + NumberFormatOptions::DEFAULT, + NumberSelection::Plural, + false, + ), + }; let add = options .get(BuiltinOptionKey::Add) - .map(|raw| parse_number_literal(&raw)); + .map(|raw| parse_integer_adjustment(&raw).ok_or_else(bad_option)) + .transpose()?; let subtract = options .get(BuiltinOptionKey::Subtract) - .map(|raw| parse_number_literal(&raw)); + .map(|raw| parse_integer_adjustment(&raw).ok_or_else(bad_option)) + .transpose()?; if add.is_none() && subtract.is_none() { return Err(bad_option()); @@ -1293,40 +1754,120 @@ fn format_offset( if add.is_some() && subtract.is_some() { return Err(bad_option()); } - if matches!(add, Some(None)) || matches!(subtract, Some(None)) { - return Err(bad_option()); + let adjustment = add + .map(|value| (value, false)) + .or_else(|| subtract.map(|value| (value, true))); + if let Some((adjustment, subtract)) = adjustment { + number = checked_offset(number, adjustment, subtract)?; + } + let format = resolve_number_format_options(inherited_format, options, false)?; + let mut resolved = ResolvedNumber::new(number, format, selection, has_explicit_select); + if matches!( + selection, + NumberSelection::Plural | NumberSelection::Ordinal + ) && !matches!(resolved.value, NumberValue::NonFinite(_)) + { + let rules = if selection == NumberSelection::Ordinal { + ordinal_rules + } else { + cardinal_rules + }; + resolved.set_selection_category(Some(resolved_plural_category(&resolved, rules)?)); + } + Ok(resolved) +} + +fn number_text(value: &NumberValue) -> String { + match value { + NumberValue::Integer(value) => value.to_string(), + NumberValue::Decimal(value) => value.to_string(), + NumberValue::NonFinite(value) => value.to_string(), + } +} + +fn parse_integer_adjustment(value: &str) -> Option { + parse_number_literal(value)?; + value.parse::().ok() +} + +fn parse_number_text(value: &str) -> Result { + if let Ok(value) = value.parse::() { + return Ok(NumberValue::Integer(value)); } + Decimal::from_str(value) + .map(|value| NumberValue::Decimal(trim_decimal_end(value))) + .map_err(|_| bad_operand()) +} + +fn trim_decimal_end(mut value: Decimal) -> Decimal { + value.absolute = value.absolute.trimmed_end(); + value +} - let adjusted = if let Some(Some(value)) = add { - number + value - } else if let Some(Some(value)) = subtract { - number - value +fn checked_offset( + number: NumberValue, + adjustment: i64, + subtract: bool, +) -> Result { + if let NumberValue::NonFinite(value) = number { + return Ok(NumberValue::NonFinite(value)); + } + let text = number_text(&number); + let unsigned = text.trim_start_matches(['-', '+']); + let scale = unsigned + .split_once('.') + .map_or(0, |(_, fraction)| fraction.len()); + if scale > 38 { + return Err(unsupported_operation( + UnsupportedOperation::NumericMagnitude, + )); + } + let digits = unsigned.replace('.', ""); + let magnitude = digits + .parse::() + .map_err(|_| unsupported_operation(UnsupportedOperation::NumericMagnitude))?; + let signed = if text.starts_with('-') { + -magnitude } else { - number + magnitude }; - let sign_display = if preserve_plus { - SignDisplay::Always + let factor = 10_i128 + .checked_pow( + u32::try_from(scale) + .map_err(|_| unsupported_operation(UnsupportedOperation::NumericMagnitude))?, + ) + .ok_or_else(|| unsupported_operation(UnsupportedOperation::NumericMagnitude))?; + let adjustment = i128::from(adjustment) + .checked_mul(factor) + .ok_or_else(|| unsupported_operation(UnsupportedOperation::NumericMagnitude))?; + let result = if subtract { + signed + .checked_sub(adjustment) + .ok_or_else(|| unsupported_operation(UnsupportedOperation::NumericMagnitude))? } else { - parse_sign_display(options) + signed + .checked_add(adjustment) + .ok_or_else(|| unsupported_operation(UnsupportedOperation::NumericMagnitude))? }; - Ok(format_signed_number(sign_display, adjusted)) -} - -fn parse_offset_operand(value: &Value, catalog: &Catalog) -> Result { - match value { - Value::Int(v) => exact_i64_to_f64(*v), - Value::Float(v) => Ok(*v), - _ => { - let raw = value_text(catalog, value).ok_or_else(bad_operand)?; - if let Some(parsed) = parse_number_literal(raw) { - return Ok(parsed); - } - let Some(stripped) = raw.strip_prefix('+') else { - return Err(bad_operand()); - }; - parse_number_literal(stripped).ok_or_else(bad_operand) + let magnitude = result.unsigned_abs(); + let mut rendered = magnitude.to_string(); + if scale != 0 { + if rendered.len() <= scale { + rendered = format!("{}{}", "0".repeat(scale + 1 - rendered.len()), rendered); } + let split = rendered.len() - scale; + rendered.insert(split, '.'); + while rendered.ends_with('0') { + rendered.pop(); + } + if rendered.ends_with('.') { + rendered.pop(); + } + } + if result < 0 { + rendered.insert(0, '-'); } + parse_number_text(&rendered) } fn format_test_select( @@ -1340,6 +1881,13 @@ fn format_test_select( { return Err(implementation_failure(ImplementationFailure::TestSelect)); } + // Preserve only an explicitly resolved test-select value. Raw strings + // still follow the ordinary numeric conversion path below. + if options.get(BuiltinOptionKey::DecimalPlaces).is_none() + && let Value::ResolvedSelect(value) = value + { + return Ok(value.text().to_string()); + } let number = numeric_operand(value, catalog)?; if let Some(raw) = options.get(BuiltinOptionKey::DecimalPlaces) { let dp = raw.parse::().map_err(|_| bad_option())?; @@ -1380,6 +1928,10 @@ struct EffectiveOptions<'a> { } impl<'a> EffectiveOptions<'a> { + fn has_runtime(&self, key: BuiltinOptionKey) -> bool { + self.runtime[key.index()].is_some() + } + fn new( base: &'a [Option; BUILTIN_OPTION_KEY_COUNT], runtime: &'a [(u32, Value)], @@ -1842,7 +2394,15 @@ mod tests { args: &[Value], opts: &[(u32, Value)], ) -> Result { - Host::call(&mut self.host, self.catalog, &self.index, fn_id, args, opts) + Host::call( + &mut self.host, + self.catalog, + &self.index, + fn_id, + args, + opts, + &mut |_| {}, + ) } fn call_select( @@ -1851,7 +2411,15 @@ mod tests { args: &[Value], opts: &[(u32, Value)], ) -> Result { - Host::call_select(&mut self.host, self.catalog, &self.index, fn_id, args, opts) + Host::call_select( + &mut self.host, + self.catalog, + &self.index, + fn_id, + args, + opts, + &mut |_| {}, + ) } } @@ -1990,6 +2558,25 @@ mod tests { } } + fn assert_number_rendered(host: &mut TestBuiltinHost, value: Value, expected: &str) { + assert!( + matches!(value, Value::Number(_)), + "expected resolved number" + ); + let rendered = host + .host + .format_default(host.catalog, &host.index, &value) + .expect("resolved number renders"); + assert_eq!(rendered, expected); + } + + fn assert_string_resolved(value: Value, expected_text: &str) { + match value { + Value::String(value) => assert_eq!(value.text(), expected_text), + other => panic!("expected resolved string, got {other:?}"), + } + } + #[cfg(debug_assertions)] #[test] #[should_panic(expected = "builtin host must only surface function-shaped errors")] @@ -2013,13 +2600,164 @@ mod tests { assert_eq!(host.index.by_id.len(), 1); } + #[test] + fn test_select_preserves_only_explicitly_resolved_precision() { + let mut host = builtin_host(&["test:select decimalPlaces=1", "test:select"]); + let resolved = host + .call(0, &[Value::Int(1)], &[]) + .expect("resolved selector"); + let Value::ResolvedSelect(value) = &resolved else { + panic!("test:select must return a resolved selector"); + }; + assert_eq!(value.text(), "1.0"); + + let aliased = host + .call(1, core::slice::from_ref(&resolved), &[]) + .expect("aliased selector"); + let Value::ResolvedSelect(value) = &aliased else { + panic!("test:select must return a resolved selector"); + }; + assert_eq!(value.text(), "1.0"); + + let raw = host + .call(1, &[Value::Str("1.0".to_string())], &[]) + .expect("raw selector"); + let Value::ResolvedSelect(value) = &raw else { + panic!("test:select must return a resolved selector"); + }; + assert_eq!(value.text(), "1"); + } + + #[test] + fn resolved_number_exposes_exact_text_without_formatting_options() { + let mut host = builtin_host(&["number minimumFractionDigits=2 useGrouping=always"]); + let out = host + .call(0, &[Value::Int(9_007_199_254_740_993)], &[]) + .expect("resolved"); + let Value::Number(number) = out else { + panic!("number function must return a resolved number"); + }; + assert_eq!(number.text(), "9007199254740993"); + assert_eq!( + host.host + .format_default(host.catalog, &host.index, &Value::Number(number)), + Some("9,007,199,254,740,993.00".to_string()) + ); + } + + #[test] + fn dynamic_select_reports_error_and_marks_resolved_number_unselectable() { + let mut host = builtin_host_with_funcs(&["number"], &["select", "exact"]); + let select_id = host.catalog.string_id("select").expect("select in pool"); + let mut errors = Vec::new(); + let out = Host::call( + &mut host.host, + host.catalog, + &host.index, + 0, + &[Value::Int(1)], + &[(select_id, Value::Str("exact".to_string()))], + &mut |error| errors.push(error), + ) + .expect("formatting remains available"); + assert_eq!(errors, vec![MessageFunctionError::BadOption]); + let Value::Number(number) = out else { + panic!("number function must return a resolved number"); + }; + assert_eq!(number.selection, NumberSelection::Invalid); + } + + #[test] + fn static_select_keeps_selection_provenance_on_resolved_number() { + let mut host = builtin_host(&["number select=plural"]); + let out = host + .call(0, &[Value::Int(1)], &[]) + .expect("formatting remains available"); + let Value::Number(number) = out else { + panic!("number function must return a resolved number"); + }; + assert_eq!(number.selection, NumberSelection::Plural); + } + + #[test] + fn inherited_select_reports_error_on_reannotation() { + let mut host = builtin_host(&["number select=plural", "number"]); + let first = host + .call(0, &[Value::Int(1)], &[]) + .expect("first annotation"); + let mut errors = Vec::new(); + let second = Host::call( + &mut host.host, + host.catalog, + &host.index, + 1, + &[first], + &[], + &mut |error| errors.push(error), + ) + .expect("formatting remains available"); + assert_eq!(errors, vec![MessageFunctionError::BadOption]); + let Value::Number(number) = second else { + panic!("number function must return a resolved number"); + }; + assert_eq!(number.selection, NumberSelection::Invalid); + } + + #[test] + fn call_select_does_not_recover_invalid_stored_selection() { + let mut host = + builtin_host_with_funcs(&["number", "number select=plural"], &["select", "exact"]); + let select_id = host.catalog.string_id("select").expect("select in pool"); + let mut errors = Vec::new(); + let stored = Host::call( + &mut host.host, + host.catalog, + &host.index, + 0, + &[Value::Int(1)], + &[(select_id, Value::Str("exact".to_string()))], + &mut |error| errors.push(error), + ) + .expect("formatting remains available"); + assert_eq!(errors, vec![MessageFunctionError::BadOption]); + let selected = host + .call_select(1, &[stored], &[]) + .expect("invalid selection uses default"); + assert_eq!(selected, Value::Null); + } + + #[test] + fn call_select_uses_valid_mode_from_stored_number() { + let mut host = builtin_host(&["number select=plural", "number"]); + let stored = host.call(0, &[Value::Int(1)], &[]).expect("stored number"); + let selected = host + .call_select(1, &[stored], &[]) + .expect("stored selection"); + assert_selector_result(host.catalog, selected, "one"); + } + + #[test] + fn reannotation_validates_inherited_fraction_options() { + let mut host = builtin_host(&[ + "number minimumFractionDigits=3", + "number maximumFractionDigits=2", + ]); + let resolved = host + .call(0, &[Value::Float(4.2)], &[]) + .expect("first annotation"); + let err = host + .call(1, &[resolved], &[]) + .expect_err("merged options must be validated"); + assert_function_error(err, MessageFunctionError::BadOption); + } + #[test] fn builtin_host_applies_number_minimum_fraction_digits() { let mut host = builtin_host(&["number minimumFractionDigits=2"]); let out = host .call(0, &[Value::Str("4.2".to_string())], &[]) .expect("formatted"); - assert_eq!(out, Value::Str("4.20".to_string())); + assert_number_rendered(&mut host, out, "4.20"); } #[test] @@ -2049,7 +2787,41 @@ mod tests { fn builtin_host_formats_integral_float_minimum_fraction_digits() { let mut host = builtin_host(&["number minimumFractionDigits=3"]); let out = host.call(0, &[Value::Float(42.0)], &[]).expect("formatted"); - assert_eq!(out, Value::Str("42.000".to_string())); + assert_number_rendered(&mut host, out, "42.000"); + } + + #[test] + fn builtin_host_keeps_exact_integral_float_payloads_integer() { + let mut host = builtin_host(&["number"]); + for (input, expected) in [ + (MAX_EXACT_I64_IN_F64 as f64, MAX_EXACT_I64_IN_F64), + (-(MAX_EXACT_I64_IN_F64 as f64), -MAX_EXACT_I64_IN_F64), + ] { + let out = host + .call(0, &[Value::Float(input)], &[]) + .expect("formatted"); + let Value::Number(number) = out else { + panic!("expected resolved number"); + }; + assert_eq!(number.value, NumberValue::Integer(expected)); + } + } + + #[test] + fn builtin_host_keeps_fractional_and_large_integral_float_decimals() { + let mut host = builtin_host(&["number"]); + let fractional = host.call(0, &[Value::Float(4.25)], &[]).expect("formatted"); + let Value::Number(number) = fractional else { + panic!("expected resolved number"); + }; + assert!(matches!(number.value, NumberValue::Decimal(_))); + + let large = host.call(0, &[Value::Float(1e23)], &[]).expect("formatted"); + let Value::Number(number) = large else { + panic!("expected resolved number"); + }; + assert!(matches!(number.value, NumberValue::Decimal(_))); + assert_eq!(number_text(&number.value), "100000000000000000000000"); } #[test] @@ -2058,7 +2830,7 @@ mod tests { let out = host .call(0, &[Value::Int(i64::MAX)], &[]) .expect("formatted"); - assert_eq!(out, Value::Str(format!("{}.00", i64::MAX))); + assert_number_rendered(&mut host, out, &format!("{}.00", i64::MAX)); } #[test] @@ -2067,7 +2839,7 @@ mod tests { let out = host .call(0, &[Value::Float(4.256)], &[]) .expect("formatted"); - assert_eq!(out, Value::Str("4.26".to_string())); + assert_number_rendered(&mut host, out, "4.26"); } #[test] @@ -2105,7 +2877,31 @@ mod tests { fn builtin_host_preserves_negative_zero_fraction_formatting() { let mut host = builtin_host(&["number minimumFractionDigits=2"]); let out = host.call(0, &[Value::Float(-0.0)], &[]).expect("formatted"); - assert_eq!(out, Value::Str("-0.00".to_string())); + assert_number_rendered(&mut host, out, "-0.00"); + } + + #[test] + fn builtin_host_preserves_negative_zero_string_sign() { + let mut host = builtin_host(&["number"]); + let out = host + .call(0, &[Value::Str("-0".to_string())], &[]) + .expect("formatted"); + assert_number_rendered(&mut host, out, "-0"); + } + + #[test] + fn builtin_host_preserves_nonfinite_number_rendering() { + let mut host = builtin_host(&["number"]); + let infinity = host + .call(0, &[Value::Float(f64::INFINITY)], &[]) + .expect("formatted"); + assert_number_rendered(&mut host, infinity, "inf"); + + let mut offset = builtin_host(&["offset add=1"]); + let infinity = offset + .call(0, &[Value::Float(f64::INFINITY)], &[]) + .expect("formatted"); + assert_number_rendered(&mut offset, infinity, "inf"); } #[test] @@ -2125,7 +2921,7 @@ mod tests { &[(mfd_str_id, Value::Str("3".to_string()))], ) .expect("formatted"); - assert_eq!(out, Value::Str("4.200".to_string())); + assert_number_rendered(&mut host, out, "4.200"); } #[test] @@ -2140,7 +2936,7 @@ mod tests { &[(mystery_str_id, Value::Str("7".to_string()))], ) .expect("formatted"); - assert_eq!(out, Value::Str("4.20".to_string())); + assert_number_rendered(&mut host, out, "4.20"); } #[test] @@ -2162,21 +2958,19 @@ mod tests { ( "string u:dir=ltr", "\u{0645}\u{0631}\u{062D}\u{0628}\u{0627}", - "\u{2066}\u{0645}\u{0631}\u{062D}\u{0628}\u{0627}\u{2069}", ), - ("string u:dir=rtl", "hello", "\u{2067}hello\u{2069}"), + ("string u:dir=rtl", "hello"), ( "string u:dir=auto", "\u{05E9}\u{05DC}\u{05D5}\u{05DD} world", - "\u{2068}\u{05E9}\u{05DC}\u{05D5}\u{05DD} world\u{2069}", ), ]; - for (func, input, expected) in cases { + for (func, input) in cases { let mut host = builtin_host(&[func]); let out = host .call(0, &[Value::Str(input.to_string())], &[]) .expect("formatted"); - assert_eq!(out, Value::Str(expected.to_string())); + assert_string_resolved(out, input); } } @@ -2186,14 +2980,14 @@ mod tests { let out = host .call(0, &[Value::Str("abc".to_string())], &[]) .expect("formatted"); - assert_eq!(out, Value::Str("\u{2067}abc\u{2069}".to_string())); + assert_string_resolved(out, "abc"); } #[test] fn number_option_key_with_bidi_controls_is_recognized() { let mut host = builtin_host(&["number \u{2068}minimumFractionDigits\u{2069}=2"]); let out = host.call(0, &[Value::Float(4.2)], &[]).expect("formatted"); - assert_eq!(out, Value::Str("4.20".to_string())); + assert_number_rendered(&mut host, out, "4.20"); } #[test] @@ -2201,7 +2995,7 @@ mod tests { let mut host = builtin_host(&["string u:dir=auto"]); let input = Value::Str("\u{2066}world\u{2069}".to_string()); let out = host.call(0, &[input], &[]).expect("formatted"); - assert_eq!(out, Value::Str("\u{2066}world\u{2069}".to_string())); + assert_string_resolved(out, "\u{2066}world\u{2069}"); } #[test] @@ -2213,12 +3007,12 @@ mod tests { let string_out = host .call(0, &[Value::StrRef(hello_id)], &[]) .expect("formatted"); - assert_eq!(string_out, Value::Str("\u{2068}hello\u{2069}".to_string())); + assert_string_resolved(string_out, "hello"); let number_out = host .call(1, &[Value::StrRef(number_id)], &[]) .expect("formatted"); - assert_eq!(number_out, Value::Str("42.5".to_string())); + assert_number_rendered(&mut host, number_out, "42.5"); } #[test] @@ -2228,12 +3022,12 @@ mod tests { let string_out = host .call(0, &[Value::LitRef { off: 0, len: 5 }], &[]) .expect("formatted"); - assert_eq!(string_out, Value::Str("\u{2068}hello\u{2069}".to_string())); + assert_string_resolved(string_out, "hello"); let number_out = host .call(1, &[Value::LitRef { off: 5, len: 4 }], &[]) .expect("formatted"); - assert_eq!(number_out, Value::Str("42.5".to_string())); + assert_number_rendered(&mut host, number_out, "42.5"); } #[test] @@ -2241,8 +3035,16 @@ mod tests { let mut host = builtin_host(&["number select=plural"]); let one = host.call(0, &[Value::Int(1)], &[]).expect("formatted"); let other = host.call(0, &[Value::Int(2)], &[]).expect("formatted"); - assert_eq!(one, Value::Str("one".to_string())); - assert_eq!(other, Value::Str("other".to_string())); + let Value::Number(one_number) = &one else { + panic!("number function must return a resolved number"); + }; + assert_eq!(one_number.selection_category, Some(PluralCategory::One)); + let Value::Number(other_number) = &other else { + panic!("number function must return a resolved number"); + }; + assert_eq!(other_number.selection_category, Some(PluralCategory::Other)); + assert_number_rendered(&mut host, one, "1"); + assert_number_rendered(&mut host, other, "2"); } #[test] @@ -2252,10 +3054,14 @@ mod tests { let two = host.call(0, &[Value::Int(2)], &[]).expect("formatted"); let few = host.call(0, &[Value::Int(3)], &[]).expect("formatted"); let other = host.call(0, &[Value::Int(11)], &[]).expect("formatted"); - assert_eq!(one, Value::Str("one".to_string())); - assert_eq!(two, Value::Str("two".to_string())); - assert_eq!(few, Value::Str("few".to_string())); - assert_eq!(other, Value::Str("other".to_string())); + let Value::Number(two_number) = &two else { + panic!("number function must return a resolved number"); + }; + assert_eq!(two_number.selection_category, Some(PluralCategory::Two)); + assert_number_rendered(&mut host, one, "1"); + assert_number_rendered(&mut host, two, "2"); + assert_number_rendered(&mut host, few, "3"); + assert_number_rendered(&mut host, other, "11"); } #[test] @@ -2271,28 +3077,38 @@ mod tests { fn number_call_select_runtime_override_still_uses_dynamic_select() { let mut host = builtin_host_with_funcs(&["number select=plural"], &["select", "ordinal"]); let select_id = host.catalog.string_id("select").expect("select in pool"); - let out = host - .call_select( - 0, - &[Value::Int(2)], - &[(select_id, Value::Str("ordinal".to_string()))], - ) - .expect("formatted"); - assert_selector_result(host.catalog, out, "two"); + let mut errors = Vec::new(); + let out = Host::call_select( + &mut host.host, + host.catalog, + &host.index, + 0, + &[Value::Int(2)], + &[(select_id, Value::Str("ordinal".to_string()))], + &mut |error| errors.push(error), + ) + .expect("formatted"); + assert_eq!(out, Value::Null); + assert_eq!(errors, vec![MessageFunctionError::BadOption]); } #[test] fn number_call_select_rejects_invalid_runtime_select_override() { let mut host = builtin_host_with_funcs(&["number select=plural"], &["select", "bogus"]); let select_id = host.catalog.string_id("select").expect("select in pool"); - let err = host - .call_select( - 0, - &[Value::Int(1)], - &[(select_id, Value::Str("bogus".to_string()))], - ) - .expect_err("must fail"); - assert_function_error(err, MessageFunctionError::BadOption); + let mut errors = Vec::new(); + let out = Host::call_select( + &mut host.host, + host.catalog, + &host.index, + 0, + &[Value::Int(1)], + &[(select_id, Value::Str("bogus".to_string()))], + &mut |error| errors.push(error), + ) + .expect("selector fallback"); + assert_eq!(out, Value::Null); + assert_eq!(errors, vec![MessageFunctionError::BadOption]); } #[test] @@ -2301,14 +3117,14 @@ mod tests { let out = host .call_select(0, &[Value::Int(1)], &[]) .expect("formatted"); - assert_eq!(out, Value::Str("other".to_string())); + assert_selector_result(host.catalog, out, "other"); } #[test] fn number_select_exact_returns_formatted_number() { let mut host = builtin_host(&["number select=exact"]); let out = host.call(0, &[Value::Int(42)], &[]).expect("formatted"); - assert_eq!(out, Value::Str("42".to_string())); + assert_number_rendered(&mut host, out, "42"); } #[test] @@ -2316,15 +3132,15 @@ mod tests { let mut host = builtin_host(&["integer select=plural"]); let one = host.call(0, &[Value::Int(1)], &[]).expect("formatted"); let other = host.call(0, &[Value::Int(2)], &[]).expect("formatted"); - assert_eq!(one, Value::Str("one".to_string())); - assert_eq!(other, Value::Str("other".to_string())); + assert_number_rendered(&mut host, one, "1"); + assert_number_rendered(&mut host, other, "2"); } #[test] fn number_without_select_returns_formatted_number() { let mut host = builtin_host(&["number"]); let out = host.call(0, &[Value::Int(42)], &[]).expect("formatted"); - assert_eq!(out, Value::Str("42".to_string())); + assert_number_rendered(&mut host, out, "42"); } #[test] @@ -2384,12 +3200,48 @@ mod tests { } #[test] - fn offset_rejects_large_integer_that_would_lose_precision() { + fn offset_rejects_large_integer_that_exceeds_checked_range() { let mut host = builtin_host(&["offset add=1"]); let err = host .call(0, &[Value::Int(i64::MAX)], &[]) - .expect_err("must fail"); - assert_function_error(err, MessageFunctionError::BadOperand); + .expect("i128 range"); + assert_number_rendered(&mut host, err, "9223372036854775808"); + + let mut huge = builtin_host(&["offset add=1"]); + let err = huge + .call( + 0, + &[Value::Str( + "99999999999999999999999999999999999999999".to_string(), + )], + &[], + ) + .expect_err("must exceed checked range"); + assert_function_error( + err, + MessageFunctionError::UnsupportedOperation(UnsupportedOperation::NumericMagnitude), + ); + } + + #[test] + fn offset_rejects_missing_or_non_integer_adjustments() { + let mut missing = builtin_host(&["offset"]); + let err = missing + .call(0, &[Value::Int(4)], &[]) + .expect_err("missing adjustment must fail"); + assert_function_error(err, MessageFunctionError::BadOption); + + let mut fractional = builtin_host(&["offset add=1.5"]); + let err = fractional + .call(0, &[Value::Int(4)], &[]) + .expect_err("fractional adjustment must fail"); + assert_function_error(err, MessageFunctionError::BadOption); + + let mut invalid = builtin_host(&["offset add=bogus"]); + let err = invalid + .call(0, &[Value::Int(4)], &[]) + .expect_err("invalid adjustment must fail"); + assert_function_error(err, MessageFunctionError::BadOption); } #[test] @@ -2463,4 +3315,108 @@ mod tests { "9007199254740993.26" ); } + + fn resolved_plural_category_via_text( + number: &ResolvedNumber, + rules: &PluralRules, + ) -> Result { + let minimum = number.format.minimum_fraction_digits; + let maximum = number.format.maximum_fraction_digits; + validate_digit_range_relationship(minimum, maximum)?; + let text = minimum.map_or_else( + || number.text(), + |minimum| format_int_or_decimal_with_min_fraction_digits(number.text(), minimum), + ); + let text = apply_maximum_fraction_digits(text, maximum); + let decimal = Decimal::from_str(&text).map_err(|_| bad_operand())?; + Ok(rules.category_for(&decimal)) + } + + #[test] + fn resolved_plural_category_decimal_path_matches_text_reference() { + let cases = [ + ("1.25", None, Some(1)), + ("-1.25", None, Some(1)), + ("9.995", None, Some(2)), + ("-9.995", None, Some(2)), + ("1.5", None, Some(0)), + ("-1.5", None, Some(0)), + ("1.2", Some(2), Some(2)), + ("-0", Some(2), Some(2)), + ("1.2300", Some(2), Some(4)), + ("1.2300", Some(2), Some(3)), + ]; + + for locale_name in ["en", "ru"] { + let locale = locale_name.parse().expect("locale"); + let host = BuiltinHost::new(&locale).expect("host"); + for (text, minimum, maximum) in cases { + let number = ResolvedNumber::new( + NumberValue::Decimal(Decimal::from_str(text).expect("decimal")), + NumberFormatOptions { + minimum_fraction_digits: minimum, + maximum_fraction_digits: maximum, + ..NumberFormatOptions::DEFAULT + }, + NumberSelection::Plural, + true, + ); + assert_eq!( + resolved_plural_category(&number, &host.cardinal_rules), + resolved_plural_category_via_text(&number, &host.cardinal_rules), + "locale={locale_name} value={text} min={minimum:?} max={maximum:?}" + ); + } + + let integer = ResolvedNumber::new( + NumberValue::Integer(1), + NumberFormatOptions { + maximum_fraction_digits: Some(2), + ..NumberFormatOptions::DEFAULT + }, + NumberSelection::Plural, + true, + ); + assert_eq!( + resolved_plural_category(&integer, &host.cardinal_rules), + resolved_plural_category_via_text(&integer, &host.cardinal_rules), + "locale={locale_name} integer" + ); + } + } + + #[test] + fn offset_uses_checked_decimal_scaling_without_f64_rounding() { + assert_eq!( + number_text( + &checked_offset(NumberValue::Integer(9_007_199_254_740_993), 1, false,) + .expect("sum") + ), + "9007199254740994" + ); + assert_eq!( + number_text( + &checked_offset( + NumberValue::Decimal(Decimal::from_str("0.5").expect("decimal")), + 1, + false, + ) + .expect("sum") + ), + "1.5" + ); + assert_eq!( + number_text(&checked_offset(NumberValue::Integer(i64::MAX), 1, false).expect("sum")), + "9223372036854775808" + ); + } + + #[test] + fn percent_resolved_decimal_shifts_without_f64_rounding() { + assert_eq!(multiply_decimal_by_100("0.5").expect("percent"), "50"); + assert_eq!( + multiply_decimal_by_100("9007199254740993").expect("percent"), + "900719925474099300" + ); + } } diff --git a/crates/message-format/src/runtime/catalog.rs b/crates/message-format/src/runtime/catalog.rs index 3067cdc..8dcf223 100644 --- a/crates/message-format/src/runtime/catalog.rs +++ b/crates/message-format/src/runtime/catalog.rs @@ -511,7 +511,9 @@ fn verify_code( match decoded.opcode { Opcode::ExprFallback if !expr_fallback_pending => expr_fallback_pending = true, - Opcode::CallFunc | Opcode::CallSelect => expr_fallback_pending = false, + Opcode::CallFunc | Opcode::CallSelect | Opcode::StoreLocal => { + expr_fallback_pending = false; + } _ => { if expr_fallback_pending { return Err(CatalogError::InvalidExprFallbackSequence { pc }); @@ -588,6 +590,12 @@ fn validate_instruction_operands( return Err(CatalogError::InvalidStringRef { pc: decoded.pc, id }); } } + Opcode::CheckSelector | Opcode::StoreLocal | Opcode::LoadLocal | Opcode::SelectLocal => { + // The runtime validates dense-slot ordering because the valid + // range depends on the execution path. Decoding still checks the + // complete u32 operand here, including truncated instructions. + let _ = read_u32(code, base + 1)?; + } Opcode::OutSlice | Opcode::OutExpr => { let offset = read_u32(code, base + 1)?; let len = read_u32(code, base + 5)?; @@ -636,9 +644,10 @@ fn validate_literal_ref( fn stack_effect(code: &[u8], decoded: vm::Decoded) -> (u32, u32) { let base = decoded.pc as usize; match decoded.opcode { - Opcode::JmpIfFalse | Opcode::OutVal | Opcode::SelectBegin => (1, 0), + Opcode::JmpIfFalse | Opcode::OutVal | Opcode::SelectBegin | Opcode::StoreLocal => (1, 0), Opcode::PushConst | Opcode::LoadArg => (0, 1), - Opcode::OutArg | Opcode::SelectArg => (0, 0), + Opcode::LoadLocal => (0, 1), + Opcode::CheckSelector | Opcode::OutArg | Opcode::SelectArg | Opcode::SelectLocal => (0, 0), Opcode::CallFunc | Opcode::CallSelect => { let arg_count = u32::from(code[base + 3]); let optc = u32::from(code[base + 4]); @@ -681,6 +690,8 @@ struct AbstractExecutionState { pc: u32, /// Minimum stack depth min_stack_depth: u32, + /// Number of contiguous local slots definitely initialized on every path. + initialized_locals: u32, /// Select depth select_depth: u8, } @@ -691,6 +702,7 @@ impl AbstractExecutionState { cycle: 0, pc, min_stack_depth: 0, + initialized_locals: 0, select_depth: 0, } } @@ -701,6 +713,7 @@ impl AbstractExecutionVerifier { /// /// - Termination /// - Stack depth >= 0 at all times + /// - Local slots are loaded only after definite dense-prefix initialization /// - Select depth >= 0 at all times /// - Select sequencing (begin, case*, end) and termination /// @@ -720,6 +733,7 @@ impl AbstractExecutionVerifier { mut cycle, mut pc, mut min_stack_depth, + mut initialized_locals, mut select_depth, })) = heap.pop() { @@ -732,6 +746,7 @@ impl AbstractExecutionVerifier { && peeked.select_depth == select_depth { min_stack_depth = min_stack_depth.min(peeked.min_stack_depth); + initialized_locals = initialized_locals.min(peeked.initialized_locals); heap.pop(); heap_peek = heap.peek().cloned(); } @@ -741,6 +756,26 @@ impl AbstractExecutionVerifier { update_select_depth(&mut select_depth, decoded)?; let (pops, pushes) = stack_effect(code, decoded); + match decoded.opcode { + Opcode::CheckSelector | Opcode::LoadLocal | Opcode::SelectLocal => { + let slot = read_u32(code, pc as usize + 1)?; + if slot >= initialized_locals { + return Err(CatalogError::InvalidLocalSlot { pc, slot }); + } + } + Opcode::StoreLocal => { + let slot = read_u32(code, pc as usize + 1)?; + if slot > initialized_locals { + return Err(CatalogError::InvalidLocalSlot { pc, slot }); + } + if slot == initialized_locals { + initialized_locals = initialized_locals + .checked_add(1) + .ok_or(CatalogError::InvalidLocalSlot { pc, slot })?; + } + } + _ => {} + } if (pops, pushes) != (0, 0) { if min_stack_depth < pops { return Err(CatalogError::BadPc { pc }); @@ -795,6 +830,7 @@ impl AbstractExecutionVerifier { cycle, pc: target, min_stack_depth, + initialized_locals, select_depth, })); } @@ -814,7 +850,7 @@ impl AbstractExecutionVerifier { fn update_select_depth(select_depth: &mut u8, decoded: vm::Decoded) -> Result<(), CatalogError> { match decoded.opcode { - Opcode::SelectArg | Opcode::SelectBegin => { + Opcode::SelectArg | Opcode::SelectLocal | Opcode::SelectBegin => { *select_depth = select_depth .checked_add(1) @@ -1129,6 +1165,233 @@ mod tests { assert!(matches!(err, CatalogError::ChunkOutOfBounds)); } + #[test] + fn truncated_local_operand_fails_decode() { + let bytes = build_catalog( + &["main"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &[Opcode::LoadLocal as u8, 0, 0], + ); + let err = Catalog::from_bytes(&bytes).expect_err("must reject truncated operand"); + assert!(matches!(err, CatalogError::TruncatedInstruction { pc: 0 })); + } + + #[test] + fn truncated_local_selector_operands_fail_decode() { + for opcode in [Opcode::CheckSelector, Opcode::SelectLocal] { + let bytes = build_catalog( + &["main"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &[opcode as u8, 0, 0], + ); + let err = Catalog::from_bytes(&bytes).expect_err("must reject truncated operand"); + assert!(matches!(err, CatalogError::TruncatedInstruction { pc: 0 })); + } + } + + #[test] + fn local_load_before_store_is_rejected() { + let code = [Opcode::LoadLocal as u8, 0, 0, 0, 0, Opcode::Halt as u8]; + let bytes = build_catalog( + &["main"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + assert_eq!( + Catalog::from_bytes(&bytes).expect_err("load must follow store"), + CatalogError::InvalidLocalSlot { pc: 0, slot: 0 } + ); + } + + #[test] + fn local_selector_before_store_is_rejected() { + for opcode in [Opcode::CheckSelector, Opcode::SelectLocal] { + let code = [opcode as u8, 0, 0, 0, 0, Opcode::Halt as u8]; + let bytes = build_catalog( + &["main"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + assert_eq!( + Catalog::from_bytes(&bytes).expect_err("selector must follow store"), + CatalogError::InvalidLocalSlot { pc: 0, slot: 0 } + ); + } + } + + #[test] + fn local_store_cannot_skip_dense_slot() { + let code = [ + Opcode::PushConst as u8, + 1, + 0, + 0, + 0, + Opcode::StoreLocal as u8, + 1, + 0, + 0, + 0, + Opcode::Halt as u8, + ]; + let bytes = build_catalog( + &["main", "value"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + assert_eq!( + Catalog::from_bytes(&bytes).expect_err("store must not skip slot 0"), + CatalogError::InvalidLocalSlot { pc: 5, slot: 1 } + ); + } + + #[test] + fn local_slot_replacement_is_valid_after_initialization() { + let code = TestOps::new() + .push_const(1) + .store_local(0) + .push_const(1) + .store_local(0) + .halt() + .build(); + let bytes = build_catalog( + &["main", "value"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + Catalog::from_bytes(&bytes).expect("replacement is valid"); + } + + #[test] + fn local_initialization_on_only_one_branch_is_rejected() { + let code = TestOps::new() + .push_const(1) + .jmp_if_false("init") + .push_const(1) + .store_local(0) + .label("init") + .load_local(0) + .out_val() + .halt() + .build(); + let bytes = build_catalog( + &["main", "value"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + let err = Catalog::from_bytes(&bytes).expect_err("one branch leaves slot uninitialized"); + assert!(matches!(err, CatalogError::InvalidLocalSlot { .. })); + } + + #[test] + fn local_initialization_on_both_branches_is_valid() { + let code = TestOps::new() + .push_const(1) + .jmp_if_false("else") + .push_const(1) + .store_local(0) + .jmp("join") + .label("else") + .push_const(1) + .store_local(0) + .label("join") + .load_local(0) + .out_val() + .halt() + .build(); + let bytes = build_catalog( + &["main", "value"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + Catalog::from_bytes(&bytes).expect("both branches initialize slot"); + } + + #[test] + fn local_load_before_store_on_backward_path_is_rejected() { + let code = TestOps::new() + .label("loop") + .load_local(0) + .out_val() + .push_const(1) + .store_local(0) + .jmp("loop") + .build(); + let bytes = build_catalog( + &["main", "value"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + let err = Catalog::from_bytes(&bytes).expect_err("backedge cannot initialize prior load"); + assert!(matches!(err, CatalogError::InvalidLocalSlot { .. })); + } + + #[test] + fn local_initialization_does_not_cross_message_entries() { + let first = TestOps::new().push_const(1).store_local(0).halt().build(); + let second_pc = u32::try_from(first.len()).expect("pc"); + let mut code = first; + code.extend_from_slice(&TestOps::new().load_local(0).out_val().halt().build()); + let bytes = build_catalog( + &["first", "main", "value"], + "", + &[ + MessageEntry { + name_str_id: 0, + entry_pc: 0, + }, + MessageEntry { + name_str_id: 1, + entry_pc: second_pc, + }, + ], + &code, + ); + assert_eq!( + Catalog::from_bytes(&bytes).expect_err("entries must initialize locals independently"), + CatalogError::InvalidLocalSlot { + pc: second_pc, + slot: 0, + } + ); + } + #[test] fn invalid_utf8_string_table_is_rejected() { let code = [Opcode::Halt as u8]; diff --git a/crates/message-format/src/runtime/error.rs b/crates/message-format/src/runtime/error.rs index 23c6476..9985262 100644 --- a/crates/message-format/src/runtime/error.rs +++ b/crates/message-format/src/runtime/error.rs @@ -31,6 +31,13 @@ pub enum CatalogError { /// Invalid program counter. pc: u32, }, + /// Bytecode accessed a local slot before it was initialized or skipped a slot. + InvalidLocalSlot { + /// Program counter of the offending instruction. + pc: u32, + /// Referenced local slot. + slot: u32, + }, /// A jump target did not land on an instruction boundary. BadJump { /// Source instruction program counter. @@ -152,6 +159,8 @@ pub enum UnsupportedOperation { TimeFormattingForLocale, /// Datetime formatting is unavailable for the active locale. DateTimeFormattingForLocale, + /// Numeric offset arithmetic exceeded the implementation's checked range. + NumericMagnitude, } /// Implementation-defined message-function failures. @@ -259,6 +268,8 @@ pub enum Trap { InvalidFallbackStringId, /// A program counter computation overflowed the runtime range. ProgramCounterOverflow, + /// A local slot was uninitialized or outside the dense slot range. + InvalidLocalSlot, /// A multi-formatter catalog slot index was out of range. InvalidCatalogIndex, } @@ -292,6 +303,9 @@ impl fmt::Display for CatalogError { Self::MissingChunk(tag) => write!(f, "required catalog chunk {tag} was missing"), Self::InvalidUtf8 => f.write_str("catalog string data was not valid UTF-8"), Self::BadPc { pc } => write!(f, "catalog message entry pointed at invalid pc {pc}"), + Self::InvalidLocalSlot { pc, slot } => { + write!(f, "invalid local slot {slot} at pc {pc}") + } Self::BadJump { from_pc, to_pc } => { write!( f, @@ -417,6 +431,9 @@ impl fmt::Display for UnsupportedOperation { Self::DateTimeFormattingForLocale => f.write_str( "unsupported operation: datetime formatting is not supported for this locale", ), + Self::NumericMagnitude => { + f.write_str("unsupported operation: numeric magnitude is out of range") + } } } } @@ -465,6 +482,7 @@ impl fmt::Display for Trap { Self::MarkupOptionKeyWrongType => "MARKUP option key must be int/strref", Self::InvalidFallbackStringId => "invalid fallback str id", Self::ProgramCounterOverflow => "program counter overflow", + Self::InvalidLocalSlot => "invalid local slot", Self::InvalidCatalogIndex => "invalid catalog index", }) } diff --git a/crates/message-format/src/runtime/formatter.rs b/crates/message-format/src/runtime/formatter.rs index b4a5873..9b4b0eb 100644 --- a/crates/message-format/src/runtime/formatter.rs +++ b/crates/message-format/src/runtime/formatter.rs @@ -17,6 +17,7 @@ use crate::runtime::{ pub(crate) struct VmState { pub(crate) fuel: Option, pub(crate) stack: Vec, + pub(crate) locals: Vec, pub(crate) call_args: Vec, pub(crate) call_options: Vec<(u32, Value)>, } @@ -112,7 +113,7 @@ impl<'a, H: Host> Formatter<'a, H> { ) -> Result<(), FormatError> { #[cfg(feature = "profiling")] profiling::function_scope!(); - run_bytecode( + let result = run_bytecode( self.catalog, &mut self.host, &self.index, @@ -120,11 +121,14 @@ impl<'a, H: Host> Formatter<'a, H> { args, self.vm.fuel, &mut self.vm.stack, + &mut self.vm.locals, sink, diagnostics, &mut self.vm.call_args, &mut self.vm.call_options, - )?; + ); + self.vm.locals.clear(); + result?; Ok(()) } } @@ -281,7 +285,7 @@ impl<'a, H: Host> MultiFormatter<'a, H> { .catalogs .get(message.catalog_idx as usize) .ok_or(FormatError::Trap(Trap::InvalidCatalogIndex))?; - run_bytecode( + let result = run_bytecode( catalog, &mut self.host, index, @@ -289,11 +293,14 @@ impl<'a, H: Host> MultiFormatter<'a, H> { args, self.vm.fuel, &mut self.vm.stack, + &mut self.vm.locals, sink, diagnostics, &mut self.vm.call_args, &mut self.vm.call_options, - )?; + ); + self.vm.locals.clear(); + result?; Ok(()) } } @@ -320,6 +327,10 @@ mod tests { Catalog::from_bytes(&bytes).expect("valid catalog") } + fn local_catalog(message_name: &str, code: &[u8]) -> Catalog { + one_message_catalog(&[message_name, "value"], "", code) + } + use crate::runtime::schema::TestOps; #[test] @@ -416,4 +427,142 @@ mod tests { assert!(diagnostics.is_empty()); assert_eq!(sink, "world"); } + + #[test] + fn formatter_reuses_local_scratch_capacity() { + let code = TestOps::new() + .push_const(1) + .store_local(0) + .load_local(0) + .out_val() + .halt() + .build(); + let catalog = local_catalog("main", &code); + let mut formatter = Formatter::new(&catalog, NoopHost).unwrap(); + let message = formatter.resolve("main").unwrap(); + let mut first = String::new(); + formatter + .format_to(message, &vec![] as &Vec<(u32, Value)>, &mut first, None) + .unwrap(); + let capacity = formatter.vm.locals.capacity(); + assert!(capacity >= 1); + assert!(formatter.vm.locals.is_empty()); + + let mut second = String::new(); + formatter + .format_to(message, &vec![] as &Vec<(u32, Value)>, &mut second, None) + .unwrap(); + assert_eq!(formatter.vm.locals.capacity(), capacity); + assert_eq!(first, "value"); + assert_eq!(second, "value"); + } + + #[test] + fn formatter_clears_locals_after_trap() { + let first_code = TestOps::new().push_const(1).store_local(0).halt().build(); + let second_code = TestOps::new().push_const(2).store_local(0).halt().build(); + let mut code = first_code.clone(); + let second_entry = u32::try_from(code.len()).unwrap(); + code.extend_from_slice(&second_code); + let bytes = build_catalog( + &["first", "second", "value"], + "", + &[ + MessageEntry { + name_str_id: 0, + entry_pc: 0, + }, + MessageEntry { + name_str_id: 1, + entry_pc: second_entry, + }, + ], + &code, + ); + let catalog = Catalog::from_bytes(&bytes).unwrap(); + let mut formatter = Formatter::new(&catalog, NoopHost).unwrap(); + let first = formatter.resolve("first").unwrap(); + let second = formatter.resolve("second").unwrap(); + let mut sink = String::new(); + formatter + .format_to(first, &vec![] as &Vec<(u32, Value)>, &mut sink, None) + .unwrap(); + let capacity = formatter.vm.locals.capacity(); + assert!(capacity >= 1); + assert!(formatter.vm.locals.is_empty()); + + formatter.set_fuel(Some(2)); + assert_eq!( + formatter + .format_to(second, &vec![] as &Vec<(u32, Value)>, &mut sink, None) + .unwrap_err(), + FormatError::Trap(Trap::FuelExhausted) + ); + assert!(formatter.vm.locals.is_empty()); + assert_eq!(formatter.vm.locals.capacity(), capacity); + + formatter.set_fuel(None); + formatter + .format_to(second, &vec![] as &Vec<(u32, Value)>, &mut sink, None) + .expect("rerun after clearing fuel"); + assert!(formatter.vm.locals.is_empty()); + assert_eq!(formatter.vm.locals.capacity(), capacity); + } + + #[test] + fn multi_formatter_does_not_share_locals_between_catalogs() { + let store_first = TestOps::new() + .push_const(1) + .store_local(0) + .load_local(0) + .out_val() + .halt() + .build(); + let store_second = TestOps::new() + .push_const(1) + .store_local(0) + .load_local(0) + .out_val() + .halt() + .build(); + let first = one_message_catalog(&["first", "value"], "", &store_first); + let second = one_message_catalog(&["second", "other"], "", &store_second); + let mut formatter = MultiFormatter::new([&first, &second], NoopHost).unwrap(); + let first_handle = formatter.resolve("first").unwrap(); + let second_handle = formatter.resolve("second").unwrap(); + let mut sink = String::new(); + formatter + .format_to(first_handle, &vec![] as &Vec<(u32, Value)>, &mut sink, None) + .unwrap(); + let capacity = formatter.vm.locals.capacity(); + assert!(capacity >= 1); + assert!(formatter.vm.locals.is_empty()); + formatter.set_fuel(Some(1)); + assert_eq!( + formatter + .format_to( + second_handle, + &vec![] as &Vec<(u32, Value)>, + &mut sink, + None, + ) + .unwrap_err(), + FormatError::Trap(Trap::FuelExhausted) + ); + assert!(formatter.vm.locals.is_empty()); + assert_eq!(formatter.vm.locals.capacity(), capacity); + sink.clear(); + formatter.set_fuel(None); + formatter + .format_to( + second_handle, + &vec![] as &Vec<(u32, Value)>, + &mut sink, + None, + ) + .expect("second catalog formats"); + assert_eq!(sink, "other"); + assert!(formatter.vm.locals.is_empty()); + assert_eq!(formatter.vm.locals.capacity(), capacity); + } } diff --git a/crates/message-format/src/runtime/mod.rs b/crates/message-format/src/runtime/mod.rs index 4576f22..c53de6c 100644 --- a/crates/message-format/src/runtime/mod.rs +++ b/crates/message-format/src/runtime/mod.rs @@ -151,6 +151,9 @@ //! - [`FormatError`] is returned while resolving/formatting messages. //! - [`HostCallError`] constrains host callbacks to unknown-function failures or //! typed [`MessageFunctionError`] values. +//! - [`Host::call`] and +//! [`Host::call_select`] can report a +//! recoverable function diagnostic while still returning a value. //! - [`MessageFunctionError`] carries spec-shaped function error categories, with //! structured sub-enums for unsupported operations and implementation-defined //! host failures. @@ -162,6 +165,32 @@ //! The runtime owns the executable catalog schema in [`schema`]. The compiler //! targets that shared schema when emitting binary catalogs, and the runtime //! verifier/VM interpret the same schema when loading and executing them. +//! +//! # Migration +//! +//! Host implementations must accept the `on_error` callback on `call` and +//! `call_select`. With ICU4X enabled, numeric built-ins return a resolved +//! numeric value internally; callers that inspect host results should render +//! values through [`Host::format_default`], or inspect the exact value with +//! the `ResolvedNumber::text` accessor. Exhaustive `Value` matches should +//! handle the numeric, resolved-test-selector, and fallback variants. Offset arithmetic can report +//! [`UnsupportedOperation::NumericMagnitude`] when checked storage limits +//! are exceeded. +//! +//! `Value::String` carries a resolved string payload and direction metadata. +//! Use `ResolvedString::text` for semantic text and +//! [`Host::format_default`] when rendering so direction metadata is applied at +//! output. Exhaustive `Value` matches must account for this resolved variant. +//! +//! Catalog loading now rejects local loads without prior initialization on +//! every reachable path, and stores that skip a local slot. Exhaustive +//! [`CatalogError`] matches must handle [`CatalogError::InvalidLocalSlot`]. +//! +//! Recompile catalogs to use eager string declarations and the separate +//! `CheckSelector`/`SelectLocal` instructions. Structured compiler inputs gain +//! `Part::CheckSelector` and `SelectorExpr::CheckedLocal`; exhaustive semantic +//! matches must handle these variants. The check belongs before dispatch, +//! once per source selector, including repeated uses of the same local. #[cfg(feature = "icu4x")] #[cfg_attr(docsrs, doc(cfg(feature = "icu4x")))] @@ -179,7 +208,9 @@ pub use error::{ }; pub use formatter::{Formatter, MultiFormatter, MultiMessageHandle}; pub use schema::{FuncEntry, MessageEntry, Opcode}; -pub use value::{ArgNameError, Args, MessageArgs, StrId, Value}; +pub use value::{ArgNameError, Args, MessageArgs, ResolvedSelect, StrId, Value}; +#[cfg(feature = "icu4x")] +pub use value::{ResolvedNumber, ResolvedString}; pub use vm::{FormatOption, FormatSink, Host, HostFn, MessageHandle, NoopHost}; /// Catalog decoding and verification. diff --git a/crates/message-format/src/runtime/schema.rs b/crates/message-format/src/runtime/schema.rs index 449150d..89027c4 100644 --- a/crates/message-format/src/runtime/schema.rs +++ b/crates/message-format/src/runtime/schema.rs @@ -56,6 +56,12 @@ pub enum Opcode { PushConst = 0x10, /// Load argument by string-pool id. LoadArg = 0x11, + /// Store the top stack value in a message-local slot. + StoreLocal = 0x12, + /// Load a cloned value from a message-local slot. + LoadLocal = 0x13, + /// Validate a message-local selector value without consuming it. + CheckSelector = 0x15, /// Output pool string by id. OutLit = 0x20, /// Output literal slice by offset and length. @@ -68,6 +74,8 @@ pub enum Opcode { OutArg = 0x24, /// Load one selector argument directly by string-pool id. SelectArg = 0x25, + /// Select using a previously validated message-local value. + SelectLocal = 0x26, /// Begin select dispatch. SelectBegin = 0x30, /// Case compare against string-pool id. @@ -98,6 +106,7 @@ impl Opcode { Self::JmpIfFalse => 5, Self::PushConst => 5, Self::LoadArg => 5, + Self::CheckSelector | Self::StoreLocal | Self::LoadLocal | Self::SelectLocal => 5, Self::OutLit => 5, Self::OutSlice => 9, Self::OutVal => 1, @@ -127,12 +136,16 @@ impl TryFrom for Opcode { 0x02 => Ok(Self::JmpIfFalse), 0x10 => Ok(Self::PushConst), 0x11 => Ok(Self::LoadArg), + 0x15 => Ok(Self::CheckSelector), + 0x12 => Ok(Self::StoreLocal), + 0x13 => Ok(Self::LoadLocal), 0x20 => Ok(Self::OutLit), 0x21 => Ok(Self::OutSlice), 0x22 => Ok(Self::OutVal), 0x23 => Ok(Self::OutExpr), 0x24 => Ok(Self::OutArg), 0x25 => Ok(Self::SelectArg), + 0x26 => Ok(Self::SelectLocal), 0x30 => Ok(Self::SelectBegin), 0x31 => Ok(Self::CaseStr), 0x32 => Ok(Self::CaseDefault), @@ -354,6 +367,27 @@ impl TestOps { self } + /// Store the top stack value in a message-local slot. + pub fn store_local(mut self, slot: u32) -> Self { + self.code.push(Opcode::StoreLocal as u8); + self.code.extend_from_slice(&slot.to_le_bytes()); + self + } + + /// Load a message-local slot onto the stack. + pub fn load_local(mut self, slot: u32) -> Self { + self.code.push(Opcode::LoadLocal as u8); + self.code.extend_from_slice(&slot.to_le_bytes()); + self + } + + /// Validate a message-local selector value without consuming it. + pub fn check_selector(mut self, slot: u32) -> Self { + self.code.push(Opcode::CheckSelector as u8); + self.code.extend_from_slice(&slot.to_le_bytes()); + self + } + pub fn out_lit(mut self, str_id: u32) -> Self { self.code.push(Opcode::OutLit as u8); self.code.extend_from_slice(&str_id.to_le_bytes()); @@ -372,6 +406,13 @@ impl TestOps { self } + /// Select using a previously validated message-local value. + pub fn select_local(mut self, slot: u32) -> Self { + self.code.push(Opcode::SelectLocal as u8); + self.code.extend_from_slice(&slot.to_le_bytes()); + self + } + pub fn expr_fallback(mut self, str_id: u32) -> Self { self.code.push(Opcode::ExprFallback as u8); self.code.extend_from_slice(&str_id.to_le_bytes()); diff --git a/crates/message-format/src/runtime/value.rs b/crates/message-format/src/runtime/value.rs index cc40e28..8e38aba 100644 --- a/crates/message-format/src/runtime/value.rs +++ b/crates/message-format/src/runtime/value.rs @@ -3,11 +3,18 @@ //! Runtime value and argument model. -use alloc::{collections::BTreeMap, string::String, vec::Vec}; +#[cfg(feature = "icu4x")] +use alloc::string::ToString; +use alloc::{boxed::Box, collections::BTreeMap, string::String, vec::Vec}; use core::{error::Error, fmt}; use crate::runtime::Catalog; +#[cfg(feature = "icu4x")] +use fixed_decimal::Decimal; +#[cfg(feature = "icu4x")] +use icu_plurals::PluralCategory; + /// String-pool identifier. pub type StrId = u32; @@ -26,8 +33,17 @@ pub enum Value { Float(f64), /// Owned UTF-8 string. Str(String), + /// String resolved by the `string` function, retaining direction metadata. + #[cfg(feature = "icu4x")] + String(ResolvedString), /// Reference to a catalog string-pool entry. StrRef(StrId), + /// Fallback expression text from the catalog string pool. + /// + /// This preserves the recoverable-error state of a failed declaration so + /// the value can render its fallback while remaining ineligible for + /// selector matching. + Fallback(StrId), /// Reference to a literal slice in the catalog literal blob. LitRef { /// Offset into literal blob bytes. @@ -35,6 +51,193 @@ pub enum Value { /// Length in bytes. len: u32, }, + /// A number resolved by a built-in numeric function. + #[cfg(feature = "icu4x")] + Number(Box), + /// A value resolved by the test-only `test:select` function. + /// + /// The private payload preserves the function's selected precision across + /// aliases and reannotations. Raw strings continue to follow ordinary + /// string/numeric conversion rules. + ResolvedSelect(Box), +} + +/// String payload resolved by the `string` function. +#[cfg(feature = "icu4x")] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedString { + /// Raw resolved text without bidi isolation controls. + pub(crate) text: Box, + /// Direction requested by the string function. + pub(crate) direction: StringDirection, +} + +/// Direction metadata retained on a resolved string. +#[cfg(feature = "icu4x")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StringDirection { + /// Automatic direction selection. + Auto, + /// Left-to-right isolation. + Ltr, + /// Right-to-left isolation. + Rtl, +} + +#[cfg(feature = "icu4x")] +impl ResolvedString { + /// Return the raw resolved text without direction isolation controls. + #[must_use] + pub fn text(&self) -> &str { + &self.text + } +} + +/// Value produced by the test-only `test:select` function. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedSelect { + text: String, +} + +impl ResolvedSelect { + #[cfg(feature = "icu4x")] + pub(crate) fn new(text: String) -> Self { + Self { text } + } + + /// Return the resolved selector text without applying another function. + #[must_use] + pub fn text(&self) -> &str { + &self.text + } +} + +/// A numeric value resolved by a built-in numeric function. +// +// The fields stay private so hosts cannot accidentally manufacture a value +// whose options and numeric payload disagree. The runtime uses this value to +// carry an exact integer or decimal through subsequent annotations. +#[cfg(feature = "icu4x")] +#[derive(Debug, Clone, PartialEq)] +pub struct ResolvedNumber { + pub(crate) value: NumberValue, + pub(crate) format: NumberFormatOptions, + pub(crate) selection: NumberSelection, + /// Whether a `select` option was explicitly resolved for this value. + pub(crate) has_explicit_select: bool, + /// Category computed when this value was resolved by the locale-aware + /// built-in host. Stored values can therefore be matched without + /// re-running their function call. + pub(crate) selection_category: Option, +} + +/// Exact numeric payload retained by [`ResolvedNumber`]. +#[cfg(feature = "icu4x")] +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum NumberValue { + /// Exact signed integer payload. + Integer(i64), + /// Exact finite decimal payload. + Decimal(Decimal), + /// A floating-point non-finite value retained for compatibility with the + /// existing runtime rendering behavior. + NonFinite(f64), +} + +/// Parsed options needed to render a resolved number. Keeping these values +/// parsed makes default interpolation infallible after a function call has +/// validated the merged option set. +#[cfg(feature = "icu4x")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct NumberFormatOptions { + pub(crate) minimum_fraction_digits: Option, + pub(crate) maximum_fraction_digits: Option, + pub(crate) minimum_integer_digits: Option, + pub(crate) sign_display: NumberSignDisplay, + pub(crate) notation_scientific: bool, + pub(crate) grouping: NumberGrouping, +} + +#[cfg(feature = "icu4x")] +impl NumberFormatOptions { + pub(crate) const DEFAULT: Self = Self { + minimum_fraction_digits: None, + maximum_fraction_digits: None, + minimum_integer_digits: None, + sign_display: NumberSignDisplay::Auto, + notation_scientific: false, + grouping: NumberGrouping::Auto, + }; +} + +#[cfg(feature = "icu4x")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NumberSignDisplay { + Auto, + Always, + Never, +} + +#[cfg(feature = "icu4x")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NumberGrouping { + Auto, + Always, + Never, + Min2, +} + +/// Selection provenance retained with a resolved number. +#[cfg(feature = "icu4x")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum NumberSelection { + /// No selection annotation was involved. + None, + /// The value came from a plural selection annotation. + Plural, + /// The value came from an ordinal selection annotation. + Ordinal, + /// The value came from an exact selection annotation. + Exact, + /// A variable selection annotation reported `BadOption` and cannot be + /// used to match a later selection. + Invalid, +} + +#[cfg(feature = "icu4x")] +impl ResolvedNumber { + pub(crate) fn new( + value: NumberValue, + format: NumberFormatOptions, + selection: NumberSelection, + has_explicit_select: bool, + ) -> Self { + Self { + value, + format, + selection, + has_explicit_select, + selection_category: None, + } + } + + pub(crate) fn set_selection_category(&mut self, category: Option) { + self.selection_category = category; + } + + /// Return the exact finite numeric value as an ASCII decimal string. + /// + /// The returned text contains no locale formatting, grouping, or + /// annotation options. Hosts can use it when handling a [`Value::Number`] + /// operand without depending on the runtime's private numeric payload. + #[must_use] + pub fn text(&self) -> String { + match &self.value { + NumberValue::Integer(value) => value.to_string(), + NumberValue::Decimal(value) => value.to_string(), + NumberValue::NonFinite(value) => value.to_string(), + } + } } /// Error returned when building [`MessageArgs`] with a name that is not interned diff --git a/crates/message-format/src/runtime/vm.rs b/crates/message-format/src/runtime/vm.rs index d5b5283..2dc83df 100644 --- a/crates/message-format/src/runtime/vm.rs +++ b/crates/message-format/src/runtime/vm.rs @@ -12,12 +12,16 @@ use alloc::{ }; use core::str; +#[cfg(feature = "icu4x")] +use crate::runtime::value::{NumberSelection, ResolvedNumber}; use crate::runtime::{ catalog::{Catalog, read_i32}, - error::{FormatError, HostCallError, Trap}, + error::{FormatError, HostCallError, MessageFunctionError, Trap}, schema::decode_opcode_and_next_pc, value::{Args, StrId, Value}, }; +#[cfg(feature = "icu4x")] +use icu_plurals::PluralCategory; pub use crate::runtime::schema::{Decoded, FlowKind, Opcode, decode}; @@ -53,6 +57,9 @@ pub trait Host { fn index(&mut self, catalog: &Catalog) -> Result; /// Call function id with positional args and `(key, value)` options. + /// + /// Recoverable function diagnostics are reported through `on_error`; a + /// host may still return a value when formatting can continue. fn call( &mut self, catalog: &Catalog, @@ -60,12 +67,17 @@ pub trait Host { fn_id: u16, args: &[Value], opts: &[(u32, Value)], + on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result; /// Call function id for selection (e.g. plural/ordinal category). /// /// The default delegates to [`call`](Host::call). Hosts may override this /// to return `Value::StrRef` for known categories, avoiding allocation. + /// Recoverable diagnostics are reported through `on_error`. + /// Returning [`Value::Null`] disables matching and makes the VM select the + /// default arm; this is useful when a recoverable option error leaves a + /// value suitable for formatting but not for selection. fn call_select( &mut self, catalog: &Catalog, @@ -73,8 +85,9 @@ pub trait Host { fn_id: u16, args: &[Value], opts: &[(u32, Value)], + on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { - self.call(catalog, index, fn_id, args, opts) + self.call(catalog, index, fn_id, args, opts, on_error) } /// Optionally format a value for default interpolation. @@ -105,8 +118,9 @@ impl Host for Box { fn_id: u16, args: &[Value], opts: &[(u32, Value)], + on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { - H::call(self, catalog, index, fn_id, args, opts) + H::call(self, catalog, index, fn_id, args, opts, on_error) } fn call_select( @@ -116,8 +130,9 @@ impl Host for Box { fn_id: u16, args: &[Value], opts: &[(u32, Value)], + on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { - H::call_select(self, catalog, index, fn_id, args, opts) + H::call_select(self, catalog, index, fn_id, args, opts, on_error) } fn format_default( @@ -148,6 +163,7 @@ impl Host for NoopHost { fn_id: u16, _args: &[Value], _opts: &[(u32, Value)], + _on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { Err(HostCallError::UnknownFunction { fn_id }) } @@ -178,6 +194,7 @@ where fn_id: u16, args: &[Value], opts: &[(u32, Value)], + _on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { (self.0)(fn_id, args, opts) } @@ -257,10 +274,20 @@ enum SelectorValue<'a> { view: ValueView<'a>, str_id: Option, }, + /// Message-local selector resolved lazily for each case comparison. + Local(usize), InvalidBorrowed, Owned(Value), } +#[derive(Clone, Copy, PartialEq, Eq)] +enum CaseMatch { + No, + Exact, + #[cfg(feature = "icu4x")] + Category, +} + enum ExprStatePendingErrors { /// Record the minimum amount of information required Minimal { @@ -289,6 +316,14 @@ impl<'a> SelectorValue<'a> { str_id: Some(*id), }) } + Value::Fallback(id) => { + catalog + .pool_string_opt(*id) + .map_or(Self::InvalidBorrowed, |text| Self::Borrowed { + view: ValueView::Fallback(text), + str_id: None, + }) + } Value::LitRef { off, len } => { catalog .literal_opt(*off, *len) @@ -305,31 +340,40 @@ impl<'a> SelectorValue<'a> { } } - fn from_stack(stack: &mut Vec) -> Result { - stack - .pop() - .map(Self::Owned) - .ok_or(FormatError::StackUnderflow) - } - - fn matches_case(&self, case_str_id: u32, catalog: &Catalog) -> Result { + fn case_match( + &self, + locals: &[Value], + case_str_id: u32, + catalog: &Catalog, + ) -> Result { + if let Self::Local(slot) = self { + let value = locals + .get(*slot) + .ok_or(FormatError::Trap(Trap::InvalidLocalSlot))?; + let case = catalog + .string(case_str_id) + .map_err(|_| FormatError::Trap(Trap::InvalidCaseStringId))?; + return Ok(value_case_match(value, case, catalog)); + } if self.fast_str_id().is_some_and(|id| id == case_str_id) { - return Ok(true); + return Ok(CaseMatch::Exact); } let case = catalog .string(case_str_id) .map_err(|_| FormatError::Trap(Trap::InvalidCaseStringId))?; Ok(match self { - Self::Borrowed { view, .. } => view.matches_case(case), - Self::InvalidBorrowed => false, - Self::Owned(value) => value_matches_case(value, case, catalog), + Self::Borrowed { view, .. } => view.case_match(case), + Self::Local(_) => unreachable!("local selectors are handled above"), + Self::InvalidBorrowed => CaseMatch::No, + Self::Owned(value) => value_case_match(value, case, catalog), }) } fn fast_str_id(&self) -> Option { match self { Self::Borrowed { str_id, .. } => *str_id, + Self::Local(_) => None, Self::InvalidBorrowed => None, Self::Owned(Value::StrRef(id)) => Some(*id), Self::Owned(_) => None, @@ -379,7 +423,8 @@ impl ExprState { stack: &mut Vec, catalog: &Catalog, ) -> Result<(), FormatError> { - push_expr_fallback(stack, catalog, self.fallback_id.take()) + let fallback_id = self.fallback_id.take(); + push_expr_fallback(stack, catalog, fallback_id) } fn take_pending_errors(&mut self) -> Option> { @@ -400,6 +445,36 @@ impl ExprState { ExprStatePendingErrors::All(errors) => errors.clear(), } } + + fn finish_declaration( + &mut self, + value: Value, + catalog: &Catalog, + diagnostics: &mut Option<&mut dyn DiagnosticsSink>, + ) -> Result { + let failed = self.should_skip_call() || matches!(value, Value::Fallback(_)); + if let Some(pending_errors) = self.take_pending_errors() { + for error in pending_errors { + record_diagnostic(diagnostics, error); + } + } + + let value = if failed { + if let Some(fallback_id) = self.fallback_id.take() { + catalog + .string(fallback_id) + .map_err(|_| FormatError::Trap(Trap::InvalidFallbackStringId))?; + Value::Fallback(fallback_id) + } else { + value + } + } else { + value + }; + + self.clear_fallback(); + Ok(value) + } } pub(crate) fn run_bytecode( @@ -410,6 +485,7 @@ pub(crate) fn run_bytecode( args: &dyn Args, fuel: Option, stack: &mut Vec, + locals: &mut Vec, sink: &mut S, mut diagnostics: Option<&mut dyn DiagnosticsSink>, call_args: &mut Vec, @@ -421,8 +497,10 @@ where let code = catalog.code(); let mut pc = entry_pc; stack.clear(); + locals.clear(); let mut selector: Option> = None; let mut expr_state = ExprState::new(&diagnostics); + let mut deferred_case = None; let mut remaining_fuel = fuel; loop { @@ -460,6 +538,33 @@ where let value = load_arg_value(args, catalog, id, &mut expr_state); stack.push(value); } + Opcode::StoreLocal => { + let slot = local_slot(code, base)?; + let value = stack.pop().ok_or(FormatError::StackUnderflow)?; + let value = expr_state.finish_declaration(value, catalog, &mut diagnostics)?; + if slot == locals.len() { + locals.push(value); + } else if let Some(existing) = locals.get_mut(slot) { + *existing = value; + } else { + return Err(FormatError::Trap(Trap::InvalidLocalSlot)); + } + } + Opcode::LoadLocal => { + let slot = local_slot(code, base)?; + let value = locals + .get(slot) + .cloned() + .ok_or(FormatError::Trap(Trap::InvalidLocalSlot))?; + stack.push(value); + } + Opcode::CheckSelector => { + let slot = local_slot(code, base)?; + let value = locals + .get(slot) + .ok_or(FormatError::Trap(Trap::InvalidLocalSlot))?; + check_selector_value(value, &mut diagnostics); + } Opcode::OutLit | Opcode::OutSlice | Opcode::OutExpr @@ -479,12 +584,14 @@ where } Opcode::SelectArg | Opcode::SelectBegin + | Opcode::SelectLocal | Opcode::CaseStr | Opcode::CaseDefault | Opcode::SelectEnd => { if let Some(jump_pc) = handle_select_instruction( args, stack, + locals, &mut selector, catalog, &mut diagnostics, @@ -493,6 +600,7 @@ where next_pc, base, code, + &mut deferred_case, )? { pc = jump_pc; continue; @@ -527,12 +635,29 @@ where Ok(()) } +fn local_slot(code: &[u8], base: usize) -> Result { + usize::try_from(read_u32(code, base + 1)?) + .map_err(|_| FormatError::Trap(Trap::InvalidLocalSlot)) +} + fn record_diagnostic(diagnostics: &mut Option<&mut dyn DiagnosticsSink>, error: FormatError) { if let Some(sink) = diagnostics.as_deref_mut() { sink.record(error); } } +fn check_selector_value(value: &Value, diagnostics: &mut Option<&mut dyn DiagnosticsSink>) { + #[cfg(feature = "icu4x")] + if matches!(value, Value::Number(number) if number.selection == NumberSelection::Invalid) { + // The declaration already reported the option error. The local + // selector contributes only its own selector diagnostic. + record_bad_selector(diagnostics, None); + } + if matches!(value, Value::Fallback(_)) { + record_bad_selector(diagnostics, None); + } +} + fn handle_output_instruction( sink: &mut S, host: &mut H, @@ -579,6 +704,7 @@ where fn handle_select_instruction<'a>( args: &'a dyn Args, stack: &mut Vec, + locals: &[Value], selector: &mut Option>, catalog: &'a Catalog, diagnostics: &mut Option<&mut dyn DiagnosticsSink>, @@ -587,15 +713,29 @@ fn handle_select_instruction<'a>( next_pc: u32, base: usize, code: &[u8], + deferred_case: &mut Option, ) -> Result, FormatError> { match opcode { Opcode::SelectArg => { + *deferred_case = None; let key_id = read_u32(code, base + 1)?; *selector = Some(load_selector_value(args, catalog, key_id, diagnostics)); Ok(None) } + Opcode::SelectLocal => { + *deferred_case = None; + let slot = local_slot(code, base)?; + if locals.get(slot).is_none() { + return Err(FormatError::Trap(Trap::InvalidLocalSlot)); + } + *selector = Some(SelectorValue::Local(slot)); + Ok(None) + } Opcode::SelectBegin => { - *selector = Some(SelectorValue::from_stack(stack)?); + *deferred_case = None; + let value = stack.pop().ok_or(FormatError::StackUnderflow)?; + check_selector_value(&value, diagnostics); + *selector = Some(SelectorValue::Owned(value)); Ok(None) } Opcode::CaseStr => { @@ -603,18 +743,31 @@ fn handle_select_instruction<'a>( .as_ref() .ok_or(FormatError::Trap(Trap::CaseStringWithoutSelector))?; let case_str_id = read_u32(code, base + 1)?; - if selector.matches_case(case_str_id, catalog)? { - let rel = read_i32(code, base + 5)?; - apply_rel_jump(pc, next_pc, rel).map(Some) - } else { - Ok(None) + match selector.case_match(locals, case_str_id, catalog)? { + CaseMatch::Exact => { + *deferred_case = None; + let rel = read_i32(code, base + 5)?; + apply_rel_jump(pc, next_pc, rel).map(Some) + } + #[cfg(feature = "icu4x")] + CaseMatch::Category => { + let rel = read_i32(code, base + 5)?; + *deferred_case = Some(apply_rel_jump(pc, next_pc, rel)?); + Ok(None) + } + CaseMatch::No => Ok(None), } } Opcode::CaseDefault => { let rel = read_i32(code, base + 1)?; - apply_rel_jump(pc, next_pc, rel).map(Some) + if let Some(jump_pc) = deferred_case.take() { + Ok(Some(jump_pc)) + } else { + apply_rel_jump(pc, next_pc, rel).map(Some) + } } Opcode::SelectEnd => { + *deferred_case = None; *selector = None; Ok(None) } @@ -792,7 +945,14 @@ fn handle_call_instruction( // If a missing variable was loaded as an operand for this function call, // skip the call and use the expression fallback (e.g. `{$varname}`) per // TR35 §16. - if expr_state.should_skip_call() { + // A fallback can be propagated from a previously resolved local without + // leaving a pending error in this expression. Such an operand still + // short-circuits function resolution per MF2 formatting §16.1. + if expr_state.should_skip_call() + || call_args + .first() + .is_some_and(|value| matches!(value, Value::Fallback(_))) + { if let Some(pending_errors) = expr_state.take_pending_errors() { let mut pending_errors = pending_errors.into_iter(); if opcode == Opcode::CallSelect { @@ -811,16 +971,39 @@ fn handle_call_instruction( return Ok(()); } + let mut on_error = |error| record_diagnostic(diagnostics, FormatError::Function(error)); let call_result = if opcode == Opcode::CallSelect { - host.call_select(catalog, index, fn_id, call_args, call_options) + host.call_select( + catalog, + index, + fn_id, + call_args, + call_options, + &mut on_error, + ) } else { - host.call(catalog, index, fn_id, call_args, call_options) + host.call( + catalog, + index, + fn_id, + call_args, + call_options, + &mut on_error, + ) }; match call_result { Ok(result) => { expr_state.clear_fallback(); expr_state.clear_pending_errors(); + if opcode == Opcode::CallSelect && matches!(result, Value::Null) { + // `Null` is the explicit host contract for an unselectable + // result. The callback has already reported the function + // diagnostic; this records the selector failure so the VM + // takes the default arm without inventing a second option + // error. + record_bad_selector(diagnostics, None); + } stack.push(result); Ok(()) } @@ -855,7 +1038,7 @@ fn push_expr_fallback( catalog .string(fb_id) .map_err(|_| FormatError::Trap(Trap::InvalidFallbackStringId))?; - stack.push(Value::StrRef(fb_id)); + stack.push(Value::Fallback(fb_id)); } else { stack.push(Value::Null); } @@ -924,6 +1107,12 @@ enum ValueView<'a> { Int(i64), Float(f64), Text(&'a str), + #[cfg(feature = "icu4x")] + ExactText(&'a str), + Fallback(&'a str), + ResolvedSelect(&'a str), + #[cfg(feature = "icu4x")] + Number(&'a ResolvedNumber), } impl<'a> ValueView<'a> { @@ -934,8 +1123,14 @@ impl<'a> ValueView<'a> { Value::Int(v) => Some(Self::Int(*v)), Value::Float(v) => Some(Self::Float(*v)), Value::Str(v) => Some(Self::Text(v)), + #[cfg(feature = "icu4x")] + Value::String(v) => Some(Self::ExactText(v.text())), Value::StrRef(id) => catalog.pool_string_opt(*id).map(Self::Text), + Value::Fallback(id) => catalog.pool_string_opt(*id).map(Self::Fallback), Value::LitRef { off, len } => catalog.literal_opt(*off, *len).map(Self::Text), + Value::ResolvedSelect(value) => Some(Self::ResolvedSelect(value.text())), + #[cfg(feature = "icu4x")] + Value::Number(value) => Some(Self::Number(value)), } } @@ -949,6 +1144,15 @@ impl<'a> ValueView<'a> { } Self::Float(v) => sink.expression(&v.to_string()), Self::Text(v) => sink.expression(v), + #[cfg(feature = "icu4x")] + Self::ExactText(v) => sink.expression(v), + Self::Fallback(v) => sink.expression(v), + Self::ResolvedSelect(v) => sink.expression(v), + #[cfg(feature = "icu4x")] + Self::Number(v) => { + let rendered = v.text(); + sink.expression(&rendered); + } } } @@ -959,16 +1163,97 @@ impl<'a> ValueView<'a> { Self::Int(v) => Cow::Owned(v.to_string()), Self::Float(v) => Cow::Owned(v.to_string()), Self::Text(v) => Cow::Borrowed(v), + #[cfg(feature = "icu4x")] + Self::ExactText(v) => Cow::Borrowed(v), + Self::Fallback(v) => Cow::Borrowed(v), + Self::ResolvedSelect(v) => Cow::Borrowed(v), + #[cfg(feature = "icu4x")] + Self::Number(v) => Cow::Owned(v.text()), } } + #[cfg(test)] fn matches_case(self, case: &str) -> bool { + self.case_match(case) != CaseMatch::No + } + + fn case_match(self, case: &str) -> CaseMatch { match self { - Self::Null => false, - Self::Bool(v) => (if v { "true" } else { "false" }) == case, - Self::Int(v) => int_matches_case(v, case), - Self::Float(v) => string_value_matches_case(&v.to_string(), case), - Self::Text(v) => string_value_matches_case(v, case), + Self::Null => CaseMatch::No, + Self::Bool(v) => { + if (if v { "true" } else { "false" }) == case { + CaseMatch::Exact + } else { + CaseMatch::No + } + } + Self::Int(v) => { + if int_matches_case(v, case) { + CaseMatch::Exact + } else { + CaseMatch::No + } + } + Self::Float(v) => { + if string_value_matches_case(&v.to_string(), case) { + CaseMatch::Exact + } else { + CaseMatch::No + } + } + Self::Text(v) => { + if string_value_matches_case(v, case) { + CaseMatch::Exact + } else { + CaseMatch::No + } + } + #[cfg(feature = "icu4x")] + Self::ExactText(v) => { + if v == case { + CaseMatch::Exact + } else { + CaseMatch::No + } + } + Self::Fallback(_) => CaseMatch::No, + Self::ResolvedSelect(v) => { + if string_value_matches_case(v, case) { + CaseMatch::Exact + } else { + CaseMatch::No + } + } + #[cfg(feature = "icu4x")] + Self::Number(v) => match v.selection { + NumberSelection::Invalid => CaseMatch::No, + NumberSelection::Plural | NumberSelection::Ordinal => { + if string_value_matches_case(&v.text(), case) { + CaseMatch::Exact + } else if v.selection_category.is_some_and(|category| { + matches!( + (category, case), + (PluralCategory::Zero, "zero") + | (PluralCategory::One, "one") + | (PluralCategory::Two, "two") + | (PluralCategory::Few, "few") + | (PluralCategory::Many, "many") + | (PluralCategory::Other, "other") + ) + }) { + CaseMatch::Category + } else { + CaseMatch::No + } + } + NumberSelection::Exact | NumberSelection::None => { + if string_value_matches_case(&v.text(), case) { + CaseMatch::Exact + } else { + CaseMatch::No + } + } + }, } } @@ -979,6 +1264,12 @@ impl<'a> ValueView<'a> { Self::Int(v) => v == 0, Self::Float(v) => v == 0.0, Self::Text(v) => v.is_empty(), + #[cfg(feature = "icu4x")] + Self::ExactText(v) => v.is_empty(), + Self::Fallback(v) => v.is_empty(), + Self::ResolvedSelect(v) => v == "0", + #[cfg(feature = "icu4x")] + Self::Number(v) => v.text() == "0", } } } @@ -1053,8 +1344,8 @@ fn format_value_display<'a>(value: &'a Value, catalog: &'a Catalog) -> Cow<'a, s ValueView::from_value(value, catalog).map_or(Cow::Borrowed(""), ValueView::format_display) } -fn value_matches_case(value: &Value, case: &str, catalog: &Catalog) -> bool { - ValueView::from_value(value, catalog).is_some_and(|view| view.matches_case(case)) +fn value_case_match(value: &Value, case: &str, catalog: &Catalog) -> CaseMatch { + ValueView::from_value(value, catalog).map_or(CaseMatch::No, |view| view.case_match(case)) } fn string_value_matches_case(value: &str, case: &str) -> bool { @@ -1249,8 +1540,10 @@ mod tests { use crate::runtime::catalog::{ FuncEntry, MessageEntry, build_catalog, build_catalog_with_funcs, }; - use crate::runtime::error::{ImplementationFailure, MessageFunctionError}; + use crate::runtime::error::{CatalogError, ImplementationFailure, MessageFunctionError}; use crate::runtime::schema::TestOps; + #[cfg(feature = "icu4x")] + use crate::runtime::value::{NumberFormatOptions, NumberSelection, NumberValue}; fn catalog_for_test(strings: &[&str], literals: &str, code: &[u8]) -> Catalog { let bytes = if let Some(func_count) = max_function_id(code).map(|id| usize::from(id) + 1) { @@ -1377,6 +1670,97 @@ mod tests { assert_eq!(out, "{$name}"); } + #[test] + fn local_store_load_round_trip_and_replace() { + let code = TestOps::new() + .push_const(1) + .store_local(0) + .push_const(2) + .store_local(0) + .load_local(0) + .out_val() + .halt() + .build(); + let catalog = catalog_for_test(&["main", "first", "second"], "", &code); + let mut formatter = formatter_noop(&catalog); + let out = formatter + .format_by_id_for_test("main", &[]) + .expect("formatted"); + assert_eq!(out, "second"); + } + + #[test] + fn local_slots_are_fresh_for_each_invocation() { + let first_code = TestOps::new().push_const(1).store_local(0).halt().build(); + let second_entry = u32::try_from(first_code.len()).expect("code length fits u32"); + let mut code = first_code; + code.extend_from_slice(&TestOps::new().load_local(0).out_val().halt().build()); + let catalog_bytes = build_catalog( + &["first", "second", "stored"], + "", + &[ + MessageEntry { + name_str_id: 0, + entry_pc: 0, + }, + MessageEntry { + name_str_id: 1, + entry_pc: second_entry, + }, + ], + &code, + ); + assert_eq!( + Catalog::from_bytes(&catalog_bytes).expect_err("uninitialized local must reject"), + CatalogError::InvalidLocalSlot { + pc: second_entry, + slot: 0, + } + ); + } + + #[test] + fn invalid_local_slots_are_rejected_without_resizing() { + for slot in [0, u32::MAX] { + let code = TestOps::new().load_local(slot).out_val().halt().build(); + let bytes = build_catalog( + &["main"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + assert_eq!( + Catalog::from_bytes(&bytes).expect_err("invalid local must reject"), + CatalogError::InvalidLocalSlot { pc: 0, slot } + ); + } + + let code = TestOps::new() + .push_const(0) + .store_local(u32::MAX) + .halt() + .build(); + let bytes = build_catalog( + &["main", "value"], + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + assert_eq!( + Catalog::from_bytes(&bytes).expect_err("huge store must reject"), + CatalogError::InvalidLocalSlot { + pc: 5, + slot: u32::MAX, + } + ); + } + #[test] fn missing_arg_direct_interpolation_records_diagnostic() { let code = TestOps::new().out_arg(1).halt().build(); @@ -1390,6 +1774,61 @@ mod tests { assert_eq!(errors, vec![FormatError::MissingArg("name".to_string())]); } + #[test] + fn local_binding_finalizes_missing_arg_fallback_and_diagnostic() { + let code = TestOps::new() + .load_arg(1) + .expr_fallback(2) + .store_local(0) + .load_local(0) + .out_val() + .halt() + .build(); + let catalog = catalog_for_test(&["main", "missing", "{$missing}"], "", &code); + let mut formatter = formatter_noop(&catalog); + let mut sink = String::new(); + let errors = formatter + .format_to_for_test_by_id("main", &[], &mut sink) + .expect("formatted"); + assert_eq!(sink, "{$missing}"); + assert_eq!(errors, vec![FormatError::MissingArg("missing".to_string())]); + } + + #[test] + fn fallback_local_is_not_a_selector_and_reports_bad_selector() { + let code = TestOps::new() + .load_arg(1) + .expr_fallback(2) + .store_local(0) + .load_local(0) + .select_begin() + .case_str(3, "hit") + .case_default("other") + .label("hit") + .out_slice(0, 1) + .jmp("end") + .label("other") + .out_slice(1, 5) + .label("end") + .select_end() + .halt() + .build(); + let catalog = catalog_for_test(&["main", "missing", "{$missing}", "hit"], "Xother", &code); + let mut formatter = formatter_noop(&catalog); + let mut sink = String::new(); + let errors = formatter + .format_to_for_test_by_id("main", &[], &mut sink) + .expect("formatted"); + assert_eq!(sink, "other"); + assert_eq!( + errors, + vec![ + FormatError::MissingArg("missing".to_string()), + FormatError::BadSelector { source: None }, + ] + ); + } + #[test] fn missing_arg_in_function_call_records_diagnostic_and_uses_expr_fallback() { let code = TestOps::new() @@ -1409,6 +1848,83 @@ mod tests { assert_eq!(errors, vec![FormatError::MissingArg("name".to_string())]); } + #[test] + fn propagated_local_fallback_skips_reannotation() { + let code = TestOps::new() + .load_arg(1) + .expr_fallback(2) + .store_local(0) + .load_local(0) + .expr_fallback(3) + .call_func(0, 1, 0) + .out_val() + .halt() + .build(); + let catalog = catalog_for_test(&["main", "missing", "{$missing}", "{$a}"], "", &code); + let mut formatter = Formatter::new( + &catalog, + HostFn(|_, _, _| panic!("fallback operand must not call the host")), + ) + .expect("host"); + let mut sink = String::new(); + let errors = formatter + .format_to_for_test_by_id("main", &[], &mut sink) + .expect("formatted"); + assert_eq!(sink, "{$a}"); + assert_eq!(errors, vec![FormatError::MissingArg("missing".to_string())]); + } + + #[test] + fn propagated_fallback_skips_nested_call_after_option_operands() { + let code = TestOps::new() + .load_arg(1) + .expr_fallback(2) + .store_local(0) + .load_local(0) + .push_const(4) + .push_const(5) + .expr_fallback(3) + .call_func(0, 1, 1) + .out_val() + .halt() + .build(); + let catalog = catalog_for_test( + &[ + "main", + "missing", + "{$missing}", + "{$outer}", + "option", + "value", + ], + "", + &code, + ); + let mut formatter = Formatter::new( + &catalog, + HostFn(|_, _, _| panic!("propagated fallback must not call the host")), + ) + .expect("host"); + let mut sink = String::new(); + let errors = formatter + .format_to_for_test_by_id("main", &[], &mut sink) + .expect("formatted"); + assert_eq!(sink, "{$outer}"); + assert_eq!(errors, vec![FormatError::MissingArg("missing".to_string())]); + + let mut formatter = Formatter::new( + &catalog, + HostFn(|_, _, _| panic!("propagated fallback must not call the host")), + ) + .expect("host"); + assert_eq!( + formatter + .format_by_id_for_test("main", &[]) + .expect("formatted"), + "{$outer}" + ); + } + #[test] fn expr_fallback_uses_strref_when_catalog_string_exists() { let catalog = catalog_for_test(&["main", "{$name}"], "", &[Opcode::Halt as u8]); @@ -1416,7 +1932,7 @@ mod tests { push_expr_fallback(&mut stack, &catalog, Some(1)).expect("fallback"); - assert_eq!(stack, vec![Value::StrRef(1)]); + assert_eq!(stack, vec![Value::Fallback(1)]); } #[test] @@ -1449,6 +1965,77 @@ mod tests { ); } + #[test] + fn checked_local_selector_reports_fallback_once() { + let code = TestOps::new() + .load_arg(1) + .expr_fallback(3) + .store_local(0) + .check_selector(0) + .select_local(0) + .case_str(2, "hit") + .case_default("fallback") + .label("hit") + .out_slice(0, 1) + .jmp("end") + .label("fallback") + .out_slice(1, 1) + .label("end") + .select_end() + .halt() + .build(); + let catalog = catalog_for_test(&["main", "missing", "hit", "{$missing}"], "HF", &code); + let mut formatter = formatter_noop(&catalog); + let mut sink = String::new(); + let errors = formatter + .format_to_for_test_by_id("main", &[], &mut sink) + .expect("formatted"); + assert_eq!(sink, "F"); + assert_eq!( + errors, + vec![ + FormatError::MissingArg("missing".to_string()), + FormatError::BadSelector { source: None }, + ] + ); + } + + #[cfg(feature = "icu4x")] + #[test] + fn invalid_stored_number_selection_does_not_match_and_reports_once() { + let code = TestOps::new() + .load_arg(1) + .store_local(0) + .load_local(0) + .select_begin() + .case_str(2, "hit") + .case_default("other") + .label("hit") + .out_slice(0, 1) + .jmp("end") + .label("other") + .out_slice(1, 5) + .label("end") + .select_end() + .halt() + .build(); + let catalog = catalog_for_test(&["main", "value", "1", "other"], "Xother", &code); + let mut formatter = formatter_noop(&catalog); + let value = Value::Number(Box::new(ResolvedNumber::new( + NumberValue::Integer(1), + NumberFormatOptions::DEFAULT, + NumberSelection::Invalid, + true, + ))); + let args = vec![(arg_id(&catalog, "value"), value)]; + let mut sink = String::new(); + let errors = formatter + .format_to_for_test_by_id("main", &args, &mut sink) + .expect("formatted"); + assert_eq!(sink, "other"); + assert_eq!(errors, vec![FormatError::BadSelector { source: None }]); + } + #[test] fn selector_call_error_records_bad_selector_and_uses_default_arm() { #[derive(Default)] @@ -1466,6 +2053,7 @@ mod tests { _fn_id: u16, _args: &[Value], _opts: &[(u32, Value)], + _on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { panic!("call_select opcode must not dispatch to call()") } @@ -1477,6 +2065,7 @@ mod tests { _fn_id: u16, _args: &[Value], _opts: &[(u32, Value)], + _on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { Err(HostCallError::Function(MessageFunctionError::BadOperand)) } @@ -1774,6 +2363,7 @@ mod tests { _fn_id: u16, _args: &[Value], _opts: &[(u32, Value)], + _on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { panic!("call_select opcode must not dispatch to call()") } @@ -1785,6 +2375,7 @@ mod tests { _fn_id: u16, _args: &[Value], _opts: &[(u32, Value)], + _on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { // Return StrRef pointing to "yes" (str_id=2) Ok(Value::StrRef(2)) @@ -1844,6 +2435,7 @@ mod tests { _fn_id: u16, _args: &[Value], _opts: &[(u32, Value)], + _on_error: &mut dyn FnMut(MessageFunctionError), ) -> Result { unreachable!("plain interpolation should only use format_default") } diff --git a/docs/runtime-vm-semantics.md b/docs/runtime-vm-semantics.md index 7a14aed..bc80384 100644 --- a/docs/runtime-vm-semantics.md +++ b/docs/runtime-vm-semantics.md @@ -34,6 +34,31 @@ Pipeline: - Unknown host functions are reported as `FormatError::UnknownFunction` unless host overrides behavior. - Missing arguments are reported as `FormatError::MissingArg` under default semantics. +Local slots are private to one message execution. `StoreLocal` initializes the +next dense slot or replaces an initialized slot; `LoadLocal` requires prior +initialization on every reachable path. The verifier checks each entry point +independently and intersects initialization state at control-flow joins. +Formatters reuse the local buffer's capacity, but clear its values before +returning from execution on both success and error. + +Declarations that need runtime evaluation are resolved eagerly in source order, +once per message execution. An unused declaration can therefore report an +error: `.local $x = {$missing :number} {{hello}}` renders `hello` and reports +`MissingArg`. Constant literal bindings can still be substituted at compilation. + +The compiler emits `CheckSelector` once for each source selector, before variant +dispatch. Each check reports `BadSelector` if the local is a fallback or an +unselectable resolved number. `SelectLocal` then compares that local directly, +without cloning it or repeating the diagnostic when generated branches retry +exact and plural candidates. Both instructions require a definitely initialized +slot. Repeated default subtrees share a target; jumps to an ancestor default +close each intervening select scope with `SelectEnd`. + +Failed expressions carry `Value::Fallback` through nested calls and declarations. +Failure state is attached to the value rather than to the following instruction. +Resolved strings retain raw text and direction metadata separately; bidi +isolation is applied at output, so later annotations consume the raw text. + ## Host Contract `Host::call` receives: diff --git a/wind_tunnel/benches/formatting.rs b/wind_tunnel/benches/formatting.rs index 5ed8b37..a6819d0 100644 --- a/wind_tunnel/benches/formatting.rs +++ b/wind_tunnel/benches/formatting.rs @@ -316,6 +316,31 @@ fn build_many_segments_catalog() -> Catalog { Catalog::from_bytes(&bytes).expect("valid benchmark catalog") } +fn build_repeated_local_catalog() -> Catalog { + let strings = ["main", "value"]; + let mut code = Vec::new(); + for slot in 0_u32..16 { + code.push(vm::Opcode::PushConst as u8); + code.extend_from_slice(&1_u32.to_le_bytes()); + code.push(vm::Opcode::StoreLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + code.push(vm::Opcode::LoadLocal as u8); + code.extend_from_slice(&slot.to_le_bytes()); + code.push(vm::Opcode::OutVal as u8); + } + code.push(vm::Opcode::Halt as u8); + let bytes = build_catalog( + &strings, + "", + &[MessageEntry { + name_str_id: 0, + entry_pc: 0, + }], + &code, + ); + Catalog::from_bytes(&bytes).expect("valid repeated-local benchmark catalog") +} + fn build_markup_catalog() -> Catalog { // MARKUP_OPEN "b"(1) optc=0, OUT_SLICE "Hello "(0,6), LOAD_ARG "name"(2), OUT_VAL, // MARKUP_CLOSE "b"(1) optc=0, HALT @@ -605,6 +630,7 @@ fn bench_formatting(c: &mut Criterion) { let call_catalog_no_opts = build_call_catalog_no_opts(); let call_catalog_with_opts = build_call_catalog_with_opts(); let many_segments_catalog = build_many_segments_catalog(); + let repeated_local_catalog = build_repeated_local_catalog(); let markup_catalog = build_markup_catalog(); let markup_option_literal_catalog = build_markup_option_literal_catalog(); let markup_option_variable_catalog = build_markup_option_variable_catalog(); @@ -676,6 +702,17 @@ fn bench_formatting(c: &mut Criterion) { }); }); + group.bench_function("repeated_local_storage", |b| { + let mut formatter = Formatter::new(&repeated_local_catalog, NoopHost).expect("formatter"); + let empty_args: Vec<(u32, Value)> = Vec::new(); + b.iter(|| { + let out = formatter + .format_by_id_for_bench("main", black_box(&empty_args)) + .expect("format"); + black_box(out); + }); + }); + group.finish(); let mut call_group = c.benchmark_group("runtime_call_paths"); @@ -1291,5 +1328,89 @@ fn bench_plural_ordinal(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_formatting, bench_plural_ordinal); +fn bench_runtime_compiled_declarations(c: &mut Criterion) { + let mut group = c.benchmark_group("runtime_compiled_declarations"); + + let number_catalog = + compile_catalog(".input {$n :number} .local $x = {$n :number} {{a={$x} b={$x} c={$x}}}"); + let number_entries = [("n", Value::Int(21))]; + let number_args = message_args(&number_catalog, &number_entries); + let number_host = BuiltinHost::new(&locale("en-US")).expect("host"); + let mut number_formatter = Formatter::new(&number_catalog, number_host).expect("formatter"); + let number_message = number_formatter.resolve("main").expect("resolved message"); + assert_eq!( + number_formatter + .format_for_bench(number_message, &number_args) + .expect("format"), + "a=21 b=21 c=21", + "the benchmark must format all three uses of the numeric declaration" + ); + group.bench_function("number_declaration_reused_three_places", |b| { + b.iter(|| { + let out = number_formatter + .format_for_bench(number_message, black_box(&number_args)) + .expect("format"); + black_box(out); + }); + }); + + let chain_catalog = + compile_catalog(".input {$x :string} .local $n = {$x :number} {{value={$n}}}"); + let chain_entries = [("x", Value::Int(21))]; + let chain_args = message_args(&chain_catalog, &chain_entries); + let chain_host = BuiltinHost::new(&locale("en-US")).expect("host"); + let mut chain_formatter = Formatter::new(&chain_catalog, chain_host).expect("formatter"); + let chain_message = chain_formatter.resolve("main").expect("resolved message"); + assert_eq!( + chain_formatter + .format_for_bench(chain_message, &chain_args) + .expect("format"), + "value=21", + "the string declaration must supply raw text to the numeric annotation" + ); + group.bench_function("string_to_number_chain", |b| { + b.iter(|| { + let out = chain_formatter + .format_for_bench(chain_message, black_box(&chain_args)) + .expect("format"); + black_box(out); + }); + }); + + let selector_catalog = compile_catalog( + ".input {$a :number} .input {$b :number}\n.match $a $b\n1 1 {{exact}}\none one {{category}}\n* * {{fallback}}", + ); + let selector_entries = [("a", Value::Int(1)), ("b", Value::Int(21))]; + let selector_args = message_args(&selector_catalog, &selector_entries); + let selector_host = BuiltinHost::new(&locale("ru")).expect("host"); + let mut selector_formatter = + Formatter::new(&selector_catalog, selector_host).expect("formatter"); + let selector_message = selector_formatter + .resolve("main") + .expect("resolved message"); + assert_eq!( + selector_formatter + .format_for_bench(selector_message, &selector_args) + .expect("format"), + "category", + "rejecting the exact variant must retry the plural category" + ); + group.bench_function("numeric_exact_rejected_then_category", |b| { + b.iter(|| { + let out = selector_formatter + .format_for_bench(selector_message, black_box(&selector_args)) + .expect("format"); + black_box(out); + }); + }); + + group.finish(); +} + +criterion_group!( + benches, + bench_formatting, + bench_plural_ordinal, + bench_runtime_compiled_declarations +); criterion_main!(benches);