diff --git a/Cargo.toml b/Cargo.toml index 233268b4..5d0b11ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ default = [ "serde" ] serde = ["dep:serde", "dep:serde_json"] docs = [] external-jets = [] +fmt = [] [dependencies] base64 = "0.21.2" diff --git a/src/ast.rs b/src/ast.rs index 8d4a0c93..e3a084cb 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -343,10 +343,11 @@ impl PartialEq for CallName { } /// Definition of a custom function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct CustomFunction { params: Arc<[FunctionParam]>, body: Arc, + span: Span, } impl CustomFunction { @@ -360,6 +361,11 @@ impl CustomFunction { &self.body } + /// Access the span of the complete function declaration. + pub fn span(&self) -> &Span { + &self.span + } + /// Return a pattern for the parameters of the function. pub fn params_pattern(&self) -> Pattern { Pattern::tuple( @@ -372,11 +378,14 @@ impl CustomFunction { } } +impl_eq_hash!(CustomFunction; params, body); + /// Parameter of a function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct FunctionParam { identifier: Identifier, ty: ResolvedType, + span: Span, } impl FunctionParam { @@ -389,8 +398,15 @@ impl FunctionParam { pub fn ty(&self) -> &ResolvedType { &self.ty } + + /// Access the span of the complete parameter declaration. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(FunctionParam; identifier, ty); + /// Match expression. #[derive(Clone, Debug)] pub struct Match { @@ -461,6 +477,7 @@ pub struct EnumMatchArm { /// variants. pattern: Pattern, body: Arc, + span: Span, } impl EnumMatchArm { @@ -473,6 +490,11 @@ impl EnumMatchArm { pub fn body(&self) -> &Expression { &self.body } + + /// Access the span of the complete enum match arm. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumMatchArm; pattern, body); @@ -514,10 +536,11 @@ impl AsRef for EnumMatch { } /// Arm of a [`Match`] expression. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct MatchArm { pattern: MatchPattern, expression: Arc, + span: Span, } impl MatchArm { @@ -530,8 +553,15 @@ impl MatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the span of the complete match arm. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(MatchArm; pattern, expression); + #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] pub enum ExprTree<'a> { Expression(&'a Expression), @@ -1379,7 +1409,11 @@ impl AbstractSyntaxTree for Function { .map(|param| { let identifier = param.identifier().clone(); let ty = scope.resolve(param.ty())?; - Ok(FunctionParam { identifier, ty }) + Ok(FunctionParam { + identifier, + ty, + span: *param.span(), + }) }) .collect::, Error>>() .with_span(from)?; @@ -1398,7 +1432,11 @@ impl AbstractSyntaxTree for Function { scope.exit_block(); debug_assert!(scope.is_outside_function()); - let function = CustomFunction { params, body }; + let function = CustomFunction { + params, + body, + span: *from.span(), + }; scope .insert_function(from.name().clone(), from.visibility().clone(), function) .with_span(from)?; @@ -1945,10 +1983,11 @@ impl AbstractSyntaxTree for EnumMatch { .into_iter() .zip(info.variants()) .map(|(arm, variant)| { - let pattern = analyze_enum_arm_bindings(arm, variant, scope, span)?; + let arm_span = *arm.span(); + let pattern = analyze_enum_arm_bindings(arm, variant, scope, arm_span)?; scope.enter_block(); let payload_ty = variant.payload_type(); - let typed_variables = pattern.is_of_type(payload_ty).with_span(span)?; + let typed_variables = pattern.is_of_type(payload_ty).with_span(arm_span)?; for (identifier, variable_ty) in typed_variables { scope.insert_variable(identifier, variable_ty); } @@ -1957,6 +1996,7 @@ impl AbstractSyntaxTree for EnumMatch { Ok(EnumMatchArm { pattern, body: body?, + span: arm_span, }) }) .collect::, Diagnostic>>()?; @@ -2347,8 +2387,8 @@ impl AbstractSyntaxTree for Match { scope.enter_block(); if let Some((pat_l, ty_l)) = from.left().pattern().as_typed_pattern() { - let ty_l = scope.resolve(ty_l).with_span(from)?; - let typed_variables = pat_l.is_of_type(&ty_l).with_span(from)?; + let ty_l = scope.resolve(ty_l).with_span(from.left())?; + let typed_variables = pat_l.is_of_type(&ty_l).with_span(from.left())?; for (identifier, ty) in typed_variables { scope.insert_variable(identifier, ty); } @@ -2357,8 +2397,8 @@ impl AbstractSyntaxTree for Match { scope.exit_block(); scope.enter_block(); if let Some((pat_r, ty_r)) = from.right().pattern().as_typed_pattern() { - let ty_r = scope.resolve(ty_r).with_span(from)?; - let typed_variables = pat_r.is_of_type(&ty_r).with_span(from)?; + let ty_r = scope.resolve(ty_r).with_span(from.right())?; + let typed_variables = pat_r.is_of_type(&ty_r).with_span(from.right())?; for (identifier, ty) in typed_variables { scope.insert_variable(identifier, ty); } @@ -2371,10 +2411,12 @@ impl AbstractSyntaxTree for Match { left: MatchArm { pattern: from.left().pattern().clone(), expression: ast_l, + span: *from.left().span(), }, right: MatchArm { pattern: from.right().pattern().clone(), expression: ast_r, + span: *from.right().span(), }, span: *from.as_ref(), }) @@ -2387,6 +2429,18 @@ impl AsRef for Assignment { } } +impl AsRef for FunctionParam { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for CustomFunction { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for Expression { fn as_ref(&self) -> &Span { &self.span @@ -2411,6 +2465,110 @@ impl AsRef for Match { } } +impl AsRef for MatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for EnumMatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +#[cfg(test)] +mod span_tests { + use crate::parse::ParseFromStr; + + use super::*; + + #[test] + fn analyzed_custom_function_preserves_declaration_and_parameter_spans() { + let source = "fn helper(value: u8) -> u8 { value }"; + let parsed = parse::Function::parse_from_str(source).expect("function parses"); + let mut scope = Scope::new(Box::new(ElementsJetHinter)); + + Function::analyze(&parsed, &ResolvedType::unit(), &mut scope).expect("function analyzes"); + let function = scope + .get_function(parsed.name()) + .expect("function is registered in scope"); + + assert_eq!(function.span().to_slice(source), Some(source)); + assert_eq!( + function.params()[0].span().to_slice(source), + Some("value: u8") + ); + } + + #[test] + fn analyzed_match_arms_preserve_their_parsed_spans() { + let source = r#"fn main() { + let input: Either = Left(1); + match input { + Left(left: u8) => {}, + Right(right: u8) => {}, + } +}"#; + let parsed = parse::Program::parse_from_str(source).expect("program parses"); + let program = + Program::analyze(&parsed, Box::new(ElementsJetHinter)).expect("program analyzes"); + + let ExpressionInner::Block(_, Some(last)) = program.main().inner() else { + panic!("main body should end in a match"); + }; + let ExpressionInner::Single(single) = last.inner() else { + panic!("match should be a single expression"); + }; + let SingleExpressionInner::Match(match_) = single.inner() else { + panic!("expected a binary match"); + }; + + assert_eq!( + match_.left().span().to_slice(source), + Some("Left(left: u8) => {},") + ); + assert_eq!( + match_.right().span().to_slice(source), + Some("Right(right: u8) => {},") + ); + } + + #[test] + fn analyzed_enum_match_arms_preserve_their_parsed_spans() { + let source = r#"enum Choice { First, Second, } +fn main() { + let input: Choice = Choice::First; + match input { + Choice::First => {}, + Choice::Second => {}, + } +}"#; + let parsed = parse::Program::parse_from_str(source).expect("program parses"); + let program = + Program::analyze(&parsed, Box::new(ElementsJetHinter)).expect("program analyzes"); + + let ExpressionInner::Block(_, Some(last)) = program.main().inner() else { + panic!("main body should end in an enum match"); + }; + let ExpressionInner::Single(single) = last.inner() else { + panic!("enum match should be a single expression"); + }; + let SingleExpressionInner::EnumMatch(match_) = single.inner() else { + panic!("expected an enum match"); + }; + + assert_eq!( + match_.arms()[0].span().to_slice(source), + Some("Choice::First => {},") + ); + assert_eq!( + match_.arms()[1].span().to_slice(source), + Some("Choice::Second => {},") + ); + } +} + #[cfg(test)] mod scope_resolution_tests { use super::{ElementsJetHinter, Program}; @@ -3446,3 +3604,45 @@ mod enum_tests { assert!(result.is_err(), "enum syntax is gated behind -Z enums"); } } + +#[cfg(feature = "fmt")] +#[cfg(test)] +mod literal_tests { + use crate::parse::ParseFromStr; + use crate::value::{UIntValue, Value}; + + use super::*; + + #[test] + fn analyzed_numeric_literals_accept_digit_separators() { + let cases = [ + ("1_337", UIntType::U16, Value::from(UIntValue::U16(1_337))), + ( + "0b1010_0101", + UIntType::U8, + Value::from(UIntValue::U8(0b1010_0101)), + ), + ( + "0xDE_AD_BE_EF", + UIntType::U32, + Value::from(UIntValue::U32(0xdead_beef)), + ), + ]; + + for (source, integer_type, expected) in cases { + let parsed = parse::Expression::parse_from_str(source).expect("literal parses"); + let analyzed = + Expression::analyze_const(&parsed, &integer_type.into()).expect("literal analyzes"); + + let ExpressionInner::Single(single) = analyzed.inner() else { + panic!("expected a single expression") + }; + let SingleExpressionInner::Constant(value) = single.inner() else { + panic!("expected a constant expression") + }; + + assert_eq!(value, &expected, "unexpected value for {source:?}"); + assert_eq!(single.span().to_slice(source), Some(source)); + } + } +} diff --git a/src/lexer.rs b/src/lexer.rs index 862195e1..f71375a2 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -11,7 +11,11 @@ use crate::version::SIMC_STR; pub type Spanned = (T, SimpleSpan); pub type Tokens<'src> = Vec<(Token<'src>, crate::error::Span)>; +#[cfg(feature = "fmt")] +pub type FmtTokens<'src> = Vec<(FmtToken<'src>, crate::error::Span)>; + #[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum Token<'src> { // Keywords Pub, @@ -67,12 +71,81 @@ pub enum Token<'src> { // Built-in functions Macro(&'src str), +} - // Comments and block comments - // - // We would discard them for the compiler, but they are needed, for example, for the formatter. - Comment, +#[cfg(feature = "fmt")] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum TriviaKind { + LineComment, BlockComment, + Newline, + Whitespace, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LineEnding { + /// Windows carriage-return/line-feed (`\r\n`). + CrLf, + /// Unix line-feed (`\n`). + Lf, + /// Classic Mac OS carriage-return (`\r`). + Cr, +} + +impl LineEnding { + pub const fn as_str(self) -> &'static str { + match self { + Self::CrLf => "\r\n", + Self::Lf => "\n", + Self::Cr => "\r", + } + } +} + +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Trivia<'src> { + LineComment(&'src str), + BlockComment(&'src str), + Newline(LineEnding), + Whitespace(&'src str), +} + +#[cfg(feature = "fmt")] +impl<'src> Trivia<'src> { + pub const fn line_comment(text: &'src str) -> Self { + Self::LineComment(text) + } + + pub const fn block_comment(text: &'src str) -> Self { + Self::BlockComment(text) + } + + pub const fn newline(line_ending: LineEnding) -> Self { + Self::Newline(line_ending) + } + + pub const fn whitespace(text: &'src str) -> Self { + Self::Whitespace(text) + } + + pub const fn kind(&self) -> TriviaKind { + match self { + Self::LineComment(_) => TriviaKind::LineComment, + Self::BlockComment(_) => TriviaKind::BlockComment, + Self::Newline(_) => TriviaKind::Newline, + Self::Whitespace(_) => TriviaKind::Whitespace, + } + } +} + +#[cfg(feature = "fmt")] +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum FmtToken<'src> { + Token(Token<'src>), + Trivia(Trivia<'src>), } impl<'src> fmt::Display for Token<'src> { @@ -120,9 +193,21 @@ impl<'src> fmt::Display for Token<'src> { Token::Param(s) => write!(f, "param::{}", s), Token::Bool(b) => write!(f, "{}", b), + } + } +} - Token::Comment => write!(f, "comment"), - Token::BlockComment => write!(f, "block_comment"), +#[cfg(feature = "fmt")] +impl<'src> fmt::Display for FmtToken<'src> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FmtToken::Token(t) => { + write!(f, "{}", t) + } + FmtToken::Trivia( + Trivia::LineComment(text) | Trivia::BlockComment(text) | Trivia::Whitespace(text), + ) => write!(f, "{text}"), + FmtToken::Trivia(Trivia::Newline(line_ending)) => write!(f, "{}", line_ending.as_str()), } } } @@ -130,8 +215,41 @@ impl<'src> fmt::Display for Token<'src> { /// Recognizer for a `// ...` line comment. fn line_comment<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + let newline = line_ending(); + just("//") - .then(any().and_is(just('\n').not()).repeated()) + .ignore_then(any().and_is(newline.not()).repeated()) + .ignored() +} + +/// Recognizer for different newline encodings (`Windows`: `\r\n`, `Unix`: `\n`, `Mac`: `\r`). +fn line_ending<'src>( +) -> impl Parser<'src, &'src str, LineEnding, extra::Err>> + Clone { + choice(( + just("\r\n").to(LineEnding::CrLf), + just("\n").to(LineEnding::Lf), + just("\r").to(LineEnding::Cr), + )) +} + +/// Recognizer for whitespace. +#[cfg(feature = "fmt")] +fn whitespace<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any() + .filter(|c: &char| c.is_whitespace() && *c != '\n' && *c != '\r') + .repeated() + .at_least(1) + .ignored() +} + +/// Recognizer for whitespace or a newline. +fn whitespace_or_newline<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + any() + .filter(|c: &char| c.is_whitespace()) + .repeated() + .at_least(1) .ignored() } @@ -152,44 +270,61 @@ fn block_comment<'src>( }) } +/// One non-empty trivia item. Keeping this separate from [`trivia`] lets the +/// ordinary lexer include trivia in the same recovery boundary as tokens. +fn trivia_item<'src>( +) -> impl Parser<'src, &'src str, (), extra::Err>> + Clone { + choice((line_comment(), block_comment(), whitespace_or_newline())) +} + /// Trivia — whitespace and comments — shared with the version-directive scanner /// (`version::SimcDirective::scan`) so the lexer and the scanner agree on comment /// syntax. pub(crate) fn trivia<'src>( ) -> impl Parser<'src, &'src str, (), extra::Err>> { - choice(( - line_comment(), - block_comment(), - any().filter(|c: &char| c.is_whitespace()).ignored(), - )) - .repeated() - .ignored() + trivia_item().repeated().ignored() } -pub fn lexer<'src>( -) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> -{ - let digits_with_underscore = |radix: u32| { - any() - .filter(move |c: &char| c.is_digit(radix)) - .then( - any() - .filter(move |c: &char| c.is_digit(radix) || *c == '_') - .repeated(), - ) - .to_slice() - }; +/// Parses the digit body of a numeric literal. +fn digits_with_underscores<'src>( + radix: u32, +) -> impl Parser<'src, &'src str, &'src str, extra::Err>> { + any() + .filter(move |c: &char| c.is_digit(radix)) + .then( + any() + .filter(move |c: &char| c.is_digit(radix) || *c == '_') + .repeated(), + ) + .to_slice() +} - let num = digits_with_underscore(10) - .map(|s: &str| Token::DecLiteral(Decimal::from_str_unchecked(s.replace('_', "").as_str()))); +fn digit_literal_text(input: &str) -> std::borrow::Cow<'_, str> { + if PRESERVE || !input.contains('_') { + std::borrow::Cow::Borrowed(input) + } else { + std::borrow::Cow::Owned(input.replace('_', "")) + } +} + +fn to_token<'src, const PRESERVE: bool>( +) -> impl Parser<'src, &'src str, Token<'src>, extra::Err>> { + let num = digits_with_underscores(10).map(|s: &str| { + let text = digit_literal_text::(s); + Token::DecLiteral(Decimal::from_str_unchecked(text.as_ref())) + }); let hex = just("0x") - .ignore_then(digits_with_underscore(16)) + .ignore_then(digits_with_underscores(16)) .map(|s: &str| { - Token::HexLiteral(Hexadecimal::from_str_unchecked(s.replace('_', "").as_str())) + let text = digit_literal_text::(s); + Token::HexLiteral(Hexadecimal::from_str_unchecked(text.as_ref())) }); let bin = just("0b") - .ignore_then(digits_with_underscore(2)) - .map(|s: &str| Token::BinLiteral(Binary::from_str_unchecked(s.replace('_', "").as_str()))); + .ignore_then(digits_with_underscores(2)) + .map(|s: &str| { + let text = digit_literal_text::(s); + Token::BinLiteral(Binary::from_str_unchecked(text.as_ref())) + }); let macros = choice((just("assert!"), just("panic!"), just("dbg!"), just("list!"))).map(Token::Macro); @@ -243,25 +378,53 @@ pub fn lexer<'src>( just(">").to(Token::RAngle), )); - let comment = line_comment().to(Token::Comment); - let block_comment = block_comment().to(Token::BlockComment); - let token = choice(( - comment, - block_comment, - jet, - witness, - param, - macros, - keyword, - hex, - bin, - num, - op, - )); + choice((jet, witness, param, macros, hex, bin, num, keyword, op)) +} + +pub fn lexer<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + const REMOVE_SEPARATORS: bool = false; + + let lexeme = choice(( + trivia_item().to(None), + to_token::().map(Some), + )) + .map_with(|token, e| (token, e.span())) + .recover_with(skip_then_retry_until(any().ignored(), end())); - token - .map_with(|tok, e| (tok, e.span())) - .padded() + lexeme.repeated().collect::>().map(|lexemes| { + lexemes + .into_iter() + .filter_map(|(token, span)| token.map(|token| (token, span))) + .collect() + }) +} + +#[cfg(feature = "fmt")] +pub fn lexer_lossless<'src>( +) -> impl Parser<'src, &'src str, Vec>>, extra::Err>> +{ + const PRESERVE_SEPARATORS: bool = true; + + let token = to_token::().map(FmtToken::Token); + + let newline = line_ending().map(Trivia::newline).map(FmtToken::Trivia); + let whitespace = whitespace() + .to_slice() + .map(Trivia::whitespace) + .map(FmtToken::Trivia); + let line_comment = line_comment() + .to_slice() + .map(Trivia::line_comment) + .map(FmtToken::Trivia); + let block_comment = block_comment() + .to_slice() + .map(Trivia::block_comment) + .map(FmtToken::Trivia); + + choice((line_comment, block_comment, newline, whitespace, token)) + .map_with(|lexeme, e| (lexeme, e.span())) .recover_with(skip_then_retry_until(any().ignored(), end())) .repeated() .collect() @@ -271,7 +434,7 @@ pub fn lexer<'src>( /// offset `start` — the end of the version directive per `SimcDirective::prescan`, /// or `0`. Spans are reported relative to the full input. /// -/// All comments in the input code are discarded. +/// All comments, newlines, and spaces in the input code are discarded. pub fn lex( file_id: usize, input: &str, @@ -280,26 +443,59 @@ pub fn lex( let (tokens, lex_errors) = lexer().parse(&input[start..]).into_output_errors(); let shift = |span| Span::from_chumsky(file_id, span, start); - let mut diagnostics: Vec = Vec::new(); + let mut diagnostics: Vec = lex_errors + .into_iter() + .map(|err| { + Diagnostic::new( + Error::CannotParse { + msg: err.reason().to_string(), + }, + shift(*err.span()), + ) + }) + .collect(); - diagnostics.extend(lex_errors.into_iter().map(|err| { - Diagnostic::new( - Error::CannotParse { - msg: err.reason().to_string(), - }, - shift(*err.span()), - ) - })); + let tokens = tokens.map(|vec| { + vec.into_iter() + .filter_map(|(tok, span)| filter_token(tok, span, &mut diagnostics, shift)) + .collect() + }); + + (tokens, diagnostics) +} + +/// Lexes an input string into a lossles stream of tokens with spans, beginning at byte +/// offset `start` — the end of the version directive per `SimcDirective::prescan`, +/// or `0`. Spans are reported relative to the full input. +/// +/// All comments, newlines, and spaces in the input code are remained. +#[cfg(feature = "fmt")] +pub fn lex_lossless( + file_id: usize, + input: &str, + start: usize, +) -> (Option>, Vec) { + let (tokens, lex_errors) = lexer_lossless().parse(&input[start..]).into_output_errors(); + let shift = |span: SimpleSpan| Span::from_chumsky(file_id, span, start); + + let mut diagnostics: Vec = lex_errors + .into_iter() + .map(|err| { + Diagnostic::new( + Error::CannotParse { + msg: err.reason().to_string(), + }, + shift(*err.span()), + ) + }) + .collect(); let tokens = tokens.map(|vec| { vec.into_iter() - .filter_map(|(tok, span)| match tok { - Token::Comment | Token::BlockComment => None, - Token::Simc => { - diagnostics.push(Diagnostic::new(Error::ReservedSimcKeyword, shift(span))); - None - } - tok => Some((tok, shift(span))), + .filter_map(|(fmt_tok, span)| match fmt_tok { + FmtToken::Token(tok) => filter_token(tok, span, &mut diagnostics, shift) + .map(|(t, s)| (FmtToken::Token(t), s)), + FmtToken::Trivia(t) => Some((FmtToken::Trivia(t), shift(span))), }) .collect() }); @@ -307,6 +503,26 @@ pub fn lex( (tokens, diagnostics) } +fn filter_token<'src, F: Fn(SimpleSpan) -> Span>( + tok: Token<'src>, + span: SimpleSpan, + errors: &mut Vec, + convert_span: F, +) -> Option<(Token<'src>, Span)> { + match tok { + // The reserved keyword is a sentinel: the prescan consumed the one + // legitimate directive before lexing, so any occurrence is misplaced. + Token::Simc => { + errors.push(Diagnostic::new( + Error::ReservedSimcKeyword, + convert_span(span), + )); + None + } + tok => Some((tok, convert_span(span))), + } +} + /// A list of all reserved keywords. pub const KEYWORDS: &[&str] = &[ "pub", "use", "as", "fn", "let", "type", "mod", "const", "match", "enum", CRATE_STR, SIMC_STR, @@ -319,19 +535,20 @@ pub fn is_keyword(s: &str) -> bool { } #[cfg(test)] -mod tests { - use chumsky::error::Rich; - +mod original_lexer { use super::*; fn lex<'src>( input: &'src str, ) -> (Option>>, Vec>) { let (tokens, errors) = lexer().parse(input).into_output_errors(); - let tokens = tokens.map(|vec| vec.iter().map(|(tok, _)| tok.clone()).collect::>()); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(tok, _)| tok.clone()) + .collect::>() + }); (tokens, errors) } - #[test] fn test_block_comment_simple() { let input = "/* hello world */"; @@ -340,7 +557,7 @@ mod tests { assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); assert_eq!( tokens, - Some(vec![Token::BlockComment]), + Some(vec![]), "Should produce a single block comment token" ); } @@ -350,8 +567,8 @@ mod tests { let input = "/* outer /* inner */ outer */"; let (tokens, errors) = lex(input); - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); } #[test] @@ -359,8 +576,8 @@ mod tests { let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; let (tokens, errors) = lex(input); - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); } #[test] @@ -368,8 +585,8 @@ mod tests { let input = "/* \n line 1 \n /* inner \n line */ \n */"; let (tokens, errors) = lex(input); - assert!(errors.is_empty()); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); } #[test] @@ -384,7 +601,7 @@ mod tests { assert_eq!(err.span().end, 2); assert_eq!(err.to_string(), "Unclosed block comment"); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); } #[test] @@ -394,7 +611,7 @@ mod tests { assert_eq!(errors.len(), 1); assert_eq!(errors[0].span().start, 0); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); } #[test] @@ -410,7 +627,70 @@ mod tests { assert_eq!(errors[1].span().start, 0); assert_eq!(errors[1].to_string(), "Unclosed block comment"); - assert_eq!(tokens, Some(vec![Token::BlockComment])); + assert_eq!(tokens, Some(vec![])); + } + + #[test] + fn test_spaces_resolution() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![])); + } + + #[test] + fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { + let input = "fn main() {} @// fn hello(){} simc"; + let (tokens, errors) = lex(input); + + assert_eq!( + errors.len(), + 1, + "comment contents must not be retried as code" + ); + assert!(errors[0].to_string().contains("found '@' expected")); + assert_eq!( + tokens, + Some(vec![ + Token::Fn, + Token::Ident("main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + ]) + ); + + let (_tokens, diagnostics) = super::lex(0, input, 0); + assert_eq!(diagnostics.len(), 1); + assert!(matches!(diagnostics[0].error(), Error::CannotParse { .. })); + } + + #[test] + fn test_ignoring_tokens_after_comment() { + let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ + // enum Name {} match true {} \n fn other_main() {} "; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![ + Token::Fn, + Token::Ident("main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + Token::Fn, + Token::Ident("other_main"), + Token::LParen, + Token::RParen, + Token::LBrace, + Token::RBrace, + ]) + ); } #[test] @@ -434,7 +714,7 @@ mod tests { // Identifiers merely starting with `simc` are ordinary identifiers. let (tokens, errors) = lex("simcfoo"); - assert!(errors.is_empty(), "unexpected: {errors:?}"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); assert_eq!(tokens, Some(vec![Token::Ident("simcfoo")])); } @@ -456,6 +736,68 @@ mod tests { ); } + #[test] + fn numeric_literals_with_underscores_and_underscore_digit_identifiers() { + let cases = [ + ( + "[u8; 1_6]", + vec![ + Token::LBracket, + Token::Ident("u8"), + Token::Semi, + Token::DecLiteral(Decimal::from_str_unchecked("16")), + Token::RBracket, + ], + ), + ( + "List", + vec![ + Token::Ident("List"), + Token::LAngle, + Token::Ident("u8"), + Token::Comma, + Token::DecLiteral(Decimal::from_str_unchecked("16")), + Token::RAngle, + ], + ), + ( + "1_3_3_7", + vec![Token::DecLiteral(Decimal::from_str_unchecked("1337"))], + ), + ( + "0x1_", + vec![Token::HexLiteral(Hexadecimal::from_str_unchecked("1"))], + ), + ( + "0b1010_0101", + vec![Token::BinLiteral(Binary::from_str_unchecked("10100101"))], + ), + ("_34", vec![Token::Ident("_34")]), + ("_34foo", vec![Token::Ident("_34foo")]), + ]; + + for (input, expected) in cases { + let (tokens, errors) = lex(input); + assert!( + errors.is_empty(), + "Expected no errors for {input:?}: {errors:?}" + ); + assert_eq!(tokens, Some(expected), "Unexpected tokens for {input:?}"); + } + } + + #[test] + fn leading_whitespaces_before_main() { + let input = " fn main(){}"; + let (tokens, errors) = lex(input); + + assert!(errors.is_empty()); + assert!(tokens.is_some()); + + let tokens = tokens.unwrap(); + assert_eq!(tokens[0], Token::Fn); + } + #[test] fn lexer_test() { use chumsky::prelude::*; @@ -469,3 +811,366 @@ mod tests { assert!(lex_errs.is_empty()); } } + +#[cfg(feature = "fmt")] +#[cfg(test)] +mod fmt_lexer { + use super::*; + + fn lex_lossless<'src>( + input: &'src str, + ) -> ( + Option>>, + Vec>, + ) { + let (tokens, errors) = lexer_lossless().parse(input).into_output_errors(); + let tokens = tokens.map(|vec| { + vec.into_iter() + .map(|(fmt_tok, _)| fmt_tok) + .collect::>() + }); + (tokens, errors) + } + + fn fmt_trivia<'src>(kind: TriviaKind, text: &'src str) -> FmtToken<'src> { + let trivia = match kind { + TriviaKind::LineComment => Trivia::line_comment(text), + TriviaKind::BlockComment => Trivia::block_comment(text), + TriviaKind::Newline => Trivia::newline(match text { + "\r\n" => LineEnding::CrLf, + "\n" => LineEnding::Lf, + "\r" => LineEnding::Cr, + _ => panic!("invalid newline spelling: {text:?}"), + }), + TriviaKind::Whitespace => Trivia::whitespace(text), + }; + + FmtToken::Trivia(trivia) + } + + #[test] + fn test_block_comment_simple_fmt() { + let input = "/* hello world */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]), + "Should produce a single block comment token" + ); + } + + #[test] + fn lossless_trivia_display_preserves_source_text() { + let input = "// comment\r\n\t"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let rendered: String = tokens + .expect("lossless lexing succeeds") + .iter() + .map(ToString::to_string) + .collect(); + + assert_eq!(rendered, input); + } + + #[test] + fn lossless_lexer_keeps_each_newline_kind_with_its_span() { + let input = "first\r\nsecond\rthird\nfourth"; + let (tokens, errors) = super::lex_lossless(0, input, 0); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + let tokens = tokens.expect("lossless lexing succeeds"); + let line_endings: Vec<_> = tokens + .iter() + .filter_map(|(token, _)| match token { + FmtToken::Trivia(Trivia::Newline(line_ending)) => Some(*line_ending), + _ => None, + }) + .collect(); + let newlines: Vec<_> = tokens + .iter() + .filter(|(token, _)| { + matches!(token, FmtToken::Trivia(trivia) if trivia.kind() == TriviaKind::Newline) + }) + .map(|(_, span)| span.to_slice(input)) + .collect(); + + assert_eq!( + line_endings, + vec![LineEnding::CrLf, LineEnding::Cr, LineEnding::Lf] + ); + assert_eq!(newlines, vec![Some("\r\n"), Some("\r"), Some("\n")]); + } + + #[test] + fn test_block_comment_nested_fmt() { + let input = "/* outer /* inner */ outer */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_deeply_nested_fmt() { + let input = "/* 1 /* 2 /* 3 */ 2 */ 1 */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_multiline_fmt() { + let input = "/* \n line 1 \n /* inner \n line */ \n */"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_unclosed_fmt() { + let input = "/* unclosed comment start"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "Expected exactly 1 error"); + + let err = &errors[0]; + assert_eq!(err.span().start, 0); + assert_eq!(err.span().end, 2); + assert_eq!(err.to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_partial_nesting_unclosed_fmt() { + let input = "/* outer /* inner */"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1); + assert_eq!(errors[0].span().start, 0); + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_block_comment_double_unclosed_fmt() { + let input = "/* outer /* inner"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 2); + + assert_eq!(errors[0].span().start, 9); + assert_eq!(errors[0].to_string(), "Unclosed block comment"); + + assert_eq!(errors[1].span().start, 0); + assert_eq!(errors[1].to_string(), "Unclosed block comment"); + + assert_eq!( + tokens, + Some(vec![fmt_trivia(TriviaKind::BlockComment, input)]) + ); + } + + #[test] + fn test_spaces_resolution_fmt() { + let input = "\r\n\n\r\r\r\r\r\n\r\n \n\n\r\n\n\r"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!( + tokens, + Some(vec![ + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r\n"), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Newline, "\r"), + ]) + ); + } + + #[test] + fn test_ignoring_tokens_after_comment_with_incorrect_symbol() { + let input = "fn main() {} @// fn hello(){} simc"; + let (tokens, errors) = lex_lossless(input); + + assert_eq!(errors.len(), 1, "only the invalid character is an error"); + assert!(errors[0].to_string().contains("found '@' expected")); + + let tokens = tokens.expect("recovery keeps the lossless stream"); + assert!(matches!( + tokens.last(), + Some(FmtToken::Trivia(Trivia::LineComment(_))) + )); + assert!( + tokens + .iter() + .all(|token| !matches!(token, FmtToken::Token(Token::Simc))), + "comment contents must not become semantic tokens" + ); + } + + #[test] + fn test_not_ignoring_tokens_after_comment() { + let input = "fn main() {} /* fn hello(){} \n simc 0.6.0; */ \n\ + // enum Name {} match true {} \n fn other_main() {} "; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert_eq!( + tokens, + Some(vec![ + FmtToken::Token(Token::Fn), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Ident("main")), + FmtToken::Token(Token::LParen), + FmtToken::Token(Token::RParen), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::LBrace), + FmtToken::Token(Token::RBrace), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia( + TriviaKind::BlockComment, + "/* fn hello(){} \n simc 0.6.0; */" + ), + fmt_trivia(TriviaKind::Whitespace, " "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::LineComment, "// enum Name {} match true {} "), + fmt_trivia(TriviaKind::Newline, "\n"), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Fn), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::Ident("other_main")), + FmtToken::Token(Token::LParen), + FmtToken::Token(Token::RParen), + fmt_trivia(TriviaKind::Whitespace, " "), + FmtToken::Token(Token::LBrace), + FmtToken::Token(Token::RBrace), + fmt_trivia(TriviaKind::Whitespace, " "), + ]) + ); + } + + #[test] + fn simc_is_reserved_fmt() { + // The prescan consumes the one legitimate leading directive before lexing, + // so `lex` reports any `simc` it sees and drops the sentinel token. + for src in ["simc", "fn simc() {}", "fn f() {}\nsimc"] { + let (tokens, errors) = super::lex_lossless(0, src, 0); + assert!( + errors.iter().any(|e| e.to_string().contains("reserved")), + "expected a reserved-keyword error for {src:?}, got: {errors:?}" + ); + + assert!( + tokens + .expect("recovery keeps the stream") + .iter() + .all(|(tok, _)| !matches!(tok, FmtToken::Token(Token::Simc))), + "the sentinel must not reach the token stream for {src:?}" + ); + } + + // Identifiers merely starting with `simc` are ordinary identifiers. + let (tokens, errors) = lex_lossless("simcfoo"); + assert!(errors.is_empty(), "Expected no errors, found: {:?}", errors); + assert_eq!(tokens, Some(vec![FmtToken::Token(Token::Ident("simcfoo"))])); + } + + #[test] + fn numeric_literals_preserve_underscores() { + let input = "1_234_567_89 0xDEAD_BEEF 0b1010_0101_ 0b1010_0101 1_234__567___890____000_____ 0xDEAD_BEEF__BEEF_DEAD_"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty(), "Expected no errors, found: {errors:?}"); + assert!(tokens.is_some(), "lexing must succeed"); + let tokens = tokens + .unwrap() + .into_iter() + .filter(|x| !matches!(x, FmtToken::Trivia(Trivia::Whitespace(_)))) + .collect::>(); + assert_eq!( + tokens, + vec![ + FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( + "1_234_567_89" + ))), + FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( + "DEAD_BEEF" + ))), + FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked("1010_0101_"))), + FmtToken::Token(Token::BinLiteral(Binary::from_str_unchecked("1010_0101"))), + FmtToken::Token(Token::DecLiteral(Decimal::from_str_unchecked( + "1_234__567___890____000_____" + ))), + FmtToken::Token(Token::HexLiteral(Hexadecimal::from_str_unchecked( + "DEAD_BEEF__BEEF_DEAD_" + ))), + ] + ); + + assert_eq!( + tokens + .iter() + .map(ToString::to_string) + .collect::>() + .join(" "), + input + ); + } + + #[test] + fn leading_whitespaces_before_main() { + let input = " fn main(){}"; + let (tokens, errors) = lex_lossless(input); + + assert!(errors.is_empty()); + assert!(tokens.is_some()); + + let tokens = dbg!(tokens).unwrap(); + assert!(matches!(tokens[0], FmtToken::Trivia(Trivia::Whitespace(_)))); + assert!(matches!(tokens[1], FmtToken::Token(Token::Fn))); + } + + #[test] + fn lossless_lexer_test() { + use chumsky::prelude::*; + + // Check if the lexer parses the example file without errors. + let src = include_str!("../examples/last_will.simf"); + + let (tokens, lex_errs) = lexer_lossless().parse(src).into_output_errors(); + let _ = tokens.unwrap(); + + assert!(lex_errs.is_empty()); + } +} diff --git a/src/parse.rs b/src/parse.rs index a627adeb..547af341 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -22,7 +22,7 @@ use crate::driver::{CRATE_STR, MAIN_MODULE}; use crate::error::DiagnosticManager; use crate::error::{Diagnostic, Error, Span}; use crate::impl_eq_hash; -use crate::lexer::Token; +use crate::lexer::{Token, Tokens}; use crate::num::NonZeroPow2Usize; use crate::pattern::Pattern; use crate::str::{ @@ -30,9 +30,12 @@ use crate::str::{ SymbolName, WitnessName, }; use crate::types::{AliasedType, BuiltinAlias, TypeConstructible, UIntType}; -use crate::unstable::{impl_require_feature, UnstableFeature, UnstableFeatures}; +use crate::unstable::{impl_require_feature, RequireFeature, UnstableFeature, UnstableFeatures}; use crate::version::SimcDirective; +#[cfg(feature = "fmt")] +use crate::lexer::{FmtToken, FmtTokens}; + /// A program is a sequence of items. #[derive(Clone, Debug)] pub struct Program { @@ -40,6 +43,40 @@ pub struct Program { span: Span, } +/// Source-aware parse result used by formatters. +/// +/// The compiler keeps using [`Program`] directly. Formatters need the same +/// semantic tree plus the lossless trivia stream that the grammar intentionally +/// does not consume. +#[cfg(feature = "fmt")] +#[derive(Clone, Debug)] +pub struct ParsedSource<'src> { + program: Program, + tokens: FmtTokens<'src>, + prefix: Span, +} + +#[cfg(feature = "fmt")] +impl<'src> ParsedSource<'src> { + /// Access the semantic program used for formatting. + pub fn program(&self) -> &Program { + &self.program + } + + /// Access the lossless token stream in source order. + pub fn tokens(&self) -> &FmtTokens<'src> { + &self.tokens + } + + /// Access the source prefix skipped by the version-directive prescan. + /// + /// A formatter should preserve this range verbatim until the directive gains + /// its own formatting grammar. + pub fn prefix(&self) -> Span { + self.prefix + } +} + impl Program { // Need for driver usage pub(crate) fn new(items: &[Item], span: Span) -> Self { @@ -53,6 +90,50 @@ impl Program { pub fn items(&self) -> &[Item] { &self.items } + + /// Parse source for formatting while retaining all comments and whitespace. + #[cfg(feature = "fmt")] + pub fn parse_with_errors_for_fmt<'src>( + file_id: usize, + source: &'src str, + unstable_features: &UnstableFeatures, + diagnostics: &mut DiagnosticManager, + ) -> Option> { + let before = diagnostics.error_count(); + + let start = pipeline::directive_prescan(source, file_id, diagnostics)?; + + let (tokens, lex_errors) = crate::lexer::lex_lossless(file_id, source, start); + let lex_ok = pipeline::is_lex_ok(lex_errors, diagnostics)?; + + let tokens = tokens?; + + let semantic_tokens = tokens + .iter() + .filter_map(|(token, span)| match token { + FmtToken::Token(token) => Some((token.clone(), *span)), + FmtToken::Trivia(_) => None, + }) + .collect::>(); + + let (program, parse_ok) = + pipeline::parse_ast(file_id, source, semantic_tokens, diagnostics); + + if parse_ok && lex_ok { + pipeline::post_check(unstable_features, program.as_ref(), diagnostics); + } + + if diagnostics.error_count() > before { + None + } else { + let program = program?; + Some(ParsedSource { + program, + tokens, + prefix: Span::new(file_id, 0..start), + }) + } + } } impl_eq_hash!(Program; items); @@ -80,6 +161,22 @@ pub enum Item { Ignored, } +impl Item { + /// Access the source span when this item was parsed successfully. + /// + /// Error-recovery placeholders have no source node to decorate. + pub fn span(&self) -> Option<&Span> { + match self { + Self::TypeAlias(alias) => Some(alias.span()), + Self::Function(function) => Some(function.span()), + Self::Use(use_decl) => Some(use_decl.span()), + Self::EnumDeclaration(declaration) => Some(declaration.span()), + Self::Module(module) => Some(module.span()), + Self::Ignored => None, + } + } +} + impl_require_feature!(Item { variants: TypeAlias(alias), @@ -277,11 +374,12 @@ impl_eq_hash!(Function; visibility, name, params, ret, body); impl_require_feature!(Function { recurse: params, ret, body; }); /// Parameter of a function. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] #[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] pub struct FunctionParam { identifier: Identifier, ty: AliasedType, + span: Span, } impl FunctionParam { @@ -294,8 +392,15 @@ impl FunctionParam { pub fn ty(&self) -> &AliasedType { &self.ty } + + /// Access the source span of the complete parameter. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(FunctionParam; identifier, ty); + impl_require_feature!(FunctionParam { recurse: ty; }); /// A statement is a component of a block expression. @@ -307,6 +412,19 @@ pub enum Statement { Expression(Expression), } +impl Statement { + /// Access the span of the statement contents. + /// + /// The terminating semicolon is deliberately not part of this span: it is a + /// separator owned by the containing block's layout. + pub fn span(&self) -> &Span { + match self { + Self::Assignment(assignment) => assignment.span(), + Self::Expression(expression) => expression.span(), + } + } +} + impl_require_feature!(Statement { variants: Assignment(assignment), @@ -502,6 +620,11 @@ impl EnumVariant { pub fn payload(&self) -> &[AliasedType] { &self.payload } + + /// Access the source span of the complete variant. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumVariant; name, payload); @@ -533,6 +656,11 @@ impl EnumDeclaration { pub fn variants(&self) -> &[EnumVariant] { &self.variants } + + /// Access the source span of the complete declaration. + pub fn span(&self) -> &Span { + &self.span + } } impl_require_feature!(EnumVariant { recurse: payload; }); @@ -829,6 +957,7 @@ pub struct EnumMatchArm { variant: Identifier, bindings: Arc<[(Pattern, AliasedType)]>, expression: Arc, + span: Span, } impl EnumMatchArm { @@ -860,6 +989,11 @@ impl EnumMatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the source span of the complete arm, including its optional comma. + pub fn span(&self) -> &Span { + &self.span + } } impl_eq_hash!(EnumMatchArm; enum_path, variant, bindings, expression); @@ -942,10 +1076,11 @@ impl AsRef for EnumConstruction { } /// Arm of a match expression. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug)] pub struct MatchArm { pattern: MatchPattern, expression: Arc, + span: Span, } impl MatchArm { @@ -958,8 +1093,15 @@ impl MatchArm { pub fn expression(&self) -> &Expression { &self.expression } + + /// Access the source span of the complete arm, including its optional comma. + pub fn span(&self) -> &Span { + &self.span + } } +impl_eq_hash!(MatchArm; pattern, expression); + impl_require_feature!(MatchArm {recurse: pattern, expression; }); /// Pattern of a match arm. @@ -1520,6 +1662,10 @@ impl_parse_wrapped_string!(WitnessName, "witness name"); impl_parse_wrapped_string!(AliasName, "alias name"); impl_parse_wrapped_string!(ModuleName, "module name"); +trait AstNode: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + +impl AstNode for T where T: ChumskyParse + crate::unstable::RequireFeature + std::fmt::Debug {} + /// Copy of [`FromStr`] that internally uses the `chumsky` parser. pub trait ParseFromStr: Sized { /// Parse a value from the string `s`. @@ -1540,6 +1686,73 @@ pub trait ParseFromStrWithErrors: Sized { ) -> Option; } +mod pipeline { + use super::*; + /// Handle the `simc` directive before lexing: an incompatible or malformed + /// directive is reported as the only diagnostic (the rest is noise), and + /// lexing starts right after a valid one, so the lexer and grammar never + /// see it. + pub fn directive_prescan( + content: &str, + file_id: usize, + diagnostics: &mut DiagnosticManager, + ) -> Option { + match SimcDirective::prescan(content, file_id) { + Ok(start) => Some(start), + Err((err, span)) => { + diagnostics.push(Diagnostic::new(err, span)); + None + } + } + } + + pub fn is_lex_ok( + mut lex_errs: Vec, + diagnostics: &mut DiagnosticManager, + ) -> Option { + // A stray `simc` makes every other diagnostic noise — its `"";` remnant + // does not lex — so the reserved-keyword errors are reported alone. + if lex_errs + .iter() + .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) + { + lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); + diagnostics.extend(lex_errs); + None + } else { + let lex_ok = lex_errs.is_empty(); + diagnostics.extend(lex_errs); + Some(lex_ok) + } + } + + pub fn parse_ast( + file_id: usize, + src: &str, + tokens: Tokens<'_>, + diagnostics: &mut DiagnosticManager, + ) -> (Option, bool) { + let eoi = Span::eof(file_id, src.len()); + let (ast, parse_errs) = T::parser() + .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) + .into_output_errors(); + + let parse_ok = parse_errs.is_empty(); + diagnostics.extend(parse_errs); + (ast, parse_ok) + } + + pub fn post_check( + unstable_features: &UnstableFeatures, + program: Option<&T>, + diagnostics: &mut DiagnosticManager, + ) { + if let Some(ast) = program { + unstable_features.check_program(ast, diagnostics); + } + } +} + /// Trait for generating parsers of themselves. /// /// Replacement for previous `PestParse` trait. @@ -1594,9 +1807,7 @@ impl ParseFromStr for A { } } -impl ParseFromStrWithErrors - for A -{ +impl ParseFromStrWithErrors for A { fn parse_from_str_with_errors( file_id: usize, content: &str, @@ -1605,42 +1816,18 @@ impl ParseF ) -> Option { let before = diagnostics.error_count(); - // Handle the `simc` directive before lexing: an incompatible or malformed - // directive is reported as the only diagnostic (the rest is noise), and - // lexing starts right after a valid one, so the lexer and grammar never - // see it. - let start = match SimcDirective::prescan(content, file_id) { - Ok(start) => start, - Err((err, span)) => { - diagnostics.push(Diagnostic::new(err, span)); - return None; - } - }; + let start = pipeline::directive_prescan(content, file_id, diagnostics)?; + + let (tokens, lex_errs) = crate::lexer::lex(file_id, content, start); + + let lex_ok = pipeline::is_lex_ok(lex_errs, diagnostics)?; - let (tokens, mut lex_errs) = crate::lexer::lex(file_id, content, start); - // A stray `simc` makes every other diagnostic noise — its `"";` remnant - // does not lex — so the reserved-keyword errors are reported alone. - if lex_errs - .iter() - .any(|e| matches!(e.error(), Error::ReservedSimcKeyword)) - { - lex_errs.retain(|e| matches!(e.error(), Error::ReservedSimcKeyword)); - diagnostics.extend(lex_errs); - return None; - } - let lex_ok = lex_errs.is_empty(); - diagnostics.extend(lex_errs); let tokens = tokens?; - let eoi = Span::eof(file_id, content.len()); - let (ast, parse_errs) = A::parser() - .parse(tokens.as_slice().map(eoi, |(t, s)| (t, s))) - .into_output_errors(); - let parse_ok = parse_errs.is_empty(); - diagnostics.extend(parse_errs); + let (ast, parse_status) = pipeline::parse_ast::(file_id, content, tokens, diagnostics); - if let (Some(ast), true) = (&ast, lex_ok && parse_ok) { - unstable_features.check_program(ast, diagnostics); + if lex_ok && parse_status { + let () = pipeline::post_check(unstable_features, ast.as_ref(), diagnostics); } // TODO: We should return parsed result if we found errors, but because analyzing in `ast` module @@ -1786,7 +1973,10 @@ impl ChumskyParse for AliasedType { .then_ignore(parse_token_with_recovery(Token::Semi)) .then(num.clone()) .map(|(ty, size)| { - AliasedType::array(ty, usize::from_str(size.as_inner()).unwrap_or_default()) + let digits = + crate::str::underscore_parsing::strip_digit_separators(size.as_inner()); + + AliasedType::array(ty, usize::from_str(digits.as_ref()).unwrap_or_default()) }), Token::LBracket, Token::RBracket, @@ -1803,7 +1993,11 @@ impl ChumskyParse for AliasedType { .ignore_then(delimited_with_recovery( ty.then_ignore(parse_token_with_recovery(Token::Comma)) .then(num.clone().validate(|num, e, emit| -> NonZeroPow2Usize { - match NonZeroPow2Usize::from_str(num.as_inner()) { + let digits = crate::str::underscore_parsing::strip_digit_separators( + num.as_inner(), + ); + + match NonZeroPow2Usize::from_str(digits.as_ref()) { Ok(number) => number, Err(err) => { emit.emit( @@ -2034,7 +2228,11 @@ impl ChumskyParse for FunctionParam { identifier .then_ignore(just(Token::Colon)) .then(ty) - .map(|(identifier, ty)| Self { identifier, ty }) + .map_with(|(identifier, ty), e| Self { + identifier, + ty, + span: e.span(), + }) } } @@ -2180,7 +2378,10 @@ impl ChumskyParse for CallName { .then(select! { Token::DecLiteral(s) => s }.labelled("list size")) .then_ignore(generics_close.clone()) .validate(|(func, bound_str), e, emit| { - let bound = match bound_str.as_inner().parse::() { + let digits = + crate::str::underscore_parsing::strip_digit_separators(bound_str.as_inner()); + + let bound = match digits.parse::() { Ok(num) => match NonZeroPow2Usize::new(num) { Some(val) => val, None => { @@ -2209,7 +2410,10 @@ impl ChumskyParse for CallName { .then(select! { Token::DecLiteral(s) => s }.labelled("array size")) .then_ignore(generics_close.clone()) .validate(|(func, size_str), e, emit| { - let size = match size_str.as_inner().parse::() { + let digits = + crate::str::underscore_parsing::strip_digit_separators(size_str.as_inner()); + + let size = match digits.parse::() { Ok(0) => { emit.emit(Error::ArraySizeNonZero { size: 0 }.with_span(e.span())); NonZeroUsize::new(1).unwrap() @@ -2613,20 +2817,20 @@ struct EnumArmHead { } impl EnumArmHead { - fn into_arm(self, expression: Arc) -> EnumMatchArm { + fn into_arm(self, expression: Arc, span: Span) -> EnumMatchArm { EnumMatchArm { enum_path: self.enum_path, variant: self.variant, bindings: self.bindings, expression, + span, } } } /// One parsed match arm. An enum variant head or a built-in pattern, -/// plus the arm body. The head classification decides below whether the -/// whole match becomes an [`EnumMatch`] or a binary [`Match`]. -type ParsedMatchArm = (Either, Arc); +/// plus the arm body and its source boundaries. +type ParsedMatchArm = (Either, Arc, Span); /// Parser for the head of an enum match arm: `EnumName::Variant` with /// optional payload bindings `(pattern: Type, ...)`. @@ -2709,7 +2913,7 @@ where .with_span(e.span()), ); } - (head, expression) + (head, expression, e.span()) }) } @@ -2719,6 +2923,7 @@ fn placeholder_match(scrutinee: Arc, span: Span) -> SingleExpression let fallback_arm = MatchArm { expression: Arc::new(Expression::empty(Span::DUMMY)), pattern: MatchPattern::False, + span: Span::DUMMY, }; SingleExpressionInner::Match(Match { scrutinee, @@ -2765,11 +2970,12 @@ fn assemble_match_arms( // reports missing variants. The binary arm-count error below would be misleading there. let (enum_arms, builtin_arms): (Vec, Vec) = arms .into_iter() - .partition_map(|(head, expression)| match head { - Either::Left(head) => Either::Left(head.into_arm(expression)), + .partition_map(|(head, expression, arm_span)| match head { + Either::Left(head) => Either::Left(head.into_arm(expression, arm_span)), Either::Right(pattern) => Either::Right(MatchArm { pattern, expression, + span: arm_span, }), }); @@ -2950,6 +3156,12 @@ impl AsRef for Function { } } +impl AsRef for FunctionParam { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for Assignment { fn as_ref(&self) -> &Span { &self.span @@ -2986,6 +3198,18 @@ impl AsRef for Match { } } +impl AsRef for MatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + +impl AsRef for EnumMatchArm { + fn as_ref(&self) -> &Span { + &self.span + } +} + impl AsRef for UseDecl { fn as_ref(&self) -> &Span { &self.span @@ -3306,10 +3530,12 @@ impl crate::ArbitraryRec for Match { left: MatchArm { pattern: pat_l, expression: expr_l, + span: Span::DUMMY, }, right: MatchArm { pattern: pat_r, expression: expr_r, + span: Span::DUMMY, }, span: Span::DUMMY, }) @@ -3317,23 +3543,9 @@ impl crate::ArbitraryRec for Match { } #[cfg(test)] -mod test { - use crate::parse; - +mod type_alias { use super::*; - impl UseDecl { - /// Creates a dummy `UseDecl` specifically for testing `DependencyMap` resolution. - pub fn dummy_path(path: Vec) -> Self { - Self { - visibility: Visibility::default(), - path, - items: UseItems::List(Vec::new()), - span: Span::DUMMY, - } - } - } - #[test] fn test_reject_redefined_builtin_type() { let ty = TypeAlias::parse_from_str("type Ctx8 = u32") @@ -3352,6 +3564,24 @@ mod test { assert!(matches!(err.error(), Error::ReservedSimcKeyword)); } +} + +#[cfg(test)] +mod regular_parsing { + use super::*; + use crate::parse; + + impl UseDecl { + /// Creates a dummy `UseDecl` specifically for testing `DependencyMap` resolution. + pub fn dummy_path(path: Vec) -> Self { + Self { + visibility: Visibility::default(), + path, + items: UseItems::List(Vec::new()), + span: Span::DUMMY, + } + } + } #[test] fn test_double_colon() { @@ -3560,6 +3790,25 @@ fn main() { } } + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + let program = Program::parse_from_str(input).expect("program parses"); + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; + + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } + fn parse_item(input: &str) -> Item { let program = parse::Program::parse_from_str(input).expect("parsing should succeed"); program.items().first().expect("expected one item").clone() @@ -3612,3 +3861,469 @@ fn main() { } } } + +#[cfg(test)] +#[cfg(feature = "fmt")] +mod fmt_parsing { + use super::*; + + /// Parse `input`, appending any formatter diagnostics to `diagnostics`. + fn parse_with_diagnostics<'src>( + input: &'src str, + features: &UnstableFeatures, + diagnostics: &mut DiagnosticManager, + ) -> Option> { + Program::parse_with_errors_for_fmt(MAIN_MODULE, input, features, diagnostics) + } + + /// Parse `input` and return whether it was rejected and the collected error text. + fn parse_with(input: &str, features: &UnstableFeatures) -> (bool, String) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + let text = diagnostics.to_string(); + + (rejected, text) + } + + /// Parse `input` and return whether it was rejected and the collected error in `DiagnosticManager`. + fn parse_with_diagnosis(input: &str, features: &UnstableFeatures) -> (bool, DiagnosticManager) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + + (rejected, diagnostics) + } + + /// Parse `input` and return whether it was rejected and the collected error text together with `ParsedSource`. + fn parse_with_extended_out<'src>( + input: &'src str, + features: &UnstableFeatures, + ) -> (bool, String, Option>) { + let mut diagnostics = DiagnosticManager::new(); + let program = parse_with_diagnostics(input, features, &mut diagnostics); + + let rejected = program.is_none(); + let text = diagnostics.to_string(); + + (rejected, text, program) + } + + fn assert_formatter_matches_regular_parser(input: &str, features: &UnstableFeatures) { + let mut regular_diagnostics = DiagnosticManager::new(); + let regular = Program::parse_from_str_with_errors( + MAIN_MODULE, + input, + features, + &mut regular_diagnostics, + ); + + let mut formatting_diagnostics = DiagnosticManager::new(); + let formatting = parse_with_diagnostics(input, features, &mut formatting_diagnostics); + + assert_eq!( + formatting.as_ref().map(ParsedSource::program), + regular.as_ref(), + "formatter and regular parsers must produce the same AST for {input:?}" + ); + assert_eq!( + formatting_diagnostics.error_count(), + regular_diagnostics.error_count(), + "formatter and regular parsers must report the same number of errors for {input:?}, \n\ + fmt errors: \n\t '{formatting_diagnostics}, \n regular errors: \n\t '{regular_diagnostics}" + ); + } + + fn assert_lossless_source(input: &str, parsed: &ParsedSource<'_>) { + let prefix = parsed.prefix(); + assert_eq!(prefix.file_id, MAIN_MODULE); + + let mut reconstructed = prefix + .to_slice(input) + .expect("the directive prefix must be a valid source slice") + .to_owned(); + let mut cursor = prefix.end; + + for (_, span) in parsed.tokens() { + assert_eq!(span.file_id, MAIN_MODULE); + assert!( + span.start >= prefix.end, + "formatter tokens must not overlap the preserved directive prefix" + ); + assert_eq!( + span.start, cursor, + "formatter token spans must be contiguous and in source order" + ); + + let text = span + .to_slice(input) + .expect("formatter token spans must be valid source slices"); + assert!( + !text.is_empty(), + "the formatter token stream must not contain empty source slices" + ); + reconstructed.push_str(text); + cursor = span.end; + } + + assert_eq!(cursor, input.len(), "formatter tokens must reach EOF"); + assert_eq!(reconstructed, input, "formatter input must be lossless"); + } + + #[test] + fn test_double_colon() { + let input = "fn main() { let ab: u8 = <(u4, u4)> : :into((0b1011, 0b1101)); }"; + + let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + + assert!(rejected); + assert!(text.contains("Expected '::', found ':'")); + } + + #[test] + fn test_double_double_colon() { + let input = "fn main() { let pk: Pubkey = witnes::::PK; }"; + + let (rejected, text) = parse_with(input, &UnstableFeatures::all()); + + assert!(rejected); + assert!( + text.contains("Expected identifier, found ::"), + "the second :: should be reported as the error site" + ); + } + + #[test] + fn inverted_empty_span_from_token_gap_does_not_panic() { + // Fuzz-found (compile_text). + // + // The input is irreducible. The broken `fn` prefix forces the parser into delimiter recovery. + // The only path that requests an empty span at a lex-error gap and the + // NUL after `enum` creates that gap. + // Chumsky builds such spans as `next_token.start .. previous_token.end`, which is inverted + // across the gap and panicked the strict `Span` constructor. + // + // Simplifying any part (even NUL to a space) loses the crash. + let src = "fn`u({?\u{12}$0;;enum\0===lHf\u{15}"; + let (rejected, text) = parse_with(src, &UnstableFeatures::all()); + + assert!(rejected, "garbage input must be rejected"); + assert!(!text.is_empty()); + } + + #[test] + fn recovery_synchronizes_at_enum_declaration() { + // Discriminating setup: an invalid prefix followed by a malformed + // enum. Without `Token::Enum` in the synchronization set the whole + // enum is swallowed into the prefix's recovery span and only one + // error is reported; with it, the enum parses as its own item and + // its malformation is reported as a second error. + + let src = "let invalid = 1;\nenum Action { A B }\nfn main() {}"; + let (rejected, diagnostics) = parse_with_diagnosis(src, &UnstableFeatures::all()); + + assert!(rejected, "the invalid program must be rejected"); + assert_eq!( + 2, + diagnostics.error_count(), + "the invalid prefix and the malformed enum must each report an error" + ); + } + + #[test] + fn recovery_after_malformed_enum_reaches_next_item() { + let src = "enum Action { A B }\nfn main() {}"; + let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); + assert!(rejected, "a malformed enum must fail compilation"); + } + + #[test] + fn malformed_construct_containing_enum_keyword_reports_errors() { + // `enum` inside a badly malformed nested construct may be mistaken + // for an item boundary; compilation must still fail cleanly. + let src = "fn broken() { let x = (enum; }\nfn main() {}"; + let (rejected, _text) = parse_with(src, &UnstableFeatures::all()); + assert!(rejected, "a malformed construct must fail compilation"); + } + + #[test] + fn test_gated_syntax_is_rejected_without_features() { + // Real `use`/`mod` syntax: rejected under none() (naming the feature + a + // -Z hint), accepted under all(). Delete when `imports` stabilizes. + for input in [ + "use crate::foo::bar;\nfn main() { }", + "mod inner { }\nfn main() { }", + ] { + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + rejected, + "gated syntax must be rejected without features (if this feature was \ + just stabilized, delete this test):\n{input}" + ); + assert!( + error.contains("imports") && error.contains("-Z"), + "rejection should name the feature and suggest -Z, got:\n{error}" + ); + + let (rejected, error) = parse_with(input, &UnstableFeatures::all()); + assert!( + !rejected, + "the same syntax must parse with all features enabled:\n{error}" + ); + } + } + + #[test] + fn test_type_heavy_program_needs_no_features() { + // Types are traversed by `RequireFeature` but not gated, so a type-heavy, + // import-free program must parse with no features (no false-positive gates). + let input = r#"type Alias = u32; +fn pick(e: Either) -> u32 { + match e { + Left(a: u32) => a, + Right(b: u32) => b, + } +} +fn main() { + let casted: u32 = <(u16, u16)>::into((0xbeef, 0xbabe)); + let chosen: Alias = pick(Left(casted)); + assert!(jet::eq_32(chosen, chosen)); +} +"#; + // `parse_from_str_with_errors` returns `Some` only when no errors were + // collected, so a non-rejection already proves there were no gate errors. + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!( + !rejected, + "type-heavy but import-free program should parse with no features:\n{error}" + ); + } + + #[test] + fn test_syntax_error_does_not_produce_spurious_feature_gate_error() { + // The gate check skips error-recovered ASTs, so a program with + // both a syntax error and gated syntax reports only the syntax error. + let input = "use crate::foo;\nfn main( {"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "broken program must be rejected"); + assert!( + !error.contains("-Z"), + "syntax error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_lexer_error_does_not_produce_spurious_feature_gate_error() { + // Lexer-side companion to the case above: the stray `@` lexes with an + // error but recovers a cleanly-parsing `use` AST, and a program that + // didn't lex cleanly skips the gate check — so no spurious `-Z` hint. + let input = "use crate@::foo;\nfn main() { }"; + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + assert!(rejected, "program with a lexer error must be rejected"); + assert!( + !error.contains("-Z"), + "lexer error should not produce a spurious feature-gate hint, got:\n{error}" + ); + } + + #[test] + fn test_use_decl_display_round_trips_with_aliases() { + for input in [ + "use lib::A::foo;", + "use lib::A::foo as bar;", + "use lib::A::{foo, bar as baz};", + ] { + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::all()); + assert!(!rejected, "parsing works"); + assert_eq!(program.unwrap().program.to_string(), format!("{input}\n")); + } + } + + #[test] + fn statement_spans_exclude_terminating_semicolons() { + let input = "fn main() { let value: u8 = 0; value; }"; + + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::none()); + assert!(!rejected, "parsing works"); + let program = program.unwrap().program; + + let Item::Function(function) = &program.items()[0] else { + panic!("expected a function item"); + }; + let ExpressionInner::Block(statements, None) = function.body().inner() else { + panic!("expected a block containing only statements"); + }; + + assert_eq!( + statements[0].span().to_slice(input), + Some("let value: u8 = 0") + ); + assert_eq!(statements[1].span().to_slice(input), Some("value")); + } + + fn parse_item(input: &str) -> Item { + let (rejected, _, program) = parse_with_extended_out(input, &UnstableFeatures::all()); + assert!(!rejected, "parsing works"); + let program = program.unwrap().program; + + program.items().first().expect("expected one item").clone() + } + + #[test] + fn test_enum_declaration_basic() { + let item = parse_item("enum Path { Inherit, ColdSpend, RefreshSpend, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration, got {item:?}"); + }; + assert_eq!(decl.name().as_inner(), "Path"); + assert_eq!(decl.variants().len(), 3); + assert_eq!(decl.variants()[0].name().as_inner(), "Inherit"); + assert_eq!(decl.variants()[2].name().as_inner(), "RefreshSpend"); + } + + #[test] + fn test_enum_declaration_pub() { + let item = parse_item("pub enum Color { Red, Green, Blue, }"); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!(decl.visibility(), &Visibility::Public); + assert_eq!(decl.name().as_inner(), "Color"); + } + + #[test] + fn test_enum_declaration_display_round_trip() { + let input = "enum Path { Inherit, ColdSpend, RefreshSpend, }"; + let item = parse_item(input); + let Item::EnumDeclaration(decl) = item else { + panic!("expected EnumDeclaration"); + }; + assert_eq!( + decl.to_string(), + "enum Path { Inherit, ColdSpend, RefreshSpend, }" + ); + } + + #[test] + fn test_enum_declaration_reserved_name() { + for reserved in RESERVED_PATTERN_NAMES { + let input = format!("enum {reserved} {{ A, B, }}"); + let (rejected, error) = parse_with(&input, &UnstableFeatures::none()); + assert!(rejected, "enum name {reserved} is reserved"); + assert!( + error.contains("reserved"), + "error should say the name is reserved: {error}" + ); + } + } + + #[test] + fn formatting_parse_is_lossless_after_directive_prescan() { + let input = "// header\r\nsimc \"*\";\r\n\t/* block */ fn main() { // body\r\n\t}\r"; + + let (_rejected, _error, parsed) = parse_with_extended_out(input, &UnstableFeatures::none()); + let parsed = parsed.expect("formatting parse should succeed"); + + assert_eq!( + parsed.prefix().to_slice(input), + Some("// header\r\nsimc \"*\";") + ); + for trivia in [ + crate::lexer::TriviaKind::BlockComment, + crate::lexer::TriviaKind::LineComment, + crate::lexer::TriviaKind::Newline, + crate::lexer::TriviaKind::Whitespace, + ] { + assert!( + parsed.tokens().iter().any( + |(token, _)| matches!(token, FmtToken::Trivia(token_trivia) if token_trivia.kind() == trivia) + ), + "formatter token stream should retain {trivia:?}" + ); + } + assert_lossless_source(input, &parsed); + assert_eq!(parsed.program().items().len(), 1); + } + + #[test] + fn formatting_parse_matches_regular_parser_pipeline() { + let no_features = UnstableFeatures::none(); + let all_features = UnstableFeatures::all(); + + for (input, features) in [ + ( + "/* header */\r\nfn\tmain() { // comment\r\n}\r", + &no_features, + ), + ("// version\r\nsimc \"*\";\r\nfn main() {}", &no_features), + ("use crate::foo::bar;\nfn main() {}", &no_features), + ("use crate::foo::bar;\nfn main() {}", &all_features), + ("fn main( {", &no_features), + ("fn main() { @// comment }", &no_features), + ] { + assert_formatter_matches_regular_parser(input, features); + } + } + + #[test] + fn formatting_parse_ignores_preexisting_errors() { + let input = "fn main() {}"; + let mut diagnostics = DiagnosticManager::new(); + diagnostics.push(Diagnostic::global(Error::CannotParse { + msg: "an error from an earlier source file".to_owned(), + })); + let before = diagnostics.error_count(); + + let parsed = parse_with_diagnostics(input, &UnstableFeatures::none(), &mut diagnostics); + + assert!( + parsed.is_some(), + "valid source must still produce ParsedSource" + ); + assert_eq!(diagnostics.error_count(), before); + assert_eq!(diagnostics.diagnostics().len(), before); + } + + #[test] + fn formatting_parse_stops_after_invalid_directive() { + for (input, is_malformed) in [ + ("simc \"*\"\nfn broken( @", true), + ("simc \">99.0.0\"; @", false), + ] { + let (rejected, diagnostics) = parse_with_diagnosis(input, &UnstableFeatures::none()); + + assert!(rejected, "invalid directive must abort formatting parse"); + assert_eq!( + diagnostics.error_count(), + 1, + "the invalid body must not add diagnostics after prescan aborts" + ); + if is_malformed { + assert!(matches!( + diagnostics.diagnostics()[0].error(), + Error::MalformedSimcDirective + )); + } else { + assert!(matches!( + diagnostics.diagnostics()[0].error(), + Error::SimcVersionMismatch { .. } + )); + } + } + } + + #[test] + fn formatting_parse_reports_missing_match_arm_comma() { + let input = "fn main() { match true { false => () true => (), } }"; + + let (rejected, error) = parse_with(input, &UnstableFeatures::none()); + + assert!(rejected); + assert!( + error.contains("Missing ',' after a match arm that isn't block expression"), + "expected the missing-comma diagnostic, got {error:?}" + ); + } +} diff --git a/src/str.rs b/src/str.rs index 95b1e1f9..1fb95cc6 100644 --- a/src/str.rs +++ b/src/str.rs @@ -254,7 +254,7 @@ impl<'a> arbitrary::Arbitrary<'a> for AliasName { } } -/// A string of decimal digits. +/// A string of decimal digits, optionally separated by underscores (requires the `"fmt"` feature). #[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] pub struct Decimal(Arc); @@ -273,7 +273,7 @@ impl<'a> arbitrary::Arbitrary<'a> for Decimal { } } -/// A string of binary digits. +/// A string of binary digits, optionally separated by underscores (requires the `"fmt"` feature). #[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] pub struct Binary(Arc); @@ -293,12 +293,26 @@ impl<'a> arbitrary::Arbitrary<'a> for Binary { } } -/// A string of hexadecimal digits. +/// A string of hexadecimal digits, optionally separated by underscores (requires the `"fmt"` feature). #[derive(Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] pub struct Hexadecimal(Arc); wrapped_string!(Hexadecimal, "hexadecimal string"); +pub(crate) mod underscore_parsing { + use std::borrow::Cow; + + /// Remove digit separators from a numeric literal while avoiding an allocation + /// when the literal contains no separators. + pub(crate) fn strip_digit_separators(input: &str) -> Cow<'_, str> { + if input.contains('_') { + Cow::Owned(input.replace('_', "")) + } else { + Cow::Borrowed(input) + } + } +} + #[cfg(feature = "arbitrary")] impl<'a> arbitrary::Arbitrary<'a> for Hexadecimal { fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { diff --git a/src/types.rs b/src/types.rs index fefa298a..b2d67f67 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1145,13 +1145,13 @@ impl TreeLike for StructuralType { impl fmt::Debug for StructuralType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) + write!(f, "{}", self.0) } } impl fmt::Display for StructuralType { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) + write!(f, "{}", self.0) } } diff --git a/src/value.rs b/src/value.rs index 49ebbc5a..2dd1ca22 100644 --- a/src/value.rs +++ b/src/value.rs @@ -122,7 +122,9 @@ impl UIntValue { /// Create an integer from a `decimal` string and type. pub fn parse_decimal(decimal: &Decimal, ty: UIntType) -> Result { - let s = decimal.as_inner(); + let digits = crate::str::underscore_parsing::strip_digit_separators(decimal.as_inner()); + + let s = digits.as_ref(); match ty { UIntType::U1 => s.parse::().map_err(Error::from).and_then(Self::u1), UIntType::U2 => s.parse::().map_err(Error::from).and_then(Self::u2), @@ -138,7 +140,9 @@ impl UIntValue { /// Create an integer from a `binary` string and type. pub fn parse_binary(binary: &Binary, ty: UIntType) -> Result { - let s = binary.as_inner(); + let digits = crate::str::underscore_parsing::strip_digit_separators(binary.as_inner()); + + let s = digits.as_ref(); let bit_len = Pow2Usize::new(s.len()).ok_or(Error::BitStringPow2 { len: s.len() })?; let bit_ty = UIntType::from_bit_width(bit_len).ok_or(Error::BitStringPow2 { len: s.len() })?; @@ -655,7 +659,10 @@ impl Value { TypeInner::Array(inner, len) if inner.as_integer() == Some(UIntType::U8) => *len, _ => return Err(Error::ExpressionUnexpectedType { ty: ty.clone() }), }; - let s = hexadecimal.as_inner(); + + let digits = crate::str::underscore_parsing::strip_digit_separators(hexadecimal.as_inner()); + + let s = digits.as_ref(); if s.len() % 2 != 0 || s.len() != expected_byte_len * 2 { return Err(Error::ExpressionUnexpectedType { ty: ty.clone() }); } @@ -981,13 +988,13 @@ impl TreeLike for StructuralValue { impl fmt::Debug for StructuralValue { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) + write!(f, "{}", self.0) } } impl fmt::Display for StructuralValue { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", &self.0) + write!(f, "{}", self.0) } } diff --git a/tests/lex_lossless_coverage.rs b/tests/lex_lossless_coverage.rs new file mode 100644 index 00000000..954ae5e2 --- /dev/null +++ b/tests/lex_lossless_coverage.rs @@ -0,0 +1,103 @@ +#[cfg(feature = "fmt")] +mod lossless_test { + use simplicityhl::lexer::{lex_lossless, FmtTokens}; + + fn get_input(input_file: &str) -> String { + use std::fs; + use std::path::Path; + + let tests_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples"); + let input_path = tests_dir.join(input_file); + fs::read_to_string(&input_path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", input_path.display())) + } + + fn assert_lossless_fixture(input_file: &str) { + const DEFAULT_FILE_ID: usize = 0; + + let original_input = get_input(input_file); + + let (tokens, diagnostics) = lex_lossless(DEFAULT_FILE_ID, &original_input, 0); + assert!( + diagnostics.is_empty(), + "Diagnostic errors aren't empty: {diagnostics:?}" + ); + let tokens = tokens.expect("lossless lexing succeeds"); + let restored_input = restore_input_by_tokens(&tokens); + assert_eq!( + restored_input, original_input, + "lossless tokens must reproduce the input" + ); + } + + fn restore_input_by_tokens(tokens: &FmtTokens<'_>) -> String { + let mut buf = String::new(); + + let mut prev_span_end = None; + for (t, s) in tokens { + match prev_span_end { + None => { + let _ = prev_span_end.insert(s.end); + } + Some(x) => { + assert_eq!(x, s.start); + let _ = prev_span_end.insert(s.end); + } + } + buf.push_str(&t.to_string()); + } + buf + } + + macro_rules! test_lex_coverage { + ($($lossless_name:ident: $input:literal,)+) => { + $( + #[test] + fn $lossless_name() { + assert_lossless_fixture($input); + } + )+ + }; +} + + test_lex_coverage! { + array_fold_2n_lossless: "array_fold_2n.simf", + array_fold_lossless: "array_fold.simf", + cat_lossless: "cat.simf", + ctv_lossless: "ctv.simf", + escrow_with_delay_lossless: "escrow_with_delay.simf", + hash_loop_lossless: "hash_loop.simf", + hodl_vault_lossless: "hodl_vault.simf", + htlc_lossless: "htlc.simf", + last_will_lossless: "last_will.simf", + modules_lossless: "modules.simf", + non_interactive_fee_bump_lossless: "non_interactive_fee_bump.simf", + p2ms_lossless: "p2ms.simf", + p2pk_lossless: "p2pk.simf", + p2pkh_lossless: "p2pkh.simf", + pattern_matching_lossless: "pattern_matching.simf", + presigned_vault_lossless: "presigned_vault.simf", + reveal_collision_lossless: "reveal_collision.simf", + reveal_fix_point_lossless: "reveal_fix_point.simf", + sighash_all_anyonecanpay_lossless: "sighash_all_anyonecanpay.simf", + sighash_all_anyprevout_lossless: "sighash_all_anyprevout.simf", + sighash_all_anyprevoutanyscript_lossless: "sighash_all_anyprevoutanyscript.simf", + sighash_none_lossless: "sighash_none.simf", + sighash_single_lossless: "sighash_single.simf", + transfer_with_timeout_lossless: "transfer_with_timeout.simf", + local_crate_main_lossless: "local_crate/main.simf", + local_crate_math_lossless: "local_crate/math.simf", + simple_multidep_hashes_lossless: "simple_multidep/crypto/hashes.simf", + simple_multidep_arithmetic_lossless: "simple_multidep/math/arithmetic.simf", + simple_multidep_flattened_lossless: "simple_multidep/flattened.simf", + simple_multidep_main_lossless: "simple_multidep/main.simf", + single_dep_flattened_lossless: "single_dep/flattened.simf", + single_dep_main_lossless: "single_dep/main.simf", + single_dep_funcs_lossless: "single_dep/temp/funcs.simf", + single_dep_utils_lossless: "single_dep/temp/constants/utils.simf", + multiple_deps_main_lossless: "multiple_deps/main.simf", + multiple_deps_flattened_lossless: "multiple_deps/flattened.simf", + multiple_deps_simple_op_lossless: "multiple_deps/math/simple_op.simf", + multiple_deps_build_root_lossless: "multiple_deps/merkle/build_root.simf", + } +}