diff --git a/crates/hir-def/src/body/lower.rs b/crates/hir-def/src/body/lower.rs index e8303ec40f74..84662aa27782 100644 --- a/crates/hir-def/src/body/lower.rs +++ b/crates/hir-def/src/body/lower.rs @@ -972,7 +972,7 @@ impl From for Literal { } } LiteralKind::FloatNumber(lit) => { - let ty = lit.suffix().and_then(BuiltinFloat::from_suffix); + let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s)); Literal::Float(Default::default(), ty) } LiteralKind::ByteString(bs) => { diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 8d2352f06ede..687c54974823 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -4,10 +4,7 @@ use base_db::{AnchoredPath, Edition, FileId}; use cfg::CfgExpr; use either::Either; use mbe::{parse_exprs_with_sep, parse_to_token_tree}; -use syntax::{ - ast::{self, AstToken}, - SmolStr, -}; +use syntax::{ast, SmolStr}; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; @@ -358,14 +355,7 @@ fn unreachable_expand( } fn unquote_str(lit: &tt::Literal) -> Option { - let lit = ast::make::tokens::literal(&lit.to_string()); - let token = ast::String::cast(lit)?; - token.value().map(|it| it.into_owned()) -} - -fn unquote_byte_string(lit: &tt::Literal) -> Option> { - let lit = ast::make::tokens::literal(&lit.to_string()); - let token = ast::ByteString::cast(lit)?; + let token = ast::make::literal(&lit.to_string()).as_string()?; token.value().map(|it| it.into_owned()) } @@ -442,12 +432,16 @@ fn concat_bytes_expand( for (i, t) in tt.token_trees.iter().enumerate() { match t { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - let token = ast::make::tokens::literal(&lit.to_string()); - match token.kind() { - syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()), - syntax::SyntaxKind::BYTE_STRING => { - let components = unquote_byte_string(lit).unwrap_or_else(Vec::new); - components.into_iter().for_each(|x| bytes.push(x.to_string())); + let lit = ast::make::literal(&lit.to_string()); + match lit.kind() { + ast::LiteralKind::ByteString(s) => { + s.value() + .unwrap_or_default() + .into_iter() + .for_each(|x| bytes.push(x.to_string())); + } + ast::LiteralKind::Byte(_) => { + bytes.push(lit.to_string()); } _ => { err.get_or_insert(mbe::ExpandError::UnexpectedToken.into()); @@ -481,10 +475,10 @@ fn concat_bytes_expand_subtree( for (ti, tt) in tree.token_trees.iter().enumerate() { match tt { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - let lit = ast::make::tokens::literal(&lit.to_string()); + let lit = ast::make::literal(&lit.to_string()); match lit.kind() { - syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => { - bytes.push(lit.text().to_string()) + ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => { + bytes.push(lit.to_string()); } _ => { return Err(mbe::ExpandError::UnexpectedToken.into()); diff --git a/crates/hir-ty/src/tests/simple.rs b/crates/hir-ty/src/tests/simple.rs index d4d61c2167cd..4b8a7e782de6 100644 --- a/crates/hir-ty/src/tests/simple.rs +++ b/crates/hir-ty/src/tests/simple.rs @@ -2733,3 +2733,14 @@ fn f() { "#, ); } + +#[test] +fn nested_tuple_index() { + check_no_mismatches( + r#" +fn main() { + let fld: i32 = ((0,),).0.0; +} +"#, + ); +} diff --git a/crates/ide-completion/src/completions/dot.rs b/crates/ide-completion/src/completions/dot.rs index 01a5f856c1b1..c660e8f8fa32 100644 --- a/crates/ide-completion/src/completions/dot.rs +++ b/crates/ide-completion/src/completions/dot.rs @@ -785,4 +785,24 @@ fn main() { ", ) } + + #[test] + fn tuple_index_completion() { + check( + r#" +struct I; +impl I { + fn i_method(&self) {} +} +struct S((), I); + +fn f(s: S) { + s.1.$0 +} +"#, + expect![[r#" + me i_method() fn(&self) + "#]], + ); + } } diff --git a/crates/ide-completion/src/completions/postfix.rs b/crates/ide-completion/src/completions/postfix.rs index 7b936f0ff777..cc2a4437000d 100644 --- a/crates/ide-completion/src/completions/postfix.rs +++ b/crates/ide-completion/src/completions/postfix.rs @@ -5,7 +5,7 @@ mod format_like; use hir::{Documentation, HasAttrs}; use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap}; use syntax::{ - ast::{self, AstNode, AstToken}, + ast::{self, AstNode, LiteralKind}, SyntaxKind::{EXPR_STMT, STMT_LIST}, TextRange, TextSize, }; @@ -191,7 +191,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) { } if let ast::Expr::Literal(literal) = dot_receiver.clone() { - if let Some(literal_text) = ast::String::cast(literal.token()) { + if let LiteralKind::String(literal_text) = literal.kind() { add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text); } } diff --git a/crates/ide-completion/src/patterns.rs b/crates/ide-completion/src/patterns.rs index 6fdec783854f..c1db8ce4a6cd 100644 --- a/crates/ide-completion/src/patterns.rs +++ b/crates/ide-completion/src/patterns.rs @@ -230,7 +230,7 @@ pub(crate) fn determine_location( let receiver = find_in_original_file(it.expr(), original_file); let receiver_is_ambiguous_float_literal = if let Some(ast::Expr::Literal(l)) = &receiver { match l.kind() { - ast::LiteralKind::FloatNumber { .. } => l.token().text().ends_with('.'), + ast::LiteralKind::FloatNumber { .. } => l.to_string().ends_with('.'), _ => false, } } else { diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index e04fd5a7c7f2..bafbf6d9cc88 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -30,7 +30,15 @@ pub(super) fn token(sema: &Semantics, token: SyntaxToken) -> Optio INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { SymbolKind::Field.into() } - INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(), + INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 + | FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(), + DOT if matches!( + token.prev_token().map(|n| n.kind()), + Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2) + ) => + { + HlTag::NumericLiteral.into() + } BYTE => HlTag::ByteLiteral.into(), CHAR => HlTag::CharLiteral.into(), IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index 60bc29012111..d7d6c79e3d34 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -119,13 +119,13 @@ println!("Hello {:05}!", 5); println!("Hello {:05}!", -5); println!("{:#010x}!", 27); - println!("Hello {0} is {1:.5}", "x", 0.01); - println!("Hello {1} is {2:.0$}", 5, "x", 0.01); - println!("Hello {0} is {2:.1$}", "x", 5, 0.01); - println!("Hello {} is {:.*}", "x", 5, 0.01); - println!("Hello {} is {2:.*}", "x", 5, 0.01); - println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); - println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); + println!("Hello {0} is {1:.5}", "x", 0.01); + println!("Hello {1} is {2:.0$}", 5, "x", 0.01); + println!("Hello {0} is {2:.1$}", "x", 5, 0.01); + println!("Hello {} is {:.*}", "x", 5, 0.01); + println!("Hello {} is {2:.*}", "x", 5, 0.01); + println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); + println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); println!("{}, `{name:.*}` has 3 characters", "Hello", 3, name="1234.56"); println!("{}, `{name:>8.*}` has 3 right-aligned characters", "Hello", 3, name="1234.56"); println!("Hello {{}}"); diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 21a0aa4284a5..79fd1250e0e3 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -260,6 +260,31 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { IDENT => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), + FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => { + // Reassemble a split-up float token. + let mut range = range; + let mut text = token.to_text(conv).to_string(); + if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 { + let (dot, dot_range) = conv.bump().unwrap(); + text += &*dot.to_text(conv); + range = TextRange::new(range.start(), dot_range.end()); + + if kind == FLOAT_NUMBER_START_2 { + let (tail, tail_range) = conv.bump().unwrap(); + text += &*tail.to_text(conv); + range = TextRange::new(range.start(), tail_range.end()); + } + } + + result.push( + tt::Leaf::from(tt::Literal { + id: conv.id_alloc().alloc(range, synth_id), + text: text.into(), + }) + .into(), + ); + continue; + } k if k.is_literal() => make_leaf!(Literal), LIFETIME_IDENT => { let char_unit = TextSize::of('\''); diff --git a/crates/mbe/src/to_parser_input.rs b/crates/mbe/src/to_parser_input.rs index 6faa147218e1..958f6433206a 100644 --- a/crates/mbe/src/to_parser_input.rs +++ b/crates/mbe/src/to_parser_input.rs @@ -35,15 +35,13 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input { let is_negated = lit.text.starts_with('-'); let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; - let kind = parser::LexedStr::single_token(inner_text) - .map(|(kind, _error)| kind) - .filter(|kind| { - kind.is_literal() - && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER)) - }) - .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit)); - - res.push(kind); + let lexed_str = parser::LexedStr::new(inner_text); + if lexed_str.is_empty() { + panic!("failed to convert literal: {:?}", lit); + } + for i in 0..lexed_str.len() { + res.push(lexed_str.kind(i)); + } } tt::Leaf::Ident(ident) => match ident.text.as_ref() { "_" => res.push(T![_]), diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 4efbf9a606e1..92f330253874 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -39,6 +39,7 @@ mod generic_params; mod types; use crate::{ + grammar::expressions::FLOAT_LITERAL_FIRST, parser::{CompletedMarker, Marker, Parser}, SyntaxKind::{self, *}, TokenSet, T, @@ -318,9 +319,15 @@ fn name_ref(p: &mut Parser) { } fn name_ref_or_index(p: &mut Parser) { - assert!(p.at(IDENT) || p.at(INT_NUMBER)); + assert!( + p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) + ); let m = p.start(); - p.bump_any(); + if p.at_ts(FLOAT_LITERAL_FIRST) { + p.bump_remap(FLOAT_NUMBER_PART); + } else { + p.bump_any(); + } m.complete(p, NAME_REF); } diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index b063c73a9d68..290083b34349 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -3,7 +3,7 @@ mod atom; use super::*; pub(crate) use self::atom::{block_expr, match_arm_list}; -pub(super) use self::atom::{literal, LITERAL_FIRST}; +pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; #[derive(PartialEq, Eq)] pub(super) enum Semicolon { @@ -452,6 +452,9 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo(); // y.bar::(1, 2,); +// +// 0e0.sin(); +// 0e0f32.sin(); // } fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); @@ -469,17 +472,16 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo; // x.0.bar; +// x.0. bar; +// x.0.1; // x.0(); // } fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.])); let m = lhs.precede(p); p.bump(T![.]); - if p.at(IDENT) || p.at(INT_NUMBER) { + if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { name_ref_or_index(p); - } else if p.at(FLOAT_NUMBER) { - // FIXME: How to recover and instead parse INT + T![.]? - p.bump_any(); } else { p.error("expected field name or number"); } diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 10e5d897e078..5ab148ff2880 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -17,22 +17,58 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[ T![true], T![false], INT_NUMBER, - FLOAT_NUMBER, + FLOAT_NUMBER_START_0, + FLOAT_NUMBER_START_1, + FLOAT_NUMBER_START_2, BYTE, CHAR, STRING, BYTE_STRING, ]); +pub(crate) const FLOAT_LITERAL_FIRST: TokenSet = + TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]); + pub(crate) fn literal(p: &mut Parser) -> Option { if !p.at_ts(LITERAL_FIRST) { return None; } let m = p.start(); - p.bump_any(); + if p.at_ts(FLOAT_LITERAL_FIRST) { + float_literal(p); + } else { + // Everything else is just one token. + p.bump_any(); + } Some(m.complete(p, LITERAL)) } +// test float_literal +// fn f() { +// 0.0; +// 1.; +// 0e0; +// 0e0f32; +// 1.23f64; +// } +pub(crate) fn float_literal(p: &mut Parser) { + // Floats can be up to 3 tokens. The first token indicates how many there are. + let f = p.start(); + if p.at(FLOAT_NUMBER_START_0) { + p.bump(FLOAT_NUMBER_START_0); + } else if p.at(FLOAT_NUMBER_START_1) { + p.bump(FLOAT_NUMBER_START_1); + p.bump(DOT); + } else if p.at(FLOAT_NUMBER_START_2) { + p.bump(FLOAT_NUMBER_START_2); + p.bump(DOT); + p.bump(FLOAT_NUMBER_PART); + } else { + unreachable!(); + } + f.complete(p, FLOAT_LITERAL); +} + // E.g. for after the break in `if break {}`, this should not match pub(super) const ATOM_EXPR_FIRST: TokenSet = LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[ diff --git a/crates/parser/src/grammar/patterns.rs b/crates/parser/src/grammar/patterns.rs index 1f622b32e5b1..c16bd8d0c7af 100644 --- a/crates/parser/src/grammar/patterns.rs +++ b/crates/parser/src/grammar/patterns.rs @@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option { } fn is_literal_pat_start(p: &Parser) -> bool { - p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER) + p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART) || p.at_ts(expressions::LITERAL_FIRST) } diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index f4b9988eacb0..9887960871f8 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -177,7 +177,7 @@ impl<'a> Converter<'a> { rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::Literal { kind, .. } => { - self.extend_literal(token_text.len(), kind); + self.extend_literal(token_text, kind); return; } @@ -223,7 +223,7 @@ impl<'a> Converter<'a> { self.push(syntax_kind, token_text.len(), err); } - fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) { + fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) { let mut err = ""; let syntax_kind = match *kind { @@ -237,7 +237,27 @@ impl<'a> Converter<'a> { if empty_exponent { err = "Missing digits after the exponent symbol"; } - FLOAT_NUMBER + + // In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0` + // is lexed as a float, we split floats that contain a `.` into 3 tokens. + // To ensure that later stages can always reconstruct the token correctly, the first + // token in the sequence indicates the number of following tokens that are part of + // the float literal. + if let Some((before, after)) = token_text.split_once('.') { + let err = if err.is_empty() { None } else { Some(err) }; + + assert!(!before.is_empty()); + let tok = + if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 }; + self.push(tok, before.len(), None); + self.push(DOT, 1, None); + if !after.is_empty() { + self.push(FLOAT_NUMBER_PART, after.len(), err); + } + return; + } + + FLOAT_NUMBER_START_0 } rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { @@ -295,6 +315,6 @@ impl<'a> Converter<'a> { }; let err = if err.is_empty() { None } else { Some(err) }; - self.push(syntax_kind, len, err); + self.push(syntax_kind, token_text.len(), err); } } diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 628fa745e752..5ddcd6ad6a42 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -110,7 +110,10 @@ pub enum SyntaxKind { RAW_KW, MACRO_RULES_KW, INT_NUMBER, - FLOAT_NUMBER, + FLOAT_NUMBER_START_0, + FLOAT_NUMBER_START_1, + FLOAT_NUMBER_START_2, + FLOAT_NUMBER_PART, CHAR, BYTE, STRING, @@ -227,6 +230,7 @@ pub enum SyntaxKind { PATH, PATH_SEGMENT, LITERAL, + FLOAT_LITERAL, RENAME, VISIBILITY, WHERE_CLAUSE, @@ -286,7 +290,8 @@ impl SyntaxKind { } pub fn is_literal(self) -> bool { match self { - INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING => true, + INT_NUMBER | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 + | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, _ => false, } } @@ -386,5 +391,5 @@ impl SyntaxKind { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; [float_number_start_0] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_0 } ; [float_number_start_1] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_1 } ; [float_number_start_2] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_2 } ; } pub use T; diff --git a/crates/parser/test_data/lexer/err/empty_exponent.rast b/crates/parser/test_data/lexer/err/empty_exponent.rast index af03d73ced9a..73de4cac2435 100644 --- a/crates/parser/test_data/lexer/err/empty_exponent.rast +++ b/crates/parser/test_data/lexer/err/empty_exponent.rast @@ -1,14 +1,14 @@ -FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" INT_NUMBER "42" DOT "." @@ -30,19 +30,35 @@ DOT "." IDENT "E" MINUS "-" WHITESPACE "\n\n" -FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_2 "42" +DOT "." +FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_2 "42" +DOT "." +FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_2 "42" +DOT "." +FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_2 "42" +DOT "." +FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_2 "42" +DOT "." +FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_2 "42" +DOT "." +FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_2 "42" +DOT "." +FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_2 "42" +DOT "." +FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" diff --git a/crates/parser/test_data/lexer/ok/numbers.rast b/crates/parser/test_data/lexer/ok/numbers.rast index 8d13c3f61068..428bdf8a1f9a 100644 --- a/crates/parser/test_data/lexer/ok/numbers.rast +++ b/crates/parser/test_data/lexer/ok/numbers.rast @@ -4,7 +4,8 @@ INT_NUMBER "00" WHITESPACE " " INT_NUMBER "0_" WHITESPACE " " -FLOAT_NUMBER "0." +FLOAT_NUMBER_START_1 "0" +DOT "." WHITESPACE " " INT_NUMBER "0z" WHITESPACE "\n" @@ -20,11 +21,13 @@ INT_NUMBER "001279" WHITESPACE " " INT_NUMBER "0_1279" WHITESPACE " " -FLOAT_NUMBER "0.1279" +FLOAT_NUMBER_START_2 "0" +DOT "." +FLOAT_NUMBER_PART "1279" WHITESPACE " " -FLOAT_NUMBER "0e1279" +FLOAT_NUMBER_START_0 "0e1279" WHITESPACE " " -FLOAT_NUMBER "0E1279" +FLOAT_NUMBER_START_0 "0E1279" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -37,7 +40,7 @@ IDENT "foo" L_PAREN "(" R_PAREN ")" WHITESPACE "\n" -FLOAT_NUMBER "0e+1" +FLOAT_NUMBER_START_0 "0e+1" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -45,13 +48,19 @@ IDENT "e" PLUS "+" INT_NUMBER "1" WHITESPACE "\n" -FLOAT_NUMBER "0.0E-2" +FLOAT_NUMBER_START_2 "0" +DOT "." +FLOAT_NUMBER_PART "0E-2" WHITESPACE "\n" -FLOAT_NUMBER "0___0.10000____0000e+111__" +FLOAT_NUMBER_START_2 "0___0" +DOT "." +FLOAT_NUMBER_PART "10000____0000e+111__" WHITESPACE "\n" INT_NUMBER "1i64" WHITESPACE " " -FLOAT_NUMBER "92.0f32" +FLOAT_NUMBER_START_2 "92" +DOT "." +FLOAT_NUMBER_PART "0f32" WHITESPACE " " INT_NUMBER "11__s" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast index 4064a7a1ff2d..70cd030da1ec 100644 --- a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast +++ b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast @@ -32,7 +32,9 @@ SOURCE_FILE INT_NUMBER "1" COMMA "," WHITESPACE " " - FLOAT_NUMBER "2.0" + FLOAT_NUMBER_START_2 "2" + DOT "." + FLOAT_NUMBER_PART "0" WHITESPACE "\n " R_CURLY "}" WHITESPACE " " diff --git a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast index 8498724b9ef0..a1efb3a9fb29 100644 --- a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast @@ -40,6 +40,39 @@ SOURCE_FILE IDENT "bar" SEMICOLON ";" WHITESPACE "\n " + EXPR_STMT + FIELD_EXPR + FIELD_EXPR + PATH_EXPR + PATH + PATH_SEGMENT + NAME_REF + IDENT "x" + DOT "." + NAME_REF + FLOAT_NUMBER_PART "0" + DOT "." + WHITESPACE " " + NAME_REF + IDENT "bar" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + FIELD_EXPR + FIELD_EXPR + PATH_EXPR + PATH + PATH_SEGMENT + NAME_REF + IDENT "x" + DOT "." + NAME_REF + FLOAT_NUMBER_PART "0" + DOT "." + NAME_REF + FLOAT_NUMBER_PART "1" + SEMICOLON ";" + WHITESPACE "\n " EXPR_STMT CALL_EXPR FIELD_EXPR diff --git a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs index b8da2ddc3094..551b1ecaf021 100644 --- a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs @@ -1,5 +1,7 @@ fn foo() { x.foo; x.0.bar; + x.0. bar; + x.0.1; x.0(); } diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast index 403c265ea35b..b3236976b984 100644 --- a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast +++ b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast @@ -57,7 +57,10 @@ SOURCE_FILE EQ "=" WHITESPACE " " LITERAL - FLOAT_NUMBER "2.0" + FLOAT_LITERAL + FLOAT_NUMBER_START_2 "2" + DOT "." + FLOAT_NUMBER_PART "0" SEMICOLON ";" WHITESPACE "\n " LET_STMT diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast index dcbcfe1231e6..69f1055b7e36 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast @@ -58,6 +58,32 @@ SOURCE_FILE COMMA "," R_PAREN ")" SEMICOLON ";" + WHITESPACE "\n\n " + EXPR_STMT + METHOD_CALL_EXPR + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_START_0 "0e0" + DOT "." + NAME_REF + IDENT "sin" + ARG_LIST + L_PAREN "(" + R_PAREN ")" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + METHOD_CALL_EXPR + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_START_0 "0e0f32" + DOT "." + NAME_REF + IDENT "sin" + ARG_LIST + L_PAREN "(" + R_PAREN ")" + SEMICOLON ";" WHITESPACE "\n" R_CURLY "}" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs index 1a3aa35ae8e7..3e5d464e238f 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs @@ -1,4 +1,7 @@ fn foo() { x.foo(); y.bar::(1, 2,); + + 0e0.sin(); + 0e0f32.sin(); } diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast new file mode 100644 index 000000000000..df4fb6eb417b --- /dev/null +++ b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast @@ -0,0 +1,51 @@ +SOURCE_FILE + FN + FN_KW "fn" + WHITESPACE " " + NAME + IDENT "f" + PARAM_LIST + L_PAREN "(" + R_PAREN ")" + WHITESPACE " " + BLOCK_EXPR + STMT_LIST + L_CURLY "{" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_START_2 "0" + DOT "." + FLOAT_NUMBER_PART "0" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_START_1 "1" + DOT "." + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_START_0 "0e0" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_START_0 "0e0f32" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_START_2 "1" + DOT "." + FLOAT_NUMBER_PART "23f64" + SEMICOLON ";" + WHITESPACE "\n" + R_CURLY "}" + WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs new file mode 100644 index 000000000000..0d51ec1252fa --- /dev/null +++ b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs @@ -0,0 +1,7 @@ +fn f() { + 0.0; + 1.; + 0e0; + 0e0f32; + 1.23f64; +} diff --git a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast index 55ce31275fba..b2f66e2f1e7c 100644 --- a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast +++ b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast @@ -19,7 +19,10 @@ SOURCE_FILE CAST_EXPR METHOD_CALL_EXPR LITERAL - FLOAT_NUMBER "1.0f32" + FLOAT_LITERAL + FLOAT_NUMBER_START_2 "1" + DOT "." + FLOAT_NUMBER_PART "0f32" DOT "." NAME_REF IDENT "floor" @@ -40,7 +43,10 @@ SOURCE_FILE CAST_EXPR METHOD_CALL_EXPR LITERAL - FLOAT_NUMBER "1.0f32" + FLOAT_LITERAL + FLOAT_NUMBER_START_2 "1" + DOT "." + FLOAT_NUMBER_PART "0f32" DOT "." NAME_REF IDENT "floor" diff --git a/crates/syntax/rust.ungram b/crates/syntax/rust.ungram index 62aa47839942..85270dee6244 100644 --- a/crates/syntax/rust.ungram +++ b/crates/syntax/rust.ungram @@ -365,13 +365,20 @@ MacroExpr = Literal = Attr* value:( - 'int_number' | 'float_number' + 'int_number' | FloatLiteral | 'string' | 'raw_string' | 'byte_string' | 'raw_byte_string' | 'true' | 'false' | 'char' | 'byte' ) +FloatLiteral = + 'float_number_start_0'? + 'float_number_start_1'? + 'float_number_start_2'? + '.'? + 'float_number_part'? + PathExpr = Attr* Path diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index 17785152bc59..11d81d47783f 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -8,7 +8,7 @@ use crate::{ operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp}, support, AstChildren, AstNode, }, - AstToken, + AstToken, SyntaxElement, SyntaxKind::*, SyntaxNode, SyntaxToken, T, }; @@ -282,30 +282,32 @@ pub enum LiteralKind { String(ast::String), ByteString(ast::ByteString), IntNumber(ast::IntNumber), - FloatNumber(ast::FloatNumber), + FloatNumber(ast::FloatLiteral), Char(ast::Char), Byte(ast::Byte), Bool(bool), } impl ast::Literal { - pub fn token(&self) -> SyntaxToken { + pub fn value(&self) -> SyntaxElement { self.syntax() .children_with_tokens() .find(|e| e.kind() != ATTR && !e.kind().is_trivia()) - .and_then(|e| e.into_token()) .unwrap() } - pub fn kind(&self) -> LiteralKind { - let token = self.token(); + let token = match self.value() { + rowan::NodeOrToken::Node(node) => { + return LiteralKind::FloatNumber( + ast::FloatLiteral::cast(node).expect("unreachable"), + ); + } + rowan::NodeOrToken::Token(token) => token, + }; if let Some(t) = ast::IntNumber::cast(token.clone()) { return LiteralKind::IntNumber(t); } - if let Some(t) = ast::FloatNumber::cast(token.clone()) { - return LiteralKind::FloatNumber(t); - } if let Some(t) = ast::String::cast(token.clone()) { return LiteralKind::String(t); } @@ -325,6 +327,26 @@ impl ast::Literal { _ => unreachable!(), } } + + pub fn as_string(&self) -> Option { + match self.kind() { + LiteralKind::String(it) => Some(it), + _ => None, + } + } + + pub fn as_byte_string(&self) -> Option { + match self.kind() { + LiteralKind::ByteString(it) => Some(it), + _ => None, + } + } +} + +impl ast::FloatLiteral { + pub fn suffix(&self) -> Option { + ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) + } } pub enum BlockModifier { @@ -364,7 +386,7 @@ impl ast::BlockExpr { fn test_literal_with_attr() { let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#); let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap(); - assert_eq!(lit.token().text(), r#""Hello""#); + assert_eq!(lit.value().to_string(), r#""Hello""#); } impl ast::RecordExprField { diff --git a/crates/syntax/src/ast/generated/nodes.rs b/crates/syntax/src/ast/generated/nodes.rs index cf90ba64cff1..9d5af8e63ca8 100644 --- a/crates/syntax/src/ast/generated/nodes.rs +++ b/crates/syntax/src/ast/generated/nodes.rs @@ -1085,6 +1085,26 @@ impl UnderscoreExpr { pub fn underscore_token(&self) -> Option { support::token(&self.syntax, T![_]) } } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FloatLiteral { + pub(crate) syntax: SyntaxNode, +} +impl FloatLiteral { + pub fn float_number_start_0_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_0]) + } + pub fn float_number_start_1_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_1]) + } + pub fn float_number_start_2_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_2]) + } + pub fn dot_token(&self) -> Option { support::token(&self.syntax, T![.]) } + pub fn float_number_part_token(&self) -> Option { + support::token(&self.syntax, T![float_number_part]) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct StmtList { pub(crate) syntax: SyntaxNode, @@ -2719,6 +2739,17 @@ impl AstNode for UnderscoreExpr { } fn syntax(&self) -> &SyntaxNode { &self.syntax } } +impl AstNode for FloatLiteral { + fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL } + fn cast(syntax: SyntaxNode) -> Option { + if Self::can_cast(syntax.kind()) { + Some(Self { syntax }) + } else { + None + } + } + fn syntax(&self) -> &SyntaxNode { &self.syntax } +} impl AstNode for StmtList { fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST } fn cast(syntax: SyntaxNode) -> Option { @@ -4608,6 +4639,11 @@ impl std::fmt::Display for UnderscoreExpr { std::fmt::Display::fmt(self.syntax(), f) } } +impl std::fmt::Display for FloatLiteral { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self.syntax(), f) + } +} impl std::fmt::Display for StmtList { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs index a3209c5abd23..1e1a55e32693 100644 --- a/crates/syntax/src/ast/generated/tokens.rs +++ b/crates/syntax/src/ast/generated/tokens.rs @@ -112,16 +112,16 @@ impl AstToken for IntNumber { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FloatNumber { +pub struct FloatNumberPart { pub(crate) syntax: SyntaxToken, } -impl std::fmt::Display for FloatNumber { +impl std::fmt::Display for FloatNumberPart { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.syntax, f) } } -impl AstToken for FloatNumber { - fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER } +impl AstToken for FloatNumberPart { + fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART } fn cast(syntax: SyntaxToken) -> Option { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) diff --git a/crates/syntax/src/ast/make.rs b/crates/syntax/src/ast/make.rs index 5908dda8e638..846f4f3c71a0 100644 --- a/crates/syntax/src/ast/make.rs +++ b/crates/syntax/src/ast/make.rs @@ -799,6 +799,11 @@ pub fn struct_( )) } +pub fn literal(text: &str) -> ast::Literal { + assert_eq!(text.trim(), text); + ast_from_text(&format!("fn f() {{ let _ = {}; }}", text)) +} + #[track_caller] fn ast_from_text(text: &str) -> N { let parse = SourceFile::parse(text); @@ -827,7 +832,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken { pub mod tokens { use once_cell::sync::Lazy; - use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; + use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; pub(super) static SOURCE_FILE: Lazy> = Lazy::new(|| { SourceFile::parse( @@ -858,12 +863,6 @@ pub mod tokens { sf.syntax().first_child_or_token().unwrap().into_token().unwrap() } - pub fn literal(text: &str) -> SyntaxToken { - assert_eq!(text.trim(), text); - let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text)); - lit.syntax().first_child_or_token().unwrap().into_token().unwrap() - } - pub fn single_newline() -> SyntaxToken { let res = SOURCE_FILE .tree() diff --git a/crates/syntax/src/ast/node_ext.rs b/crates/syntax/src/ast/node_ext.rs index f2153ca92119..5b19b5ed2c28 100644 --- a/crates/syntax/src/ast/node_ext.rs +++ b/crates/syntax/src/ast/node_ext.rs @@ -555,7 +555,9 @@ impl ast::FieldExpr { self.syntax .children_with_tokens() // FIXME: Accepting floats here to reject them in validation later - .find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER) + .find(|c| { + c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART + }) .as_ref() .and_then(SyntaxElement::as_token) .cloned() diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 4b6dc236b534..5f2e7231d930 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -321,7 +321,7 @@ impl ast::IntNumber { } } -impl ast::FloatNumber { +impl ast::FloatNumberPart { pub fn suffix(&self) -> Option<&str> { let text = self.text(); let mut indices = text.char_indices(); @@ -355,14 +355,24 @@ impl Radix { #[cfg(test)] mod tests { - use crate::ast::{self, make, FloatNumber, IntNumber}; + use crate::ast::{self, make}; fn check_float_suffix<'a>(lit: &str, expected: impl Into>) { - assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into()); + let suffix = match make::literal(lit).kind() { + ast::LiteralKind::FloatNumber(f) => f.suffix(), + // `1f32` lexes as an INT_NUMBER + ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()), + e => unreachable!("{e:?}"), + }; + assert_eq!(suffix.as_deref(), expected.into()); } fn check_int_suffix<'a>(lit: &str, expected: impl Into>) { - assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into()); + let i = match make::literal(lit).kind() { + ast::LiteralKind::IntNumber(i) => i, + _ => unreachable!(), + }; + assert_eq!(i.suffix(), expected.into()); } #[test] @@ -390,12 +400,11 @@ mod tests { } fn check_string_value<'a>(lit: &str, expected: impl Into>) { - assert_eq!( - ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) } - .value() - .as_deref(), - expected.into() - ); + let s = match make::literal(&format!("\"{}\"", lit)).kind() { + ast::LiteralKind::String(s) => s, + _ => unreachable!(), + }; + assert_eq!(s.value().as_deref(), expected.into()); } #[test] diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index 2f6932a1addd..f5a78e4119dc 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -71,7 +71,17 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", ], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], - literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"], + literals: &[ + "INT_NUMBER", + "FLOAT_NUMBER_START_0", + "FLOAT_NUMBER_START_1", + "FLOAT_NUMBER_START_2", + "FLOAT_NUMBER_PART", + "CHAR", + "BYTE", + "STRING", + "BYTE_STRING", + ], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], nodes: &[ "SOURCE_FILE", @@ -183,6 +193,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "PATH", "PATH_SEGMENT", "LITERAL", + "FLOAT_LITERAL", "RENAME", "VISIBILITY", "WHERE_CLAUSE", diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index 4cfb8075cb15..33bf2c0cbad2 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -462,6 +462,10 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [ident] => { $crate::SyntaxKind::IDENT }; [shebang] => { $crate::SyntaxKind::SHEBANG }; + [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART }; + [float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 }; + [float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 }; + [float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 }; } pub use T; }; @@ -585,7 +589,7 @@ impl Field { fn lower(grammar: &Grammar) -> AstSrc { let mut res = AstSrc { - tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident" + tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident" .split_ascii_whitespace() .map(|it| it.to_string()) .collect::>(), diff --git a/crates/syntax/src/validation.rs b/crates/syntax/src/validation.rs index c2c2c82e11fb..3edca3eb8f9d 100644 --- a/crates/syntax/src/validation.rs +++ b/crates/syntax/src/validation.rs @@ -119,8 +119,15 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) { text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end)) } - let token = literal.token(); - let text = token.text(); + let token = literal.value(); + let text; + let text = match &token { + rowan::NodeOrToken::Node(node) => { + text = node.text().to_string(); + &*text + } + rowan::NodeOrToken::Token(token) => token.text(), + }; // FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205) let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| {