Merge #1112

1112: Fix literal support in token tree to ast item list r=matklad a=edwin0cheng

This PR implements following things :

1. Expose `next_token` from `ra_parse`
2. Fix the literal conversion in `token_tree_to_ast_item_list`
3. Add test for the conversion

Co-authored-by: Edwin Cheng <[email protected]>

Author: bors[bot]

crates/ra_mbe/src/lib.rs

@@ -167,7 +167,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
     )
     }
 
-    fn create_rules(macro_definition: &str) -> MacroRules {
+    pub(crate) fn create_rules(macro_definition: &str) -> MacroRules {
         let source_file = ast::SourceFile::parse(macro_definition);
         let macro_definition =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
@@ -176,7 +176,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
         crate::MacroRules::parse(&definition_tt).unwrap()
     }
 
-    fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree {
+    pub(crate) fn expand(rules: &MacroRules, invocation: &str) -> tt::Subtree {
         let source_file = ast::SourceFile::parse(invocation);
         let macro_invocation =
             source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
@@ -186,7 +186,7 @@ impl_froms!(TokenTree: Leaf, Subtree);
         rules.expand(&invocation_tt).unwrap()
     }
 
-    fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) {
+    pub(crate) fn assert_expansion(rules: &MacroRules, invocation: &str, expansion: &str) {
         let expanded = expand(rules, invocation);
         assert_eq!(expanded.to_string(), expansion);
     }
@@ -337,4 +337,46 @@ SOURCE_FILE@[0; 40)
         );
     }
 
+    #[test]
+    fn expand_literals_to_token_tree() {
+        fn to_subtree(tt: &tt::TokenTree) -> &tt::Subtree {
+            if let tt::TokenTree::Subtree(subtree) = tt {
+                return &subtree;
+            }
+            unreachable!("It is not a subtree");
+        }
+
+        fn to_literal(tt: &tt::TokenTree) -> &tt::Literal {
+            if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = tt {
+                return lit;
+            }
+            unreachable!("It is not a literal");
+        }
+
+        let rules = create_rules(
+            r#"
+            macro_rules! literals {
+                ($i:ident) => {
+                    {
+                        let a = 'c';
+                        let c = 1000;
+                        let f = 12E+99_f64;
+                        let s = "rust1";
+                    }
+                }
+            }
+            "#,
+        );
+        let expansion = expand(&rules, "literals!(foo)");
+        let stm_tokens = &to_subtree(&expansion.token_trees[0]).token_trees;
+
+        // [let] [a] [=] ['c'] [;]
+        assert_eq!(to_literal(&stm_tokens[3]).text, "'c'");
+        // [let] [c] [=] [1000] [;]
+        assert_eq!(to_literal(&stm_tokens[5 + 3]).text, "1000");
+        // [let] [f] [=] [12E+99_f64] [;]
+        assert_eq!(to_literal(&stm_tokens[10 + 3]).text, "12E+99_f64");
+        // [let] [s] [=] ["rust1"] [;]
+        assert_eq!(to_literal(&stm_tokens[15 + 3]).text, "\"rust1\"");
+    }
 }

crates/ra_mbe/src/syntax_bridge.rs

@@ -1,7 +1,7 @@
 use ra_parser::{TokenSource, TreeSink, ParseError};
 use ra_syntax::{
     AstNode, SyntaxNode, TextRange, SyntaxKind, SmolStr, SyntaxTreeBuilder, TreeArc, SyntaxElement,
-    ast, SyntaxKind::*, TextUnit
+    ast, SyntaxKind::*, TextUnit, classify_literal
 };
 
 /// Maps `tt::TokenId` to the relative range of the original token.
@@ -103,10 +103,12 @@ fn convert_tt(
     Some(res)
 }
 
+#[derive(Debug)]
 struct TtTokenSource {
     tokens: Vec<TtToken>,
 }
 
+#[derive(Debug)]
 struct TtToken {
     kind: SyntaxKind,
     is_joint_to_next: bool,
@@ -189,7 +191,7 @@ impl TtTokenSource {
     {
         let tok = match token {
             tt::Leaf::Literal(l) => TtToken {
-                kind: SyntaxKind::INT_NUMBER, // FIXME
+                kind: classify_literal(&l.text).unwrap().kind,
                 is_joint_to_next: false,
                 text: l.text.clone(),
             },
@@ -355,3 +357,44 @@ impl<'a> TreeSink for TtTreeSink<'a> {
         self.inner.error(error, self.text_pos)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::{expand, create_rules};
+
+    #[test]
+    fn convert_tt_token_source() {
+        let rules = create_rules(
+            r#"
+            macro_rules! literals {
+                ($i:ident) => {
+                    {
+                        let a = 'c';
+                        let c = 1000;
+                        let f = 12E+99_f64;
+                        let s = "rust1";
+                    }
+                }
+            }
+            "#,
+        );
+        let expansion = expand(&rules, "literals!(foo)");
+        let tt_src = TtTokenSource::new(&expansion);
+
+        // [{]
+        // [let] [a] [=] ['c'] [;]
+        assert_eq!(tt_src.tokens[1 + 3].text, "'c'");
+        assert_eq!(tt_src.tokens[1 + 3].kind, CHAR);
+        // [let] [c] [=] [1000] [;]
+        assert_eq!(tt_src.tokens[1 + 5 + 3].text, "1000");
+        assert_eq!(tt_src.tokens[1 + 5 + 3].kind, INT_NUMBER);
+        // [let] [f] [=] [12E+99_f64] [;]
+        assert_eq!(tt_src.tokens[1 + 10 + 3].text, "12E+99_f64");
+        assert_eq!(tt_src.tokens[1 + 10 + 3].kind, FLOAT_NUMBER);
+
+        // [let] [s] [=] ["rust1"] [;]
+        assert_eq!(tt_src.tokens[1 + 15 + 3].text, "\"rust1\"");
+        assert_eq!(tt_src.tokens[1 + 15 + 3].kind, STRING);
+    }
+}

crates/ra_syntax/src/lib.rs

@@ -40,7 +40,7 @@ pub use crate::{
     syntax_text::SyntaxText,
     syntax_node::{Direction,  SyntaxNode, WalkEvent, TreeArc, SyntaxTreeBuilder, SyntaxElement, SyntaxToken},
     ptr::{SyntaxNodePtr, AstPtr},
-    parsing::{tokenize, Token},
+    parsing::{tokenize, classify_literal, Token},
 };
 
 use ra_text_edit::AtomTextEdit;

crates/ra_syntax/src/parsing.rs

@@ -11,7 +11,7 @@ use crate::{
     syntax_node::GreenNode,
 };
 
-pub use self::lexer::{tokenize, Token};
+pub use self::lexer::{tokenize, classify_literal, Token};
 
 pub(crate) use self::reparsing::incremental_reparse;
 

crates/ra_syntax/src/parsing/lexer.rs

@@ -214,3 +214,12 @@ fn scan_literal_suffix(ptr: &mut Ptr) {
     }
     ptr.bump_while(is_ident_continue);
 }
+
+pub fn classify_literal(text: &str) -> Option<Token> {
+    let tkn = next_token(text);
+    if !tkn.kind.is_literal() || tkn.len.to_usize() != text.len() {
+        return None;
+    }
+
+    Some(tkn)
+}