Merge pull request #113 from eddyb/bangless

Parse grammars with `proc_macro` tokens and remove "negative lookahead".

Author: eddyb

Cargo.toml

@@ -31,7 +31,7 @@ doctest = false
 test = false
 
 [patch.'crates-io']
-grammer = { git = "https://github.com/lykenware/grammer", rev = "e108acbe83271761538bf5b2ff2daeaaafa5919c" }
+grammer = { git = "https://github.com/lykenware/grammer", rev = "ed4fc4f9be9aa2c1a6c3245ba4e91684ddff5f2f" }
 
 [workspace]
 members = [

build.rs

@@ -9,21 +9,22 @@ extern crate proc_quote;
 mod generate;
 #[path = "src/parse_node.rs"]
 mod parse_node;
+#[path = "src/proc_macro.rs"]
+pub mod proc_macro;
 #[path = "src/scannerless.rs"]
 pub mod scannerless;
 
 use std::env;
 use std::fs;
 use std::path::PathBuf;
 
-// FIXME(eddyb) use `scannerless::Grammar` when that wrapper hack is fixed.
-type Grammar = grammer::Grammar<scannerless::Pat<&'static str>>;
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
 
     let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
 
-    let grammar: Grammar = grammer::grammar_grammar();
+    let mut grammar = proc_macro::builtin();
+    grammar.extend(grammer::grammar_grammar());
 
     fs::write(
         &out_dir.join("parse_grammar.rs"),

macros/src/lib.rs

@@ -7,28 +7,16 @@ use proc_quote::ToTokens as _;
 
 #[proc_macro]
 pub fn scannerless_parser(input: TokenStream) -> TokenStream {
-    // FIXME(eddyb) parse the `proc_macro` tokens instead of strings.
-    gll::generate::rust::generate(
-        &input
-            .to_string()
-            .parse::<gll::scannerless::Grammar>()
-            .unwrap(),
-    )
-    .into_token_stream()
-    .into()
+    let grammar: gll::scannerless::Grammar = gll::parse_grammar(input.into()).unwrap();
+    gll::generate::rust::generate(&grammar)
+        .into_token_stream()
+        .into()
 }
 
 #[proc_macro]
 pub fn proc_macro_parser(input: TokenStream) -> TokenStream {
-    // FIXME(eddyb) parse the `proc_macro` tokens instead of strings.
     let mut grammar = gll::proc_macro::builtin();
-    grammar.extend(
-        input
-            .to_string()
-            .parse::<gll::proc_macro::Grammar>()
-            .unwrap()
-            .0,
-    );
+    grammar.extend(gll::parse_grammar(input.into()).unwrap());
     gll::generate::rust::generate(&grammar)
         .into_token_stream()
         .into()

src/generate/rust.rs

@@ -116,7 +116,7 @@ impl<Pat> RuleTypeMethods for Rule<Pat> {
 
     fn field_type(&self, path: &[usize]) -> Src {
         match self {
-            Rule::Empty | Rule::Eat(_) | Rule::NegativeLookahead(_) => {
+            Rule::Empty | Rule::Eat(_) => {
                 assert_eq!(path, []);
                 quote!(())
             }
@@ -199,7 +199,6 @@ impl<Pat: Ord + Hash + RustInputPat> RuleRuleMapMethods<Pat> for Rule<Pat> {
         match self {
             Rule::Empty => "".to_string(),
             Rule::Eat(pat) => pat.rust_matcher().to_pretty_string(),
-            Rule::NegativeLookahead(pat) => format!("!{}", pat.rust_matcher().to_pretty_string()),
             Rule::Call(r) => r.clone(),
             Rule::Concat([left, right]) => format!(
                 "({} {})",
@@ -246,7 +245,7 @@ impl<Pat: Ord + Hash + RustInputPat> RuleRuleMapMethods<Pat> for Rule<Pat> {
         rules: &RuleMap<'_, Pat>,
     ) -> ParseNodeShape<ParseNodeKind> {
         match &**rc_self {
-            Rule::Empty | Rule::Eat(_) | Rule::NegativeLookahead(_) => ParseNodeShape::Opaque,
+            Rule::Empty | Rule::Eat(_) => ParseNodeShape::Opaque,
             Rule::Call(_) => unreachable!(),
             Rule::Concat([left, right]) => {
                 ParseNodeShape::Split(left.parse_node_kind(rules), right.parse_node_kind(rules))
@@ -802,10 +801,6 @@ impl<Pat: Ord + Hash + RustInputPat> RuleGenerateMethods<Pat> for Rule<Pat> {
                 let pat = pat.rust_matcher();
                 check(quote!(let Some(_range) = p.input_consume_left(_range, &(#pat)))).apply(cont)
             }
-            (Rule::NegativeLookahead(pat), _) => {
-                let pat = pat.rust_matcher();
-                check(quote!(!p.input_lookahead_left(_range, &(#pat)))).apply(cont)
-            }
             (Rule::Call(r), _) => call(Rc::new(CodeLabel::NamedRule(r.clone()))).apply(cont),
             (Rule::Concat([left, right]), None) => {
                 (left.generate_parse(None) + right.generate_parse(None)).apply(cont)
@@ -902,7 +897,6 @@ impl<Pat: Ord + Hash + RustInputPat> RuleGenerateMethods<Pat> for Rule<Pat> {
         match self {
             Rule::Empty
             | Rule::Eat(_)
-            | Rule::NegativeLookahead(_)
             | Rule::Call(_)
             | Rule::RepeatMany(..)
             | Rule::RepeatMore(..) => {

src/lib.rs

@@ -18,5 +18,12 @@ pub mod runtime;
 #[forbid(unsafe_code)]
 pub mod scannerless;
 
+// HACK(eddyb) this contains impls for types in `proc_macro`, which depend on
+// `runtime`. Those parts of `runtime` should be moved to `grammer::input`.
+#[forbid(unsafe_code)]
+mod proc_macro_runtime;
+
 #[forbid(unsafe_code)]
 mod parse_grammar;
+
+pub use parse_grammar::parse_grammar;

src/parse_grammar.rs

@@ -1,73 +1,85 @@
 // HACK(eddyb) silence warnings from unused exports in the generated code.
 #![allow(unused)]
+#![allow(non_camel_case_types)]
 
 // HACK(eddyb) needed for bootstrapping.
 use crate as gll;
 
 include!(concat!(env!("OUT_DIR"), "/parse_grammar.rs"));
 
+use crate::proc_macro::{FlatToken, Span, TokenStream};
 use crate::runtime;
 use crate::scannerless::Pat as SPat;
 use std::ops::Bound;
 use std::str::FromStr;
 
-impl<Pat: From<SPat>> FromStr for crate::scannerless::WrapperHack<grammer::Grammar<Pat>> {
-    type Err = runtime::ParseError<runtime::LineColumn>;
-    fn from_str(src: &str) -> Result<Self, Self::Err> {
-        let mut grammar = grammer::Grammar::new();
-        Grammar::parse(src)?.with(|g| {
-            for rule_def in g.one().unwrap().rules {
-                let rule_def = rule_def.unwrap().one().unwrap();
-                grammar.define(rule_def.name.source(), rule_def.rule.one().unwrap().lower());
-            }
-        });
-        Ok(crate::scannerless::WrapperHack(grammar))
-    }
+pub fn parse_grammar<Pat: From<SPat>>(
+    stream: TokenStream,
+) -> Result<grammer::Grammar<Pat>, runtime::ParseError<Span>> {
+    let mut grammar = grammer::Grammar::new();
+    Grammar::parse(stream)?.with(|g| {
+        for rule_def in g.one().unwrap().rules {
+            let rule_def = rule_def.unwrap().one().unwrap();
+            let name = match rule_def.name.source() {
+                [FlatToken::Ident(ident)] => ident.to_string(),
+                _ => unreachable!(),
+            };
+            grammar.define(&name, rule_def.rule.one().unwrap().lower());
+        }
+    });
+    Ok(grammar)
 }
 
-impl Or<'_, '_, &str> {
+impl Or<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(self) -> grammer::RuleWithNamedFields<Pat> {
         let mut rules = self.rules.map(|rule| rule.unwrap().one().unwrap().lower());
         let first = rules.next().unwrap();
         rules.fold(first, |a, b| a | b)
     }
 }
 
-impl Concat<'_, '_, &str> {
+impl Concat<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(self) -> grammer::RuleWithNamedFields<Pat> {
         self.rules
             .map(|rule| rule.unwrap().one().unwrap().lower())
             .fold(grammer::empty(), |a, b| a + b)
     }
 }
 
-impl Rule<'_, '_, &str> {
+impl Rule<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(self) -> grammer::RuleWithNamedFields<Pat> {
         let mut rule = self.rule.one().unwrap().lower();
         if let Some(modifier) = self.modifier {
             rule = modifier.one().unwrap().lower(rule);
         }
         if let Some(field) = self.field {
-            rule = rule.field(field.source());
+            let field = match field.source() {
+                [FlatToken::Ident(ident)] => ident.to_string(),
+                _ => unreachable!(),
+            };
+            rule = rule.field(&field);
         }
         rule
     }
 }
 
-impl Primary<'_, '_, &str> {
+impl Primary<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(self) -> grammer::RuleWithNamedFields<Pat> {
         match self {
             Primary::Eat(pat) => grammer::eat(pat.one().unwrap().lower()),
-            Primary::NegativeLookahead { pat } => {
-                grammer::negative_lookahead(pat.one().unwrap().lower())
+            Primary::Call(name) => {
+                let name = match name.source() {
+                    [FlatToken::Ident(ident)] => ident.to_string(),
+                    _ => unreachable!(),
+                };
+                grammer::call(&name)
             }
-            Primary::Call(name) => grammer::call(name.source()),
             Primary::Group { or } => or.map_or_else(grammer::empty, |or| or.one().unwrap().lower()),
         }
     }
 }
 
-impl Modifier<'_, '_, &str> {
+impl Modifier<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(
         self,
         rule: grammer::RuleWithNamedFields<Pat>,
@@ -90,7 +102,7 @@ impl Modifier<'_, '_, &str> {
     }
 }
 
-impl SepKind<'_, '_, &str> {
+impl SepKind<'_, '_, TokenStream> {
     fn lower(&self) -> grammer::SepKind {
         match self {
             SepKind::Simple(_) => grammer::SepKind::Simple,
@@ -99,11 +111,14 @@ impl SepKind<'_, '_, &str> {
     }
 }
 
-impl Pattern<'_, '_, &str> {
+impl Pattern<'_, '_, TokenStream> {
     fn lower(self) -> SPat {
-        fn unescape<T>(handle: Handle<'_, '_, &str, T>) -> String {
+        fn unescape<T>(handle: Handle<'_, '_, TokenStream, T>) -> String {
             let mut out = String::new();
-            let s = handle.source();
+            let s = match handle.source() {
+                [FlatToken::Literal(lit)] => lit.to_string(),
+                _ => unreachable!(),
+            };
             let mut chars = s[1..s.len() - 1].chars();
             while let Some(c) = chars.next() {
                 let c = match c {