Use the proc_macro token model in the bootstrap grammar.

Author: eddyb

build.rs

@@ -9,21 +9,22 @@ extern crate proc_quote;
 mod generate;
 #[path = "src/parse_node.rs"]
 mod parse_node;
+#[path = "src/proc_macro.rs"]
+pub mod proc_macro;
 #[path = "src/scannerless.rs"]
 pub mod scannerless;
 
 use std::env;
 use std::fs;
 use std::path::PathBuf;
 
-// FIXME(eddyb) use `scannerless::Grammar` when that wrapper hack is fixed.
-type Grammar = grammer::Grammar<scannerless::Pat<&'static str>>;
 fn main() {
     println!("cargo:rerun-if-changed=build.rs");
 
     let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
 
-    let grammar: Grammar = grammer::grammar_grammar();
+    let mut grammar = proc_macro::builtin();
+    grammar.extend(grammer::grammar_grammar());
 
     fs::write(
         &out_dir.join("parse_grammar.rs"),

macros/src/lib.rs

@@ -7,28 +7,16 @@ use proc_quote::ToTokens as _;
 
 #[proc_macro]
 pub fn scannerless_parser(input: TokenStream) -> TokenStream {
-    // FIXME(eddyb) parse the `proc_macro` tokens instead of strings.
-    gll::generate::rust::generate(
-        &input
-            .to_string()
-            .parse::<gll::scannerless::Grammar>()
-            .unwrap(),
-    )
-    .into_token_stream()
-    .into()
+    let grammar: gll::scannerless::Grammar = gll::parse_grammar(input.into()).unwrap();
+    gll::generate::rust::generate(&grammar)
+        .into_token_stream()
+        .into()
 }
 
 #[proc_macro]
 pub fn proc_macro_parser(input: TokenStream) -> TokenStream {
-    // FIXME(eddyb) parse the `proc_macro` tokens instead of strings.
     let mut grammar = gll::proc_macro::builtin();
-    grammar.extend(
-        input
-            .to_string()
-            .parse::<gll::proc_macro::Grammar>()
-            .unwrap()
-            .0,
-    );
+    grammar.extend(gll::parse_grammar(input.into()).unwrap());
     gll::generate::rust::generate(&grammar)
         .into_token_stream()
         .into()

src/lib.rs

@@ -18,5 +18,12 @@ pub mod runtime;
 #[forbid(unsafe_code)]
 pub mod scannerless;
 
+// HACK(eddyb) this contains impls for types in `proc_macro`, which depend on
+// `runtime`. Those parts of `runtime` should be moved to `grammer::input`.
+#[forbid(unsafe_code)]
+mod proc_macro_runtime;
+
 #[forbid(unsafe_code)]
 mod parse_grammar;
+
+pub use parse_grammar::parse_grammar;

src/parse_grammar.rs

@@ -1,73 +1,88 @@
 // HACK(eddyb) silence warnings from unused exports in the generated code.
 #![allow(unused)]
+#![allow(non_camel_case_types)]
 
 // HACK(eddyb) needed for bootstrapping.
 use crate as gll;
 
 include!(concat!(env!("OUT_DIR"), "/parse_grammar.rs"));
 
+use crate::proc_macro::{FlatToken, Span, TokenStream};
 use crate::runtime;
 use crate::scannerless::Pat as SPat;
 use std::ops::Bound;
 use std::str::FromStr;
 
-impl<Pat: From<SPat>> FromStr for crate::scannerless::WrapperHack<grammer::Grammar<Pat>> {
-    type Err = runtime::ParseError<runtime::LineColumn>;
-    fn from_str(src: &str) -> Result<Self, Self::Err> {
-        let mut grammar = grammer::Grammar::new();
-        Grammar::parse(src)?.with(|g| {
-            for rule_def in g.one().unwrap().rules {
-                let rule_def = rule_def.unwrap().one().unwrap();
-                grammar.define(rule_def.name.source(), rule_def.rule.one().unwrap().lower());
-            }
-        });
-        Ok(crate::scannerless::WrapperHack(grammar))
-    }
+pub fn parse_grammar<Pat: From<SPat>>(
+    stream: TokenStream,
+) -> Result<grammer::Grammar<Pat>, runtime::ParseError<Span>> {
+    let mut grammar = grammer::Grammar::new();
+    Grammar::parse(stream)?.with(|g| {
+        for rule_def in g.one().unwrap().rules {
+            let rule_def = rule_def.unwrap().one().unwrap();
+            let name = match rule_def.name.source() {
+                [FlatToken::Ident(ident)] => ident.to_string(),
+                _ => unreachable!(),
+            };
+            grammar.define(&name, rule_def.rule.one().unwrap().lower());
+        }
+    });
+    Ok(grammar)
 }
 
-impl Or<'_, '_, &str> {
+impl Or<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(self) -> grammer::RuleWithNamedFields<Pat> {
         let mut rules = self.rules.map(|rule| rule.unwrap().one().unwrap().lower());
         let first = rules.next().unwrap();
         rules.fold(first, |a, b| a | b)
     }
 }
 
-impl Concat<'_, '_, &str> {
+impl Concat<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(self) -> grammer::RuleWithNamedFields<Pat> {
         self.rules
             .map(|rule| rule.unwrap().one().unwrap().lower())
             .fold(grammer::empty(), |a, b| a + b)
     }
 }
 
-impl Rule<'_, '_, &str> {
+impl Rule<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(self) -> grammer::RuleWithNamedFields<Pat> {
         let mut rule = self.rule.one().unwrap().lower();
         if let Some(modifier) = self.modifier {
             rule = modifier.one().unwrap().lower(rule);
         }
         if let Some(field) = self.field {
-            rule = rule.field(field.source());
+            let field = match field.source() {
+                [FlatToken::Ident(ident)] => ident.to_string(),
+                _ => unreachable!(),
+            };
+            rule = rule.field(&field);
         }
         rule
     }
 }
 
-impl Primary<'_, '_, &str> {
+impl Primary<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(self) -> grammer::RuleWithNamedFields<Pat> {
         match self {
             Primary::Eat(pat) => grammer::eat(pat.one().unwrap().lower()),
             Primary::NegativeLookahead { pat } => {
                 grammer::negative_lookahead(pat.one().unwrap().lower())
             }
-            Primary::Call(name) => grammer::call(name.source()),
+            Primary::Call(name) => {
+                let name = match name.source() {
+                    [FlatToken::Ident(ident)] => ident.to_string(),
+                    _ => unreachable!(),
+                };
+                grammer::call(&name)
+            }
             Primary::Group { or } => or.map_or_else(grammer::empty, |or| or.one().unwrap().lower()),
         }
     }
 }
 
-impl Modifier<'_, '_, &str> {
+impl Modifier<'_, '_, TokenStream> {
     fn lower<Pat: From<SPat>>(
         self,
         rule: grammer::RuleWithNamedFields<Pat>,
@@ -90,7 +105,7 @@ impl Modifier<'_, '_, &str> {
     }
 }
 
-impl SepKind<'_, '_, &str> {
+impl SepKind<'_, '_, TokenStream> {
     fn lower(&self) -> grammer::SepKind {
         match self {
             SepKind::Simple(_) => grammer::SepKind::Simple,
@@ -99,11 +114,14 @@ impl SepKind<'_, '_, &str> {
     }
 }
 
-impl Pattern<'_, '_, &str> {
+impl Pattern<'_, '_, TokenStream> {
     fn lower(self) -> SPat {
-        fn unescape<T>(handle: Handle<'_, '_, &str, T>) -> String {
+        fn unescape<T>(handle: Handle<'_, '_, TokenStream, T>) -> String {
             let mut out = String::new();
-            let s = handle.source();
+            let s = match handle.source() {
+                [FlatToken::Literal(lit)] => lit.to_string(),
+                _ => unreachable!(),
+            };
             let mut chars = s[1..s.len() - 1].chars();
             while let Some(c) = chars.next() {
                 let c = match c {

src/proc_macro.rs

@@ -1,15 +1,13 @@
 use crate::generate::rust::RustInputPat;
 use crate::generate::src::{quotable_to_src, quote, Src, ToSrc};
-use crate::runtime::{Input, InputMatch, Range};
 use crate::scannerless::Pat as SPat;
-use grammer::{self, call, eat, MatchesEmpty, MaybeKnown};
-use indexing::{proof::Provable, Container, Index, Unknown};
+use grammer::{call, eat, MatchesEmpty, MaybeKnown};
 pub use proc_macro2::{
     Delimiter, Ident, LexError, Literal, Punct, Spacing, Span, TokenStream, TokenTree,
 };
-use std::{ops, str::FromStr};
+use std::str::FromStr;
 
-pub type Grammar = crate::scannerless::WrapperHack<grammer::Grammar<Pat>>;
+pub type Grammar = grammer::Grammar<Pat>;
 
 pub fn builtin() -> Grammar {
     let mut g = grammer::Grammar::new();
@@ -44,7 +42,7 @@ pub fn builtin() -> Grammar {
         ident | punct | literal | group('(', ')') | group('[', ']') | group('{', '}'),
     );
 
-    crate::scannerless::WrapperHack(g)
+    g
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -67,6 +65,13 @@ impl FromStr for Pat {
     }
 }
 
+// FIXME(eddyb) perhaps support `TryFrom`/`TryInto` directly in `grammer`?
+impl From<&str> for Pat {
+    fn from(s: &str) -> Self {
+        s.parse().unwrap()
+    }
+}
+
 impl From<FlatTokenPat<String>> for Pat {
     fn from(pat: FlatTokenPat<String>) -> Self {
         Pat(vec![pat])
@@ -76,7 +81,7 @@ impl From<FlatTokenPat<String>> for Pat {
 impl From<SPat> for Pat {
     fn from(pat: SPat) -> Self {
         match pat {
-            SPat::String(s) => s.parse().unwrap(),
+            SPat::String(s) => s[..].into(),
             SPat::Range(..) => unimplemented!("char ranges are unsupported"),
         }
     }
@@ -185,7 +190,7 @@ impl FlatToken {
     }
 }
 
-fn flatten(stream: TokenStream, out: &mut Vec<FlatToken>) {
+pub(crate) fn flatten(stream: TokenStream, out: &mut Vec<FlatToken>) {
     for tt in stream {
         let flat = match tt {
             TokenTree::Group(tt) => {
@@ -216,82 +221,3 @@ fn flatten(stream: TokenStream, out: &mut Vec<FlatToken>) {
         out.push(flat);
     }
 }
-
-impl Input for TokenStream {
-    type Container = Vec<FlatToken>;
-    type Slice = [FlatToken];
-    type SourceInfo = ops::Range<Span>;
-    type SourceInfoPoint = Span;
-    fn to_container(self) -> Self::Container {
-        let mut out = vec![];
-        flatten(self, &mut out);
-        out
-    }
-    fn slice<'b, 'i>(
-        input: &'b Container<'i, Self::Container>,
-        range: Range<'i>,
-    ) -> &'b Self::Slice {
-        &input[range.0]
-    }
-    fn source_info<'i>(
-        input: &Container<'i, Self::Container>,
-        range: Range<'i>,
-    ) -> Self::SourceInfo {
-        // FIXME(eddyb) should be joining up spans, but the API
-        // for that is still "semver-exempt" in `proc-macro2`.
-        let last = range
-            .nonempty()
-            .map(|r| r.last().no_proof())
-            .unwrap_or(range.past_the_end());
-        Self::source_info_point(input, range.first())..Self::source_info_point(input, last)
-    }
-    fn source_info_point<'i>(
-        input: &Container<'i, Self::Container>,
-        index: Index<'i, Unknown>,
-    ) -> Self::SourceInfoPoint {
-        // Try to get as much information as possible.
-        let (before, after) = input.split_at(index);
-        let before = &input[before];
-        let after = &input[after];
-        if let Some(first) = after.first() {
-            first.span()
-        } else if let Some(last) = before.last() {
-            // Not correct but we're at the end of the input anyway.
-            last.span()
-        } else {
-            // HACK(eddyb) last resort, make a span up
-            // (a better option should exist)
-            Span::call_site()
-        }
-    }
-}
-
-impl InputMatch<&'static [FlatTokenPat<&'static str>]> for [FlatToken] {
-    fn match_left(&self, &pat: &&[FlatTokenPat<&str>]) -> Option<usize> {
-        if self
-            .iter()
-            .zip(pat)
-            .take_while(|(t, p)| t.matches_pat(p))
-            .count()
-            == pat.len()
-        {
-            Some(pat.len())
-        } else {
-            None
-        }
-    }
-    fn match_right(&self, &pat: &&[FlatTokenPat<&str>]) -> Option<usize> {
-        if self
-            .iter()
-            .zip(pat)
-            .rev()
-            .take_while(|(t, p)| t.matches_pat(p))
-            .count()
-            == pat.len()
-        {
-            Some(pat.len())
-        } else {
-            None
-        }
-    }
-}