Implement (possibly?) the full Rust Grammar

Cargo.toml

@@ -4,11 +4,11 @@ version = "0.1.0"
 authors = ["The Rust Project Developers"]
 
 [dependencies]
-gll = "0.0.2"
+gll = { git = "https://github.com/lykenware/gll" }
 proc-macro2 = "0.4.0"
 structopt = "0.2.12"
 walkdir = "2.2.6"
 
 [build-dependencies]
-gll = "0.0.2"
+gll = { git = "https://github.com/lykenware/gll" }
 walkdir = "2.2.6"

build.rs

@@ -21,6 +21,22 @@ fn main() {
     // Start with the builtin rules for proc-macro grammars.
     let mut grammar = gll::proc_macro::builtin();
 
+    // HACK(eddyb) inject a custom builtin - this should be upstreamed to gll!
+    {
+        use gll::proc_macro::{FlatTokenPat, Pat};
+
+        grammar.define(
+            "LIFETIME",
+            gll::grammar::eat(Pat(vec![
+                FlatTokenPat::Punct {
+                    ch: Some('\''),
+                    joint: Some(true),
+                },
+                FlatTokenPat::Ident(None),
+            ])),
+        );
+    }
+
     // Add in each grammar fragment to the grammar.
     for fragment in fragments {
         let path = fragment.into_path();

grammar/abi.lyg

@@ -0,0 +1,31 @@
+// FIXME(eddyb) implement more specific literal support in `gll::proc_macro`
+Abi = LITERAL?;
+
+/*
+// HACK(eddyb) taken from `librustc_target/spec/abi.rs`
+Abi =
+  // Defaults to "C"
+  | {}
+  // Platform-specific ABIs
+  | "\"cdecl\""
+  | "\"stdcall\""
+  | "\"fastcall\""
+  | "\"vectorcall\""
+  | "\"thiscall\""
+  | "\"aapcs\""
+  | "\"win64\""
+  | "\"sysv64\""
+  | "\"ptx-kernel\""
+  | "\"msp430-interrupt\""
+  | "\"x86-interrupt\""
+  | "\"amdgpu-kernel\""
+  // Cross-platform ABIs
+  | "\"Rust\""
+  | "\"C\""
+  | "\"system\""
+  | "\"rust-intrinsic\""
+  | "\"rust-call\""
+  | "\"platform-intrinsic\""
+  | "\"unadjusted\""
+  ;
+*/

grammar/attr.lyg

@@ -1,9 +1,8 @@
 OuterAttr = "#" attr:Attr;
-InnerAttr = "#!" attr:Attr;
+InnerAttr = "#" "!" attr:Attr;
 Attr = "[" path:Path input:AttrInput "]";
 AttrInput =
-    {} |
-    "=" LITERAL |
-    "(" TOKEN_TREE* ")" |
-    "[" TOKEN_TREE* "]" |
-    "{" TOKEN_TREE* "}";
+  | {}
+  | "=" TOKEN_TREE
+  | MacroInput
+  ;

grammar/expr.lyg

@@ -0,0 +1,130 @@
+Expr = attrs:OuterAttr* kind:ExprKind;
+ExprKind =
+  | Literal:LITERAL
+  | Paren:{ "(" attrs:InnerAttr* expr:Expr ")" }
+  | Borrow:{ "&" mutable:"mut"? expr:Expr }
+  | Box:{ "box" expr:Expr }
+  | Unary:{ op:UnaryOp expr:Expr }
+  | Try:{ expr:Expr "?" }
+  | Binary:{ left:Expr op:BinaryOp right:Expr }
+  | Assign:{ left:Expr { "=" | op:BinaryAssignOp } right:Expr }
+  | Range:{ start:Expr? ".." end:Expr? }
+  | RangeInclusive:{ start:Expr? "..=" end:Expr }
+  // unstable(type_ascription):
+  | Type:{ expr:Expr ":" ty:Type }
+  | Cast:{ expr:Expr "as" ty:Type }
+  | Field:{ base:Expr "." field:FieldName }
+  | Index:{ base:Expr "[" index:Expr "]" }
+  | Array:{ "[" attrs:InnerAttr* exprs:Expr* % "," ","? "]" }
+  | Repeat:{ "[" attrs:InnerAttr* elem:Expr ";" count:Expr "]" }
+  | Tuple:{ "(" attrs:InnerAttr* exprs:Expr* % "," ","? ")" }
+  | Path:QPath
+  | Call:{ callee:Expr "(" args:Expr* % "," ","? ")" }
+  | MethodCall:{ receiver:Expr "." method:PathSegment "(" args:Expr* % "," ","? ")" }
+  | Struct:{ path:Path "{" attrs:InnerAttr* fields:StructExprFieldsAndBase "}" }
+  | Block:{ { label:Label ":" }? unsafety:"unsafe"? block:Block }
+  // ustable(async_await):
+  | AsyncBlock:{ "async" block:Block }
+  // unstable(try_block):
+  | TryBlock:{ "try" block:Block }
+  | Continue:{ "continue" label:Label? }
+  | Break:{ "break" label:Label? value:Expr? }
+  | Return:{ "return" value:Expr? }
+  // unstable(generators):
+  | Yield:{ "yield" value:Expr? }
+  | If:If
+  | Match:{ "match" expr:Expr "{" attrs:InnerAttr* arms:MatchArm* "}" }
+  | Loop:{ { label:Label ":" }? "loop" body:Block }
+  | While:{ { label:Label ":" }? "while" cond:Cond body:Block }
+  | For:{ { label:Label ":" }? "for" pat:Pat "in" expr:Expr body:Block }
+  | Closure:{
+      // unstable(generators):
+      generator_static:"static"?
+      // unstable(async_await):
+      asyncness:"async"?
+      by_val:"move"?
+      "|" args:ClosureArg* % "," ","? "|" { "->" ret_ty:Type }? body:Expr
+    }
+  | MacroCall:MacroCall
+  ;
+
+UnaryOp =
+  | Not:"!"
+  | Neg:"-"
+  | Deref:"*";
+
+BinaryOp =
+  // Arithmetic & bitwise (these also have `BinaryAssignOp` forms)
+  | Add:"+"
+  | Sub:"-"
+  | Mul:"*"
+  | Div:"/"
+  | Rem:"%"
+  | BitXor:"^"
+  | BitAnd:"&"
+  | BitOr:"|"
+  | Shl:"<<"
+  | Shr:">>"
+  // Logic (short-circuiting)
+  | LogicAnd:"&&"
+  | LogicOr:"||"
+  // Comparisons
+  | Eq:"=="
+  | Lt:"<"
+  | Le:"<="
+  | Ne:"!="
+  | Gt:">"
+  | Ge:">="
+  ;
+
+// FIXME(eddyb) figure out how to deduplicate this with `BinaryOp`
+// The problem is something like `BinaryOp "="` allows a space in between,
+// as the last token inside each `BinaryOp` case does not require being
+// joint, but each token before the `=` here does impose that requirement.
+BinaryAssignOp =
+  | Add:"+="
+  | Sub:"-="
+  | Mul:"*="
+  | Div:"/="
+  | Rem:"%="
+  | BitXor:"^="
+  | BitAnd:"&="
+  | BitOr:"|="
+  | Shl:"<<="
+  | Shr:">>="
+  ;
+
+FieldName =
+  | Ident:IDENT
+  // FIXME(eddyb) restrict this to only integer literals
+  | Numeric:LITERAL
+  ;
+
+// FIXME(eddyb) find a way to express this `A* B?` pattern better
+StructExprFieldsAndBase =
+  | Fields:{ fields:StructExprField* % "," ","? }
+  | Base:{ ".." base:Expr }
+  | FieldsAndBase:{ fields:StructExprField+ % "," "," ".." base:Expr }
+  ;
+
+StructExprField = attrs:OuterAttr* kind:StructExprFieldKind;
+StructExprFieldKind =
+  | Shorthand:IDENT
+  | Explicit:{ field:FieldName ":" expr:Expr }
+  ;
+
+Label = LIFETIME;
+
+If = "if" cond:Cond then:Block { "else" else_expr:ElseExpr }?;
+Cond =
+  | Bool:Expr
+  | Let:{ "let" pat:Pat "=" expr:Expr }
+  ;
+ElseExpr =
+  | Block:Block
+  | If:If
+  ;
+
+MatchArm = attrs:OuterAttr* "|"? pats:Pat+ % "|" { "if" guard:Expr }? "=>" body:Expr ","?;
+
+ClosureArg = pat:Pat { ":" ty:Type }?;

grammar/generics.lyg

@@ -0,0 +1,42 @@
+Generics = "<" params:GenericParam* % "," ","? ">";
+GenericParam = attrs:OuterAttr* kind:GenericParamKind;
+GenericParamKind =
+  | Lifetime:{ name:LIFETIME { ":" bounds:LifetimeBound* % "+" "+"? }? }
+  | Type:{ name:IDENT { ":" bounds:TypeBound* % "+" "+"? }? { "=" default:Type }? }
+  ;
+
+ForAllBinder = "for" generics:Generics;
+
+WhereClause = "where" bounds:WhereBound* % "," ","?;
+WhereBound =
+  | Lifetime:{ lt:LIFETIME ":" bounds:LifetimeBound* % "+" "+"? }
+  | Type:{ binder:ForAllBinder? ty:Type ":" bounds:TypeBound* % "+" "+"? }
+  // unstable(#20041):
+  | TypeEq:{ binder:ForAllBinder? left:Type { "=" | "==" } right:Type }
+  ;
+
+LifetimeBound = outlives:LIFETIME;
+TypeBound =
+  | Outlives:LIFETIME
+  | Trait:TypeTraitBound
+  | TraitParen:{ "(" bound:TypeTraitBound ")" }
+  ;
+TypeTraitBound = unbound:"?"? binder:ForAllBinder? path:Path;
+
+GenericArgs =
+  | AngleBracket:{ "<" args_and_bindings:AngleBracketGenericArgsAndBindings? ","? ">" }
+  | Paren:{ "(" inputs:Type* % "," ","? ")" { "->" output:Type }? }
+  ;
+
+// FIXME(eddyb) find a way to express this `A* B*` pattern better
+AngleBracketGenericArgsAndBindings =
+  | Args:GenericArg+ % ","
+  | Bindings:TypeBinding+ % ","
+  | ArgsAndBindings:{ args:GenericArg+ % "," "," bindings:TypeBinding+ % "," }
+  ;
+
+GenericArg =
+  | Lifetime:LIFETIME
+  | Type:Type
+  ;
+TypeBinding = name:IDENT "=" ty:Type;

grammar/item.lyg

@@ -1,7 +1,118 @@
-ModuleContents = attrs:InnerAttr* items:ItemWithOuterAttr*;
+ModuleContents = attrs:InnerAttr* items:Item*;
 
-ItemWithOuterAttr = attrs:OuterAttr* item:Item;
-// TODO other items
-Item =
-    ExternCrate:{ "extern" "crate" name:IDENT { "as" rename:IDENT }? ";" } |
-    Use:{ "use" path:Path { "as" rename:IDENT }? ";" }; // TODO use trees
+Item = attrs:OuterAttr* vis:Vis? kind:ItemKind;
+ItemKind =
+  | Use:{ "use" use_tree:UseTree ";" }
+  | ExternCrate:{ "extern" "crate" name:IDENT { "as" rename:IDENT }? ";" }
+  | Mod:{ "mod" name:IDENT { ";" | "{" contents:ModuleContents "}" } }
+  | ForeignMod:{ "extern" abi:Abi "{" attrs:InnerAttr* items:ForeignItem* "}" }
+  | Static:{ "static" mutable:"mut"? name:IDENT ":" ty:Type "=" value:Expr ";" }
+  | Const:{ "const" name:IDENT ":" ty:Type "=" value:Expr ";" }
+  | Fn:{ header:FnHeader "fn" decl:FnDecl body:Block }
+  | TypeAlias:{ "type" name:IDENT generics:Generics? where_clause:WhereClause? "=" ty:Type ";" }
+  // unstable(existential_type):
+  | ExistentialType:{ "existential" "type" name:IDENT generics:Generics? where_clause:WhereClause? ":" bounds:TypeBound* % "+" "+"? ";" }
+  | Enum:{ "enum" name:IDENT generics:Generics? where_clause:WhereClause? "{" variants:EnumVariant* % "," ","? "}" }
+  | Struct:{ "struct" name:IDENT generics:Generics? body:StructBody }
+  | Union:{ "union" name:IDENT generics:Generics? where_clause:WhereClause? "{" fields:RecordField* % "," ","? "}" }
+  | Trait:{
+      unsafety:"unsafe"?
+      // unstable(optin_builtin_traits):
+      auto:"auto"?
+      "trait" name:IDENT generics:Generics?
+      { ":" superbounds:TypeBound* % "+" "+"? }?
+      where_clause:WhereClause? "{" trait_items:TraitItem* "}"
+    }
+  // unstable(trait_alias):
+  | TraitAlias:{
+    "trait" name:IDENT generics:Generics?
+        { ":" superbounds:TypeBound* % "+" "+"? }?
+        where_clause:WhereClause? "=" bounds:TypeBound* % "+" "+"? ";"
+    }
+  | Impl:{
+      // unstable(specialization):
+      defaultness:"default"?
+      unsafety:"unsafe"? "impl" generics:Generics?
+      { negate:"!"? trait_path:Path "for" }? ty:Type
+      where_clause:WhereClause? "{" attrs:InnerAttr* impl_items:ImplItem* "}"
+    }
+  // unstable(decl_macro):
+  | Macro:{ "macro" name:IDENT { "(" TOKEN_TREE* ")" }? "{" TOKEN_TREE* "}" }
+  | MacroCall:ItemMacroCall
+  ;
+
+UseTree =
+  | Glob:{ prefix:UseTreePrefix? "*" }
+  | Nested:{ prefix:UseTreePrefix? "{" children:UseTree* % "," ","? "}" }
+  | Simple:{ path:Path { "as" rename:IDENT }? }
+  ;
+UseTreePrefix =
+  | Path:{ path:Path "::" }
+  | Global:"::"
+  ;
+
+ForeignItem = attrs:OuterAttr* vis:Vis? kind:ForeignItemKind;
+ForeignItemKind =
+  | Static:{ "static" mutable:"mut"? name:IDENT ":" ty:Type ";" }
+  | Fn:{ "fn" decl:FnDecl ";" }
+  // unstable(extern_types):
+  | Type:{ "type" name:IDENT ";" }
+  // unstable(macros_in_extern):
+  | MacroCall:ItemMacroCall
+  ;
+
+TraitItem = attrs:OuterAttr* kind:TraitItemKind;
+TraitItemKind =
+  | Const:{ "const" name:IDENT ":" ty:Type { "=" default:Expr }? ";" }
+  | Fn:{ header:FnHeader "fn" decl:FnDecl { default_body:Block | ";" } }
+  | Type:{ "type" name:IDENT generics:Generics? { ":" bounds:TypeBound* % "+" "+"? }? where_clause:WhereClause? { "=" default:Type }? ";" }
+  | MacroCall:ItemMacroCall
+  ;
+
+ImplItem =
+  attrs:OuterAttr*
+  // unstable(specialization):
+  defaultness:"default"?
+  vis:Vis? kind:ImplItemKind
+  ;
+ImplItemKind =
+  | Const:{ "const" name:IDENT ":" ty:Type "=" value:Expr ";" }
+  | Fn:{ header:FnHeader "fn" decl:FnDecl body:Block }
+  | Type:{ "type" name:IDENT generics:Generics? where_clause:WhereClause? "=" ty:Type ";" }
+  // unstable(existential_type):
+  | ExistentialType:{ "existential" "type" name:IDENT generics:Generics? where_clause:WhereClause? ":" bounds:TypeBound* % "+" "+"? ";" }
+  | MacroCall:ItemMacroCall
+  ;
+
+FnHeader = constness:"const"? unsafety:"unsafe"? asyncness:"async"? { "extern" abi:Abi }?;
+FnDecl = name:IDENT generics:Generics? "(" args:FnArgs? ","? ")" { "->" ret_ty:Type }? where_clause:WhereClause?;
+
+// FIXME(eddyb) find a way to express this `A* B?` pattern better
+FnArgs =
+  | Regular:FnArg+ % ","
+  | Variadic:"..."
+  | RegularAndVariadic:{ args:FnArg+ % "," "," "..." }
+  ;
+FnArg =
+  | SelfValue:{ mutable:"mut"? "self" }
+  | SelfRef:{ "&" lt:LIFETIME? mutable:"mut"? "self" }
+  | Regular:FnSigInput
+  ;
+
+EnumVariant = attrs:OuterAttr* name:IDENT kind:EnumVariantKind { "=" discr:Expr }?;
+EnumVariantKind =
+  | Unit:{}
+  | Tuple:{ "(" fields:TupleField* % "," ","? ")" }
+  | Record:{ "{" fields:RecordField* % "," ","? "}" }
+  ;
+
+// FIXME(eddyb) could maybe be shared more with `EnumVariantKind`?
+// The problem is the semicolons for `Unit` and `Tuple`, and the where clauses.
+StructBody =
+  | Unit:{ where_clause:WhereClause? ";" }
+  | Tuple:{ "(" fields:TupleField* % "," ","? ")" where_clause:WhereClause? ";" }
+  | Record:{ where_clause:WhereClause? "{" fields:RecordField* % "," ","? "}" }
+  ;
+
+TupleField = attrs:OuterAttr* vis:Vis? ty:Type;
+RecordField = attrs:OuterAttr* vis:Vis? name:IDENT ":" ty:Type;

grammar/macro.lyg

@@ -0,0 +1,15 @@
+MacroCall = path:Path "!" ident_input:IDENT? input:MacroInput;
+MacroInput =
+  | "(" TOKEN_TREE* ")"
+  | "[" TOKEN_TREE* "]"
+  | "{" TOKEN_TREE* "}"
+  ;
+
+// FIXME(eddyb) could maybe be shared more with `MacroInput`?
+// The problem is the semicolons for the `()` and `[]` cases.
+ItemMacroCall = path:Path "!" ident_input:IDENT? input:ItemMacroInput;
+ItemMacroInput =
+  | "(" TOKEN_TREE* ")" ";"
+  | "[" TOKEN_TREE* "]" ";"
+  | "{" TOKEN_TREE* "}"
+  ;