lang: Migrate to nom 4.0.0

akshayknarayan · akshayknarayan · commit 16eba9b16d1f · 2018-09-06T13:50:15.000-04:00
As a workaround to rust-bakery/nom#544, migrate to nom 4 to ensure that the verbose-errors feature becomes additive and therefore portus compiles when used as a dependency. There were two classes of changes to the parser structure: 1. Error handling, as predicted and outlined here: https://github.com/Geal/nom/blob/master/doc/upgrading_to_nom_4.md#replacing-parser-result-matchers 2. Migration to "CompleteByteSlice". This was predicted in the migration notes: https://github.com/Geal/nom/blob/master/doc/upgrading_to_nom_4.md#dealing-with-incomplete-usage but caused underlying errors as reported here: rust-bakery/nom#790 (comment) To address this, shadow nom's `named!` macro with our own, `named_complete!`, which replaces the types appropriately. This is the solution proposed here: rust-bakery/nom#795 (comment)
diff --git a/Cargo.toml b/Cargo.toml
@@ -14,7 +14,7 @@ bytes = "0.4.5"
 clap = "2.29"
 libc = "0.2"
 nix = "0.9.0"
-nom = "^3.2"
+nom = "^4"
 slog = "2"
 slog-async = "2"
 slog-term = "2"
diff --git a/src/lang/ast.rs b/src/lang/ast.rs
@@ -1,4 +1,4 @@
-use nom::IResult;
+use nom;
 use super::{Error, Result};
 
 #[derive(Clone, Debug, PartialEq)]
@@ -51,7 +51,7 @@ pub enum Expr {
 }
 
 use std::str;
-named!(
+named_complete!(
     op<Result<Op>>,
     alt!(
         alt!(tag!("+") | tag!("add"))   => { |_| Ok(Op::Add) }     |
@@ -90,7 +90,7 @@ fn check_expr(op: Op, left: Expr, right: Expr) -> Result<Expr> {
 }
 
 use nom::multispace;
-named!(
+named_complete!(
     sexp<Result<Expr>>,
     ws!(delimited!(
         tag!("("),
@@ -111,26 +111,25 @@ named!(
 );
 
 use nom::digit;
+use nom::types::CompleteByteSlice;
 use std::str::FromStr;
-named!(
+named_complete!(
     pub num<u64>,
     map_res!(
-        map_res!(digit, str::from_utf8),
-        FromStr::from_str
+        digit, 
+        |d: CompleteByteSlice| {
+            let st = str::from_utf8(d.0)?;
+            FromStr::from_str(st).map_err(Error::from)
+        }
     )
 );
 
 use nom::is_alphanumeric;
-named!(
-    name_raw<&[u8]>,
-    take_while1!(|u: u8| is_alphanumeric(u) || u == b'.' || u == b'_')
-);
-
-named!(
+named_complete!(
     pub name<String>,
     map_res!(
-        name_raw,
-        |n: &[u8]| str::from_utf8(n).map_err(Error::from).and_then(|s|
+        take_while1!(|u: u8| is_alphanumeric(u) || u == b'.' || u == b'_'),
+        |n: CompleteByteSlice| str::from_utf8(n.0).map_err(Error::from).and_then(|s|
             if s.starts_with("__") {
                 Err(Error::from(
                     format!("Names beginning with \"__\" are reserved for internal use: {:?}", s),
@@ -142,7 +141,7 @@ named!(
     )
 );
 
-named!(
+named_complete!(
     pub atom<Result<Expr>>,
     ws!(do_parse!(
         val: alt!(
@@ -156,7 +155,7 @@ named!(
     ))
 );
 
-named!(
+named_complete!(
     command<Result<Expr>>,
     ws!(delimited!(
         tag!("("),
@@ -171,7 +170,7 @@ named!(
     ))
 );
 
-named!(
+named_complete!(
     pub comment<Result<Expr>>,
     ws!(do_parse!(
         tag!("#") >>
@@ -180,12 +179,12 @@ named!(
     ))
 );
 
-named!(
+named_complete!(
     pub expr<Result<Expr>>,
     alt_complete!(comment | sexp | command | atom)
 );
 
-named!(
+named_complete!(
     pub exprs<Vec<Result<Expr>>>,
     many1!(expr)
 );
@@ -194,14 +193,15 @@ impl Expr {
     // TODO make return Iter
     pub fn new(src: &[u8]) -> Result<Vec<Self>> {
         use nom::Needed;
-        match exprs(src) {
-            IResult::Done(_, me) => me.into_iter().filter(|e| match e {
+        match exprs(CompleteByteSlice(src)) {
+            Ok((_, me)) => me.into_iter().filter(|e| match e {
                 Ok(Expr::None) => false,
                 _ => true,
             }).collect(),
-            IResult::Error(e) => Err(Error::from(e)),
-            IResult::Incomplete(Needed::Unknown) => Err(Error::from("need more src")),
-            IResult::Incomplete(Needed::Size(s)) => Err(
+            Err(nom::Err::Error(e)) |
+            Err(nom::Err::Failure(e)) => Err(Error::from(e)),
+            Err(nom::Err::Incomplete(Needed::Unknown)) => Err(Error::from("need more src")),
+            Err(nom::Err::Incomplete(Needed::Size(s))) => Err(
                 Error::from(format!("need {} more bytes", s)),
             ),
         }
@@ -235,47 +235,77 @@ impl Expr {
 
 #[cfg(test)]
 mod tests {
+    use nom::types::CompleteByteSlice;
     use super::{Command, Expr, Op, Prim};
 
     #[test]
-    fn atom() {
+    fn atom_0() {
+        use super::name;
+        let foo = b"foo";
+        let er = name(CompleteByteSlice(foo));
+        println!("{:?}", er.expect("parse single atom"));
+    }
+
+    #[test]
+    fn atom_1() {
         let foo = b"1";
         let er = Expr::new(foo);
         let e = er.unwrap();
         assert_eq!(e, vec![Expr::Atom(Prim::Num(1))]);
-
+    }
+    
+    #[test]
+    fn atom_2() {
         let foo = b"1 ";
         let er = Expr::new(foo);
         let e = er.unwrap();
         assert_eq!(e, vec![Expr::Atom(Prim::Num(1))]);
+    }
 
+    #[test]
+    fn atom_3() {
         let foo = b"+";
         let er = Expr::new(foo);
         match er {
             Ok(e) => panic!("false ok: {:?}", e),
             Err(_) => (),
         }
+    }
 
+    #[test]
+    fn atom_4() {
         let foo = b"true";
         let er = Expr::new(foo);
         let e = er.unwrap();
         assert_eq!(e, vec![Expr::Atom(Prim::Bool(true))]);
+    }
 
+    #[test]
+    fn atom_5() {
         let foo = b"false";
         let er = Expr::new(foo);
         let e = er.unwrap();
         assert_eq!(e, vec![Expr::Atom(Prim::Bool(false))]);
+    }
 
+    #[test]
+    fn atom_6() {
         let foo = b"x";
         let er = Expr::new(foo);
         let e = er.unwrap();
         assert_eq!(e, vec![Expr::Atom(Prim::Name(String::from("x")))]);
+    }
 
+    #[test]
+    fn atom_7() {
         let foo = b"acbdefg";
         let er = Expr::new(foo);
         let e = er.unwrap();
         assert_eq!(e, vec![Expr::Atom(Prim::Name(String::from("acbdefg")))]);
-        
+    }
+    
+    #[test]
+    fn atom_8() {
         let foo = b"blah 10 20";
         let er = Expr::new(foo);
         let e = er.unwrap();
@@ -288,7 +318,7 @@ mod tests {
             ]
         );
     }
-
+    
     #[test]
     fn simple_exprs() {
         let foo = b"(+ 10 20)";
@@ -353,13 +383,14 @@ mod tests {
 
     #[test]
     fn expr_leftover() {
+        use nom;
         let foo = b"(+ 10 20))";
-        use nom::{IResult, Needed};
+        use nom::Needed;
         use super::exprs;
-        use lang::{Error, Result};
-        match exprs(foo) {
-            IResult::Done(r, me) => {
-                assert_eq!(r, b")");
+        use lang::Result;
+        match exprs(CompleteByteSlice(foo)) {
+            Ok((r, me)) => {
+                assert_eq!(r, CompleteByteSlice(b")"));
                 assert_eq!(
                     me.into_iter().collect::<Result<Vec<Expr>>>().unwrap(),
                     vec![
@@ -371,11 +402,10 @@ mod tests {
                     ],
                 );
             },
-            IResult::Error(e) => panic!(e),
-            IResult::Incomplete(Needed::Unknown) => panic!("need more src"),
-            IResult::Incomplete(Needed::Size(s)) => panic!(
-                Error::from(format!("need {} more bytes", s)),
-            ),
+            Err(nom::Err::Error(e)) | 
+            Err(nom::Err::Failure(e)) => panic!(e),
+            Err(nom::Err::Incomplete(Needed::Unknown)) => panic!("incomplete"),
+            Err(nom::Err::Incomplete(Needed::Size(s))) => panic!("need {} more bytes", s),
         }
     }
 
diff --git a/src/lang/mod.rs b/src/lang/mod.rs
@@ -130,22 +130,47 @@ impl<'a> From<&'a str> for Error {
         Error(String::from(e))
     }
 }
-impl From<nom::simple_errors::Err> for Error {
-    fn from(e: nom::simple_errors::Err) -> Error {
-        Error(String::from(e.description()))
+impl<I, E> From<nom::Err<I, E>> for Error {
+    fn from(e: nom::Err<I, E>) -> Error {
+        Error(String::from(e.into_error_kind().description()))
+    }
+}
+impl<I, E> From<nom::Context<I, E>> for Error {
+    fn from(e: nom::Context<I, E>) -> Error {
+        Error(String::from(e.into_error_kind().description()))
     }
 }
 impl From<std::string::FromUtf8Error> for Error {
     fn from(e: std::string::FromUtf8Error) -> Error {
-        Error(format!("err {}", e))
+        Error(format!("string err {}", e))
     }
 }
 impl From<std::str::Utf8Error> for Error {
     fn from(e: std::str::Utf8Error) -> Error {
-        Error(format!("err {}", e))
+        Error(format!("string err {}", e))
+    }
+}
+impl From<std::num::ParseIntError> for Error {
+    fn from(e: std::num::ParseIntError) -> Error {
+        Error(format!("int err {}", e))
     }
 }
 
+/// Define this helper macro to replace the named! macro provided by nom
+/// to address https://github.com/Geal/nom/issues/790 with CompleteByteSlice
+macro_rules! named_complete {
+    ($name:ident<$t:ty>, $submac:ident!( $($args:tt)* )) => (
+        fn $name( i: nom::types::CompleteByteSlice ) -> nom::IResult<nom::types::CompleteByteSlice, $t, u32> {
+            $submac!(i, $($args)*)
+        }
+    );
+    (pub $name:ident<$t:ty>, $submac:ident!( $($args:tt)* )) => (
+        pub fn $name( i: nom::types::CompleteByteSlice ) -> nom::IResult<nom::types::CompleteByteSlice, $t, u32> {
+            $submac!(i, $($args)*)
+        }
+    )
+}
+
 mod ast;
 mod datapath;
 mod prog;
diff --git a/src/lang/prog.rs b/src/lang/prog.rs
