add .consume(), tidy slice functioning

src/tokens.rs

@@ -1056,6 +1056,32 @@ pub trait Tokens: Sized {
         self.optional(|t| t.next().is_none().then_some(()))
             .is_some()
     }
+
+    /// Consume all remaining tokens. This is expected to be used in conjunction
+    /// with combinators like[`Tokens::take`] and [`Tokens::take_while`].
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// use yap::{Tokens, IntoTokens};
+    ///
+    /// let mut toks = "abc123def".into_tokens();
+    ///
+    /// // Take won't do anything unless it's consumed:
+    /// toks.take(3);
+    /// assert_eq!(toks.remaining(), "abc123def");
+    ///
+    /// // We can consume it like a normal iterator, or we
+    /// // can just use this consume call to drain it:
+    /// toks.take(3).consume();
+    /// assert_eq!(toks.remaining(), "123def");
+    ///
+    /// toks.take_while(|t| t.is_numeric()).consume();
+    /// assert_eq!(toks.remaining(), "def");
+    /// ```
+    fn consume(&mut self) {
+        while self.next().is_some() {}
+    }
 }
 
 /// Calling [`Tokens::location()`] returns an object that implements this trait.

src/tokens/slice.rs

@@ -4,7 +4,6 @@ use crate::Tokens;
 pub struct Slice<'a, T: Tokens> {
     tokens: &'a mut T,
     original: T::Location,
-    started: bool,
     from: T::Location,
     to: T::Location,
 }
@@ -16,12 +15,14 @@ impl<'a, T: Tokens> Slice<'a, T> {
         from: T::Location,
         to: T::Location,
     ) -> Slice<'a, T> {
+        // shift tokens to the start location. Dropping
+        // this will reset them back to the original one.
+        tokens.set_location(from.clone());
         Slice {
             tokens,
-            original: current,
             from,
+            original: current,
             to,
-            started: false,
         }
     }
 }
@@ -32,14 +33,6 @@ impl<'a, T: Tokens> Tokens for Slice<'a, T> {
     type Location = T::Location;
 
     fn next(&mut self) -> Option<Self::Item> {
-        // Initial prep:
-        if !self.started {
-            self.tokens.set_location(self.from.clone());
-            self.started = true;
-        }
-
-        // Once we hit the "to" location we return None
-        // (this isn't an inclusive iterator).
         if self.tokens.is_at_location(&self.to) {
             None
         } else {
@@ -56,7 +49,7 @@ impl<'a, T: Tokens> Tokens for Slice<'a, T> {
         self.tokens.is_at_location(location)
     }
 
-    // Delegate to root Tokens impl to allow optimisation.
+    // Allow toks.slice(..).parse(..) to be optimised.
     fn parse<Out, Buf>(&mut self) -> Result<Out, <Out as core::str::FromStr>::Err>
     where
         Out: core::str::FromStr,
@@ -68,7 +61,7 @@ impl<'a, T: Tokens> Tokens for Slice<'a, T> {
         // If the underlying parse worked, then "consume" the slice in case
         // we try using it again. If it didn't work then leave as is.
         if res.is_ok() {
-            self.from = self.to.clone();
+            self.set_location(self.to.clone());
         }
         res
     }
@@ -92,14 +85,20 @@ mod test {
 
         // get locations for testing:
         let from = toks.location();
-        toks.take_while(|t| t.is_numeric()).as_iter().for_each(drop);
+        toks.take_while(|t| t.is_numeric()).consume();
         let to = toks.location();
 
-        let mut s = toks.slice(from, to);
+        // Advance 1 beyond the slice end:
+        toks.take(1).consume();
 
+        let mut s = toks.slice(from, to);
         let n = s.parse::<u16, String>().unwrap();
         assert_eq!(n, 345);
         assert_eq!(s.as_iter().collect::<String>(), "");
+
+        // We should be where we were originally (1 after slice)
+        drop(s);
+        assert_eq!(toks.as_iter().collect::<String>(), "bc");
     }
 
     #[test]
@@ -108,12 +107,19 @@ mod test {
 
         // get locations for testing:
         let from = toks.location();
-        toks.take_while(|t| t.is_numeric()).as_iter().for_each(drop);
+        toks.take_while(|t| t.is_numeric()).consume();
         let to = toks.location();
 
-        let mut s = toks.slice(from, to);
+        // Advance 1 beyond the slice end:
+        toks.take(1).consume();
 
+        // Try the slice:
+        let mut s = toks.slice(from, to);
         s.parse::<u8, String>().unwrap_err();
         assert_eq!(s.as_iter().collect::<String>(), "345");
+
+        // We should be where we were originally (1 after slice)
+        drop(s);
+        assert_eq!(toks.as_iter().collect::<String>(), "bc");
     }
 }

src/types.rs

@@ -193,7 +193,7 @@ impl<'a> Tokens for StrTokens<'a> {
     {
         // Consume the n tokens.
         let from = self.location();
-        self.take(n).as_iter().for_each(drop);
+        self.take(n).consume();
         let to = self.location();
 
         let res = self.str[from.0..to.0].parse();
@@ -216,7 +216,7 @@ impl<'a> Tokens for StrTokens<'a> {
     {
         // Consume all of the tokens matching the function.
         let from = self.location();
-        self.take_while(take_while).as_iter().for_each(drop);
+        self.take_while(take_while).consume();
         let to = self.location();
 
         let res = self.str[from.0..to.0].parse();