Feature/rumil tokens 0.6.0

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
 
       - name: Set up monorepo overrides
         run: |
-          for pkg in rumil_parsers rumil_expressions; do
+          for pkg in rumil_parsers rumil_expressions rumil_tokens; do
           cat > $pkg/pubspec_overrides.yaml <<'EOF'
           dependency_overrides:
             rumil:
@@ -37,7 +37,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          for pkg in rumil rumil_codec rumil_parsers rumil_expressions rumil_codec_builder; do
+          for pkg in rumil rumil_codec rumil_parsers rumil_expressions rumil_codec_builder rumil_tokens; do
             (cd $pkg && dart pub get)
           done
 
@@ -47,7 +47,7 @@ jobs:
 
       - name: Analyze
         run: |
-          for pkg in rumil rumil_codec rumil_parsers rumil_expressions rumil_codec_builder; do
+          for pkg in rumil rumil_codec rumil_parsers rumil_expressions rumil_codec_builder rumil_tokens; do
             echo "=== $pkg ==="
             (cd $pkg && dart analyze --fatal-infos)
           done
@@ -63,7 +63,7 @@ jobs:
 
       - name: Set up monorepo overrides
         run: |
-          for pkg in rumil_parsers rumil_expressions; do
+          for pkg in rumil_parsers rumil_expressions rumil_tokens; do
           cat > $pkg/pubspec_overrides.yaml <<'EOF'
           dependency_overrides:
             rumil:
@@ -82,12 +82,12 @@ jobs:
 
       - name: Install dependencies
         run: |
-          for pkg in rumil rumil_codec rumil_parsers rumil_expressions rumil_codec_builder; do
+          for pkg in rumil rumil_codec rumil_parsers rumil_expressions rumil_codec_builder rumil_tokens; do
             (cd $pkg && dart pub get)
           done
 
       - name: Check formatting
-        run: dart format --output=none --set-exit-if-changed rumil rumil_codec rumil_parsers rumil_expressions rumil_codec_builder
+        run: dart format --output=none --set-exit-if-changed rumil rumil_codec rumil_parsers rumil_expressions rumil_codec_builder rumil_tokens
 
   test:
     name: Test
@@ -100,7 +100,7 @@ jobs:
 
       - name: Set up monorepo overrides
         run: |
-          for pkg in rumil_parsers rumil_expressions; do
+          for pkg in rumil_parsers rumil_expressions rumil_tokens; do
           cat > $pkg/pubspec_overrides.yaml <<'EOF'
           dependency_overrides:
             rumil:
@@ -110,7 +110,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          for pkg in rumil rumil_codec rumil_parsers rumil_expressions; do
+          for pkg in rumil rumil_codec rumil_parsers rumil_expressions rumil_tokens; do
             (cd $pkg && dart pub get)
           done
 
@@ -130,6 +130,10 @@ jobs:
         working-directory: rumil_expressions
         run: dart test
 
+      - name: Test rumil_tokens
+        working-directory: rumil_tokens
+        run: dart test
+
   doc:
     name: Documentation
     runs-on: ubuntu-latest
@@ -141,7 +145,7 @@ jobs:
 
       - name: Set up monorepo overrides
         run: |
-          for pkg in rumil_parsers rumil_expressions; do
+          for pkg in rumil_parsers rumil_expressions rumil_tokens; do
           cat > $pkg/pubspec_overrides.yaml <<'EOF'
           dependency_overrides:
             rumil:
@@ -151,13 +155,13 @@ jobs:
 
       - name: Install dependencies
         run: |
-          for pkg in rumil rumil_codec rumil_parsers rumil_expressions; do
+          for pkg in rumil rumil_codec rumil_parsers rumil_expressions rumil_tokens; do
             (cd $pkg && dart pub get)
           done
 
       - name: Generate docs
         run: |
-          for pkg in rumil rumil_codec; do
+          for pkg in rumil rumil_codec rumil_tokens; do
             echo "=== $pkg ==="
             (cd $pkg && dart doc --validate-links)
           done

rumil/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.6.0
+
+Synchronized release across all rumil-dart packages. Additive for
+`rumil`.
+
+- `position()` primitive: a zero-width parser that yields the current
+  byte offset. Combines with `Zip` for span capture:
+  `position().zip(p).zip(position())` produces `((start, value), end)`
+  in one pass.
+
 ## 0.5.0
 
 **Interpreter optimizations and API refinements.**

rumil/lib/src/interpreter.dart

@@ -317,6 +317,9 @@ Result<E, A> interpretI<E, A>(Parser<E, A> parser, ParserState state) {
           loc,
         );
 
+      case GetPosition():
+        return Success<E, A>(state.offset as A, 0);
+
       case Mapped<E, dynamic, A>():
         return _runTrampoline<E, A>(p, state);
 

rumil/lib/src/location.dart

@@ -11,7 +11,7 @@ final class Location {
   /// 0-indexed byte offset from start of input.
   final int offset;
 
-  /// Creates a location at [offset] within [input].
+  /// Creates a location at [offset] within the given input string.
   const Location(this._input, this.offset);
 
   /// The start of input: line 1, column 1, offset 0.

rumil/lib/src/parser.dart

@@ -99,6 +99,19 @@ final class Eof<E> extends Parser<E, void> {
   bool get isSimple => true;
 }
 
+/// Succeeds without consuming input, yielding the current byte offset.
+///
+/// Use via [position] in `primitives.dart`. Typically wrapped into span
+/// tracking: `position().zip(p).zip(position())` gives the start offset,
+/// the parsed value, and the end offset in one pass.
+final class GetPosition<E> extends Parser<E, int> {
+  /// Creates a position-reading parser.
+  const GetPosition();
+
+  @override
+  bool get isSimple => true;
+}
+
 // ---------------------------------------------------------------------------
 // Composition
 // ---------------------------------------------------------------------------

rumil/lib/src/primitives.dart

@@ -112,6 +112,19 @@ Parser<ParseError, String> symbol(String s) => lexeme(string(s));
 /// Matches end of input.
 Parser<ParseError, void> eof() => const Eof<ParseError>();
 
+/// Succeeds without consuming input, yielding the current byte offset.
+///
+/// Combines with [Zip] to capture spans around a parser:
+///
+/// ```dart
+/// final spanned = position<E>().zip(myParser).zip(position<E>());
+/// // produces (((int startOffset, A value), int endOffset))
+/// ```
+///
+/// The offset is 0-indexed. Use [Location] (via `Location(input, offset)`)
+/// when converting to line/column.
+Parser<E, int> position<E>() => GetPosition<E>();
+
 /// Defers parser construction for recursive grammars.
 Parser<E, A> defer<E, A>(Parser<E, A> Function() thunk) => Defer<E, A>(thunk);
 

rumil/pubspec.yaml

@@ -2,7 +2,7 @@ name: rumil
 description: >-
   Parser combinator library for Dart with left recursion, stack-safe
   trampolining, typed errors, lazy error construction, and sealed ADT design.
-version: 0.5.0
+version: 0.6.0
 repository: https://github.com/hakimjonas/rumil-dart
 topics:
   - parser

rumil/test/smoke_test.dart

@@ -42,6 +42,30 @@ void main() {
     test('eof fails with remaining input', () {
       expectFailure(eof().run('x'));
     });
+
+    test('position at start yields 0', () {
+      final r = position<ParseError>().run('abc');
+      expect(successValue(r), 0);
+      expect((r as Success<ParseError, int>).consumed, 0);
+    });
+
+    test('position after consumption yields offset', () {
+      final r = string('abc').skipThen(position<ParseError>()).run('abcdef');
+      expect(successValue(r), 3);
+    });
+
+    test('position captures span around a parser', () {
+      final spanned = spaces()
+          .skipThen(position<ParseError>())
+          .zip(string('hello'))
+          .zip(position<ParseError>());
+      final r = spanned.run('  hello!');
+      expect(r, isA<Success<ParseError, ((int, String), int)>>());
+      final s = r as Success<ParseError, ((int, String), int)>;
+      expect(s.value.$1.$1, 2);
+      expect(s.value.$1.$2, 'hello');
+      expect(s.value.$2, 7);
+    });
   });
 
   group('Composition', () {

rumil_codec/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.0
+
+Version aligned with the rumil-dart monorepo 0.6.0 release. No
+functional changes in this package.
+
 ## 0.5.0
 
 - **New:** `dateTimeCodec` — microsecond precision, preserves UTC/local flag.

rumil_codec/pubspec.yaml

@@ -2,7 +2,7 @@ name: rumil_codec
 description: >-
   Binary codec library for Dart with ZigZag/Varint encoding,
   composable BinaryCodec instances, and product type composition via records.
-version: 0.5.0
+version: 0.6.0
 repository: https://github.com/hakimjonas/rumil-dart
 topics:
   - codec

rumil_codec_builder/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.0
+
+- Depends on `rumil_codec: ^0.6.0`, `rumil_parsers: ^0.6.0`. Version
+  aligned with the rumil-dart monorepo 0.6.0 release.
+
 ## 0.5.0
 
 - Depends on `rumil_codec: ^0.5.0`, `rumil_parsers: ^0.5.0`. Version aligned.

rumil_codec_builder/pubspec.yaml

@@ -2,7 +2,7 @@ name: rumil_codec_builder
 description: >-
   Code generator for rumil_codec: derives BinaryCodec implementations
   for annotated classes and sealed class hierarchies.
-version: 0.5.0
+version: 0.6.0
 repository: https://github.com/hakimjonas/rumil-dart
 topics:
   - codec
@@ -16,11 +16,11 @@ dependencies:
   build: ">=3.0.0 <5.0.0"
   source_gen: ^4.0.0
   analyzer: ">=12.0.0 <13.0.0"
-  rumil_codec: ^0.5.0
+  rumil_codec: ^0.6.0
 
 dev_dependencies:
   build_runner: ">=2.4.0 <3.0.0"
   build_test: ^3.5.0
-  rumil_parsers: ^0.5.0
+  rumil_parsers: ^0.6.0
   test: ^1.25.0
   lints: ^6.0.0

rumil_expressions/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.0
+
+- Depends on `rumil: ^0.6.0`. Version aligned with the rumil-dart
+  monorepo 0.6.0 release. No functional changes in this package.
+
 ## 0.5.0
 
 - Depends on rumil ^0.5.0. Benefits from interpreter optimizations (5-9% AOT, 30-52% WasmGC).

rumil_expressions/pubspec.yaml

@@ -2,7 +2,7 @@ name: rumil_expressions
 description: >-
   Formula evaluator built on Rumil: arithmetic, boolean logic, string ops,
   variables, custom functions, and precise error locations.
-version: 0.5.0
+version: 0.6.0
 repository: https://github.com/hakimjonas/rumil-dart
 topics:
   - parser
@@ -13,7 +13,7 @@ environment:
   sdk: ^3.7.0
 
 dependencies:
-  rumil: ^0.5.0
+  rumil: ^0.6.0
 
 dev_dependencies:
   test: ^1.31.0

rumil_parsers/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.0
+
+- Depends on `rumil: ^0.6.0`. Version aligned with the rumil-dart
+  monorepo 0.6.0 release. No functional changes in this package.
+
 ## 0.5.0
 
 **CommonMark Markdown parser. Architecture audit. 7376 tests.**

rumil_parsers/pubspec.yaml

@@ -2,7 +2,7 @@ name: rumil_parsers
 description: >-
   Format parsers built on Rumil: JSON, CSV, XML, TOML, YAML, Proto3, HCL,
   and CommonMark Markdown, plus typed AST decoders with ObjectAccessor pattern.
-version: 0.5.0
+version: 0.6.0
 repository: https://github.com/hakimjonas/rumil-dart
 topics:
   - parser
@@ -13,7 +13,7 @@ environment:
   sdk: ^3.7.0
 
 dependencies:
-  rumil: ^0.5.0
+  rumil: ^0.6.0
 
 dev_dependencies:
   test: ^1.31.0

rumil_tokens/CHANGELOG.md

@@ -0,0 +1,52 @@
+## 0.1.0
+
+Initial in-tree cut. Source code tokenizer built on Rumil. Not
+published to pub.dev; consumed via path dependency from elsewhere
+in the monorepo.
+
+### Tokens
+
+- Sealed `Token` ADT: `Keyword`, `TypeName`, `StringLit`, `NumberLit`,
+  `Comment`, `Punctuation`, `Operator`, `Variable`, `Identifier`,
+  `Annotation`, `Whitespace`, `Plain`.
+
+### API
+
+- `tokenize(source, grammar)` returns a lossless `List<Token>`;
+  concatenating `token.text` reconstructs the source exactly.
+- `tokenizeSpans(source, grammar)` returns `List<Spanned<Token>>`
+  carrying byte offsets. Spans are half-open `[start, end)`,
+  contiguous, and anchored to `[0, source.length)`.
+- `Spanned<T extends Token>` is an extension type over
+  `(T, int, int)`. Narrow types upcast to wider ones.
+
+### Built-in grammars
+
+- `dart`, `scala`, `yaml`, `json`, `shell`.
+- `grammarFor(name)` returns the matching grammar or `null`.
+
+### `LangGrammar` fields
+
+- Lexical: `keywords`, `types`, `lineComment`, `blockComment`,
+  `stringDelimiters`, `multiLineStringDelimiters`, `annotationPrefix`,
+  `punctuationChars`, `operatorChars`, `multiCharOperators`.
+- Flags: `identifiersAllowDollar`, `rawStringPrefix`,
+  `identifierStringPrefix`, `backtickIdentifiers`, `shellVariables`,
+  `backtickCommandSubstitution`, `heredocs`.
+
+### Known limitations
+
+- YAML block scalars (`|`, `>`) tokenize the indented body as regular
+  YAML content rather than one string literal.
+- Dart string interpolation (`"$x"`, `"${expr}"`) remains one
+  `StringLit`; no structured tokens for the interpolated parts.
+- Shell braced expansions do not balance nested braces: `${x:-${y}}`
+  closes the outer expansion prematurely.
+- Heredoc body is one `StringLit`; per-component coloring is not
+  available.
+- Nested generic close (`List<Map<String, int>>`) highlights the outer
+  `>>` as the right-shift operator.
+
+### Dependencies
+
+- `rumil: ^0.6.0` for the `position()` primitive.

rumil_tokens/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hakim Jonas Ghoula
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.