Add compile-time validation of regular expressions.

So far we would catch pattern errors only at runtime. Now we report
the following at compile time if it can be statically determined:

- Patterns that don't compile (e.g., due to syntax errors)
- Use of `$N` referring to a capture group that exceeds the number a pattern defines.
- Use of `$N' with a regular expression specifying multiple patterns
  for parallel matching (in which case group numbers are ill-defined).
- Use of `$N' with a `&nosub` field.

Closes #2131.

Author: rsmmr

hilti/toolchain/src/compiler/validator.cc

@@ -479,6 +479,11 @@ struct VisitorPost : visitor::PreOrder, public validator::VisitorMixIn {
     void operator()(ctor::Null* n) final {}
 
     void operator()(ctor::RegExp* n) final {
+        for ( const auto& pattern : n->patterns() ) {
+            if ( auto rc = pattern.validate(); ! rc )
+                error(rc.error(), n);
+        }
+
         if ( n->attributes()->find(hilti::attribute::kind::Anchor) )
             // This can end up reporting the same location multiple times,
             // which seems fine. Otherwise we'd need to explicitly track what's

spicy/toolchain/src/compiler/validator.cc

@@ -1237,21 +1237,59 @@ struct VisitorPost : visitor::PreOrder, hilti::validator::VisitorMixIn {
         // We check the original type since regexps get parsed as bytes.
         if ( n->kind() == hilti::expression::keyword::Kind::Captures ) {
             UnqualifiedType* original_type = nullptr;
+            const type::unit::item::Field* field = nullptr;
 
             // Check type in hook bodies.
             if ( auto* hook = n->parent<declaration::Hook>() ) {
                 auto idx = hook->unitFieldIndex();
-                auto* field = context()->lookup(idx)->as<type::unit::item::Field>();
+                field = context()->lookup(idx)->as<type::unit::item::Field>();
                 original_type = field->originalType()->type();
             }
 
-            // Captures can also appear in field attributes.
-            else if ( auto* field = n->parent<type::unit::item::Field>() )
-                original_type = field->originalType()->type();
+            else {
+                // Captures can also appear in field attributes.
+                field = n->parent<type::unit::item::Field>();
+
+                if ( field )
+                    original_type = field->originalType()->type();
+            }
 
             // In all other cases, or when we are not parsing a regexp raise an error.
-            if ( ! original_type || ! original_type->isA<hilti::type::RegExp>() )
+            if ( ! original_type || ! original_type->isA<hilti::type::RegExp>() ) {
                 error("capture groups can only be used in hooks for fields parsing regexp", n);
+                return;
+            }
+
+            if ( field ) {
+                if ( field->attributes()->find(attribute::kind::Nosub) ) {
+                    error("cannot use capture group with &nosub field", n);
+                    return;
+                }
+
+                if ( const auto* regexp = field->ctor()->tryAs<hilti::ctor::RegExp>() ) {
+                    if ( regexp->patterns().size() > 1 ) {
+                        error("cannot use capture groups with regular expression consisting of multiple patterns", n);
+                        return;
+                    }
+
+                    const auto* index_operator = n->parent()->as<hilti::operator_::vector::IndexNonConst>();
+                    auto index = index_operator->op1()
+                                     ->as<hilti::expression::Ctor>()
+                                     ->ctor()
+                                     ->as<hilti::ctor::UnsignedInteger>()
+                                     ->value();
+
+                    assert(! regexp->patterns().empty());
+                    const auto& pattern = regexp->patterns().front();
+
+                    if ( auto captures = pattern.numberOfCaptures();
+                         captures && index > *captures ) // ignore errors here, will be reported elsewhere
+                        error(fmt("capture group index %" PRIu64
+                                  " is too large, the regular expression defines only %" PRIu64 " group%s",
+                                  index, *captures, (*captures > 1 ? "s" : "")),
+                              n);
+                }
+            }
         }
     }
 };

tests/Baseline/hilti.types.regexp.ctor-fail/output

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[error] <...>/ctor-fail.hlt:8:21-8:26: error compiling pattern 'gh(i': bad pattern: syntax error
+[error] <...>/ctor-fail.hlt:11:17-11:22: error compiling pattern 'gh(i': bad pattern: syntax error
+[error] hiltic: aborting after errors

tests/Baseline/spicy.types.regexp.capture-groups-fail/output

@@ -0,0 +1,9 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[error] <...>/capture-groups-fail.spicy:32:7-32:8: capture groups can only be used in hooks for fields parsing regexp
+[error] <...>/capture-groups-fail.spicy:10:15-10:16: capture groups can only be used in hooks for fields parsing regexp
+[error] <...>/capture-groups-fail.spicy:14:15-14:16: capture group index 2 is too large, the regular expression defines only 1 group
+[error] <...>/capture-groups-fail.spicy:18:15-18:16: cannot use capture groups with regular expression consisting of multiple patterns
+[error] <...>/capture-groups-fail.spicy:22:15-22:16: cannot use capture group with &nosub field
+[error] <...>/capture-groups-fail.spicy:25:10-25:17: error compiling pattern '<...>/': bad pattern: syntax error
+[error] <...>/capture-groups-fail.spicy:29:11-29:12: capture groups can only be used in hooks for fields parsing regexp
+[error] spicyc: aborting after errors

tests/Baseline/spicy.types.regexp.capture-without-regexp/output

@@ -0,0 +1,14 @@
+# @TEST-EXEC-FAIL: hiltic -p %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+#
+# @TEST-DOC: Check that we capture invalid regular expression constructors at compile time.
+
+module Foo {
+
+global regexp re1 = /gh(i/;
+
+function void f() {
+    local re2 = /gh(i/;
+}
+
+}

tests/Baseline/spicy.types.regexp.parse-ctor-captures-no-sub/output

@@ -16,31 +16,19 @@ assert |*data.match(/\x00{0,3}/)| == 3;
 assert |*data.match(/\x00{0,4}/)| == 4;
 assert |*data.match(/\x00{0,5}/)| == 5;
 
-assert-exception data.match(/\x00{1,0}/);
 assert |*data.match(/\x00{1,1}/)| == 1;
 assert |*data.match(/\x00{1,2}/)| == 2;
 assert |*data.match(/\x00{1,3}/)| == 3;
 assert |*data.match(/\x00{1,4}/)| == 4;
 assert |*data.match(/\x00{1,5}/)| == 5;
 
-assert-exception data.match(/\x00{2,0}/);
-assert-exception data.match(/\x00{2,1}/);
 assert |*data.match(/\x00{2,2}/)| == 2;
 assert |*data.match(/\x00{2,3}/)| == 3;
 assert |*data.match(/\x00{2,4}/)| == 4;
 assert |*data.match(/\x00{2,5}/)| == 5;
 
-assert-exception data.match(/\x00{4,0}/);
-assert-exception data.match(/\x00{4,1}/);
-assert-exception data.match(/\x00{4,2}/);
-assert-exception data.match(/\x00{4,3}/);
 assert |*data.match(/\x00{4,4}/)| == 4;
 assert |*data.match(/\x00{4,5}/)| == 5;
 
-assert-exception data.match(/\x00{5,0}/);
-assert-exception data.match(/\x00{5,1}/);
-assert-exception data.match(/\x00{5,2}/);
-assert-exception data.match(/\x00{5,3}/);
-assert-exception data.match(/\x00{5,4}/);
 assert |*data.match(/\x00{5,5}/)| == 5;
 }

tests/hilti/types/regexp/ctor-fail.hlt

@@ -0,0 +1,32 @@
+# @TEST-DOC: Validate that we reject invaliud usage of captures.
+#
+# @TEST-EXEC-FAIL: spicyc -dj %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+module test;
+
+public type X = unit {
+    x: uint8 {
+        print $1; # not a regexp field
+    }
+
+    y: /a(b)c/ {
+        print $2; # group does not exist
+    }
+
+    z: /a(b)c/ | /d(e)f/ {
+        print $2; # capture groups not supported for parallel matching (because it's ill-defined)
+    }
+
+    nosub: /a(b)c/ &nosub {
+        print $1; # capture groups not available
+    }
+
+    err: /a(b))c/ { } # invalid regexp
+};
+
+function fn() {
+    print $1; # not inside a unit field
+}
+
+print $1; # not inside a unit field