Merge pull request #106 from google/cpp-sync

internal changes
(some fixes might be in older sync)
Fixes #88
Fixes #83
Fixes #82
Fixes #66
Fixes #57
Fixes #21
Fixes #15

Author: kyessenov

base/BUILD

@@ -10,3 +10,17 @@ cc_library(
         "status_macros.h",
     ],
 )
+
+cc_library(
+    name = "unilib",
+    srcs = [
+        "unilib.cc",
+    ],
+    hdrs = [
+        "unilib.h",
+    ],
+    deps = [
+        "@com_github_google_flatbuffers//:flatbuffers",
+        "@com_google_absl//absl/strings",
+    ],
+)

base/unilib.cc

@@ -0,0 +1,19 @@
+#include "base/unilib.h"
+
+#include "flatbuffers/util.h"
+
+namespace UniLib {
+
+// Detects whether a string is valid UTF-8.
+bool IsStructurallyValid(absl::string_view str) {
+  const char *s = &str[0];
+  const char *const sEnd = s + str.length();
+  while (s < sEnd) {
+    if (flatbuffers::FromUTF8(&s) < 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace UniLib

base/unilib.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef THIRD_PARTY_CEL_CPP_BASE_UNILIB_H_
+#define THIRD_PARTY_CEL_CPP_BASE_UNILIB_H_
+
+#include "absl/strings/string_view.h"
+
+namespace UniLib {
+
+// Detects whether a string is valid UTF-8.
+bool IsStructurallyValid(absl::string_view str);
+
+}  // namespace UniLib
+
+#endif  // THIRD_PARTY_CEL_CPP_BASE_UNILIB_H_

common/type.cc

@@ -72,7 +72,7 @@ UnrecognizedType::UnrecognizedType(absl::string_view full_name)
     : string_rep_(absl::StrCat("type(\"", full_name, "\")")),
       hash_code_(internal::Hash(full_name)) {
   assert(google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(
-             std::string(full_name)) == nullptr);
+             full_name.data()) == nullptr);
 }
 
 absl::string_view UnrecognizedType::full_name() const {
@@ -89,7 +89,7 @@ Type::Type(const std::string& full_name)
 
   auto obj_desc =
       google::protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(
-          full_name);
+          full_name.data());
   if (obj_desc != nullptr) {
     data_ = ObjectType(obj_desc);
     return;

common/value.h

@@ -330,7 +330,7 @@ class Container : public SharedValue {
   }
   template <Value::Kind ValueKind, typename V>
   static Value GetValue(V&& value) {
-    return Value::From<ValueKind>(std::move(value));
+    return Value::From<ValueKind>(std::forward<V>(value));
   }
 
  private:

conformance/BUILD

@@ -19,9 +19,8 @@ ALL_TESTS = [
     "@com_google_cel_spec//tests/simple:testdata/macros.textproto",
     "@com_google_cel_spec//tests/simple:testdata/namespace.textproto",
     "@com_google_cel_spec//tests/simple:testdata/plumbing.textproto",
-    # TODO(issues/92): Support for parse-only proto message creation within a container.
-    # "@com_google_cel_spec//tests/simple:testdata/proto2.textproto",
-    # "@com_google_cel_spec//tests/simple:testdata/proto3.textproto",
+    "@com_google_cel_spec//tests/simple:testdata/proto2.textproto",
+    "@com_google_cel_spec//tests/simple:testdata/proto3.textproto",
     "@com_google_cel_spec//tests/simple:testdata/string.textproto",
     "@com_google_cel_spec//tests/simple:testdata/timestamps.textproto",
     "@com_google_cel_spec//tests/simple:testdata/unknowns.textproto",
@@ -91,18 +90,11 @@ cc_binary(
             # uncommented when the spec changes to truncation rather than rounding.
             "--skip_test=conversions/int/double_nearest,double_nearest_neg,double_half_away_neg,double_half_away_pos",
             "--skip_test=conversions/uint/double_nearest,double_nearest_int,double_half_away",
-            # TODO(issues/82): Unexpected behavior when converting invalid bytes to string.
-            "--skip_test=conversions/string/bytes_invalid",
-            # TODO(issues/83): Missing type() conversion functions
-            "--skip_test=conversions/type",
             # TODO(issues/96): Well-known type conversion support.
             "--skip_test=proto2/literal_wellknown",
             "--skip_test=proto3/literal_wellknown",
             "--skip_test=proto2/empty_field/wkt",
             "--skip_test=proto3/empty_field/wkt",
-            # TODO(issues/92): Support for parse-only proto message creation within a container.
-            "--skip_test=proto2/has/undefined",
-            "--skip_test=proto3/has/undefined",
             # Requires container support
             "--skip_test=namespace/namespace/self_eval_container_lookup_unchecked",
             "--skip_test=basic/self_eval_nonzeroish/self_eval_bytes_invalid_utf8",

conformance/server.cc

@@ -25,8 +25,6 @@
 #include "proto/test/v1/proto3/test_all_types.pb.h"
 
 
-using absl::Status;
-using absl::StatusCode;
 using ::google::protobuf::Arena;
 using ::google::protobuf::util::JsonStringToMessage;
 using ::google::protobuf::util::MessageToJsonString;
@@ -42,10 +40,10 @@ class ConformanceServiceImpl {
  public:
   explicit ConformanceServiceImpl(std::unique_ptr<CelExpressionBuilder> builder)
       : builder_(std::move(builder)),
-        proto2Tests_(&google::api::expr::test::v1::proto2::TestAllTypes::
-                         default_instance()),
-        proto3Tests_(&google::api::expr::test::v1::proto3::TestAllTypes::
-                         default_instance()) {}
+        proto2_tests_(&google::api::expr::test::v1::proto2::TestAllTypes::
+                          default_instance()),
+        proto3_tests_(&google::api::expr::test::v1::proto3::TestAllTypes::
+                          default_instance()) {}
 
   void Parse(const v1alpha1::ParseRequest* request,
              v1alpha1::ParseResponse* response) {
@@ -63,7 +61,7 @@ class ConformanceServiceImpl {
     } else {
       google::api::expr::v1alpha1::ParsedExpr out;
       (out).MergeFrom(parse_status.value());
-      response->mutable_parsed_expr()->CopyFrom(out);
+      *response->mutable_parsed_expr() = out;
     }
   }
 
@@ -87,6 +85,7 @@ class ConformanceServiceImpl {
     google::api::expr::v1alpha1::SourceInfo source_info;
     google::api::expr::v1alpha1::Expr out;
     (out).MergeFrom(*expr);
+    builder_->set_container(request->container());
     auto cel_expression_status = builder_->CreateExpression(&out, &source_info);
 
     if (!cel_expression_status.ok()) {
@@ -144,13 +143,14 @@ class ConformanceServiceImpl {
 
  private:
   std::unique_ptr<CelExpressionBuilder> builder_;
-  const google::api::expr::test::v1::proto2::TestAllTypes* proto2Tests_;
-  const google::api::expr::test::v1::proto3::TestAllTypes* proto3Tests_;
+  const google::api::expr::test::v1::proto2::TestAllTypes* proto2_tests_;
+  const google::api::expr::test::v1::proto3::TestAllTypes* proto3_tests_;
 };
 
 int RunServer(bool optimize) {
   google::protobuf::Arena arena;
   InterpreterOptions options;
+  options.enable_qualified_type_identifiers = true;
 
   if (optimize) {
     std::cerr << "Enabling optimizations" << std::endl;
@@ -160,14 +160,15 @@ int RunServer(bool optimize) {
 
   std::unique_ptr<CelExpressionBuilder> builder =
       CreateCelExpressionBuilder(options);
-  builder->AddResolvableEnum(
+  auto type_registry = builder->GetTypeRegistry();
+  type_registry->Register(
       google::api::expr::test::v1::proto2::GlobalEnum_descriptor());
-  builder->AddResolvableEnum(
+  type_registry->Register(
       google::api::expr::test::v1::proto3::GlobalEnum_descriptor());
-  builder->AddResolvableEnum(google::api::expr::test::v1::proto2::TestAllTypes::
-                                 NestedEnum_descriptor());
-  builder->AddResolvableEnum(google::api::expr::test::v1::proto3::TestAllTypes::
-                                 NestedEnum_descriptor());
+  type_registry->Register(google::api::expr::test::v1::proto2::TestAllTypes::
+                              NestedEnum_descriptor());
+  type_registry->Register(google::api::expr::test::v1::proto3::TestAllTypes::
+                              NestedEnum_descriptor());
   auto register_status = RegisterBuiltinFunctions(builder->GetRegistry());
   if (!register_status.ok()) {
     std::cerr << "Failed to initialize: " << register_status.ToString()

eval/compiler/BUILD

@@ -17,6 +17,7 @@ cc_library(
     deps = [
         ":constant_folding",
         ":qualified_reference_resolver",
+        ":resolver",
         "//base:status_macros",
         "//eval/eval:comprehension_step",
         "//eval/eval:const_value_step",
@@ -30,6 +31,7 @@ cc_library(
         "//eval/eval:jump_step",
         "//eval/eval:logic_step",
         "//eval/eval:select_step",
+        "//eval/eval:shadowable_value_step",
         "//eval/eval:ternary_step",
         "//eval/public:ast_traverse",
         "//eval/public:ast_visitor",
@@ -147,6 +149,7 @@ cc_library(
         "qualified_reference_resolver.h",
     ],
     deps = [
+        ":resolver",
         "//base:status_macros",
         "//eval/eval:const_value_step",
         "//eval/eval:expression_build_warning",
@@ -162,6 +165,21 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "resolver",
+    srcs = ["resolver.cc"],
+    hdrs = ["resolver.h"],
+    deps = [
+        "//eval/public:cel_builtins",
+        "//eval/public:cel_function_registry",
+        "//eval/public:cel_type_registry",
+        "//eval/public:cel_value",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
+        "@com_google_protobuf//:protobuf",
+    ],
+)
+
 cc_test(
     name = "qualified_reference_resolver_test",
     srcs = [
@@ -174,6 +192,7 @@ cc_test(
         "//eval/public:cel_builtins",
         "//eval/public:cel_function",
         "//eval/public:cel_function_registry",
+        "//eval/public:cel_type_registry",
         "//testutil:util",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/types:optional",
@@ -203,3 +222,19 @@ cc_test(
         "@com_google_protobuf//:protobuf",
     ],
 )
+
+cc_test(
+    name = "resolver_test",
+    size = "small",
+    srcs = ["resolver_test.cc"],
+    deps = [
+        ":resolver",
+        "//base:status_macros",
+        "//eval/public:cel_function",
+        "//eval/public:cel_function_registry",
+        "//eval/public:cel_type_registry",
+        "//eval/testutil:test_message_cc_proto",
+        "@com_google_absl//absl/status",
+        "@com_google_googletest//:gtest_main",
+    ],
+)