add pipeline support to validation tests

.github/workflows/rust-ci.yml

@@ -130,6 +130,25 @@ jobs:
         with:
           fail_ci_if_error: true
 
+  validation_test:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
+        with:
+          submodules: true
+      - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: dtolnay/rust-toolchain@f7ccc83f9ed1e5b9c81d8a67d7ad1a747e22a561
+        with:
+          toolchain: stable
+      - name: cargo test validation_test
+        run: |
+          cargo test
+        working-directory: ./rust/validation_test
+
   fmt:
     strategy:
       fail-fast: false

.gitignore

@@ -63,4 +63,9 @@ __pycache__/
 collector/cmd/otelarrowcol/otelarrowcol
 tools/pipeline_perf_test/load_generator/loadgen-manifest.yaml.tmp
 
+# validation_test output
+tools/validation_test/target
+tools/validation_test/target/*
+tools/validation_test/Cargo.lock
+
 venv

rust/validation_test/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "validation_test"
+version = "0.1.0"
+edition = "2024"
+description = "Performs validation for encoding/decoding and pipelines with OTLP requests"
+
+
+[dependencies]
+tonic = { version = "0.14", default-features = false, features = [
+    "channel",
+    "codegen",
+    "deflate",
+    "gzip",
+    "router",
+    "server",
+    "transport",
+    "zstd",
+] }
+tonic-middleware = "0.4.0"
+tonic-prost = "0.14"
+prost = "0.14"
+tokio = { version = "1.48.0", features = [ "sync"] }
+
+
+weaver_common = { git = "https://github.com/open-telemetry/weaver.git", tag = "v0.17.0"}
+weaver_forge = { git = "https://github.com/open-telemetry/weaver.git", tag = "v0.17.0" }
+weaver_resolved_schema = { git = "https://github.com/open-telemetry/weaver.git", tag = "v0.17.0"}
+weaver_resolver = { git = "https://github.com/open-telemetry/weaver.git", tag = "v0.17.0"}
+weaver_semconv = { git = "https://github.com/open-telemetry/weaver.git", tag = "v0.17.0"}
+
+
+otap-df-engine = { path = "../otap-dataflow/crates/engine" }
+otap-df-engine-macros = { path = "../otap-dataflow/crates/engine-macros" }
+otap-df-channel = { path = "../otap-dataflow/crates/channel" }
+otap-df-config = { path = "../otap-dataflow/crates/config" }
+otap-df-pdata = { path = "../otap-dataflow/crates/pdata" }
+otap-df-query-engine = { path = "../otap-dataflow/crates/query-engine" }
+otap-df-telemetry = { path = "../otap-dataflow/crates/telemetry" }
+otap-df-telemetry-macros = { path = "../otap-dataflow/crates/telemetry-macros" }
+otap-df-state = { path = "../otap-dataflow/crates/state" }
+otap-df-otap = { path = "../otap-dataflow/crates/otap" }
+
+

rust/validation_test/README.md

@@ -0,0 +1,52 @@
+# Validation Tests
+
+The validation test validates a OTLP message after
+experiencing various processes such as encoding/decoding
+or going through a pipeline.
+
+## Encoding/Decoding Validation
+
+To validate whether encoding/decoding is working properly
+we comparing the input and output to check that they are equal
+
+## Pipeline Validation
+
+To validate pipelines we create a pipeline and check
+the input and output, to confirm that the data has not
+been altered, this assumes that the pipeline doesn't
+use processors that can alter the data. Soon we plan
+to support these processors in the validation test
+
+### Adding pipelines to the validation process
+
+Use the yaml configuration below to define additional
+pipelines just define non transformative processors to
+the defintion and save the configuration under the
+`validation_pipelines` directory
+
+```yaml
+settings:
+  default_pipeline_ctrl_msg_channel_size: 100
+  default_node_ctrl_msg_channel_size: 100
+  default_pdata_channel_size: 100
+
+nodes:
+  receiver:
+    kind: receiver
+    plugin_urn: "urn:otel:otlp:receiver"
+    out_ports:
+      out_port:
+        destinations:
+          - {insert starting processsor}
+        dispatch_strategy: round_robin
+    config:
+      listening_addr: "127.0.0.1:4317"
+
+  {DEFINE PROCESSORS HERE}
+
+  exporter:
+    kind: exporter
+    plugin_urn: "urn:otel:otlp:exporter"
+    config:
+      grpc_endpoint: "http://127.0.0.1:4318"
+```