[GRPC] Simple Transaction Filtering

Author: CapCap

Cargo.lock

@@ -123,6 +123,7 @@ members = [
     "ecosystem/indexer-grpc/indexer-grpc-server-framework",
     "ecosystem/indexer-grpc/indexer-grpc-table-info",
     "ecosystem/indexer-grpc/indexer-grpc-utils",
+    "ecosystem/indexer-grpc/transaction-filter",
     "ecosystem/nft-metadata-crawler-parser",
     "ecosystem/node-checker",
     "ecosystem/node-checker/fn-check-client",
@@ -428,6 +429,7 @@ aptos-storage-service-notifications = { path = "state-sync/inter-component/stora
 aptos-storage-service-types = { path = "state-sync/storage-service/types" }
 aptos-storage-service-server = { path = "state-sync/storage-service/server" }
 aptos-system-utils = { path = "crates/aptos-system-utils" }
+aptos-transaction-filter = { path = "ecosystem/indexer-grpc/transaction-filter" }
 aptos-telemetry = { path = "crates/aptos-telemetry" }
 aptos-telemetry-service = { path = "crates/aptos-telemetry-service" }
 aptos-temppath = { path = "crates/aptos-temppath" }
@@ -534,6 +536,7 @@ datatest-stable = "0.1.1"
 debug-ignore = { version = "1.0.3", features = ["serde"] }
 derivative = "2.2.0"
 derivation-path = "0.2.0"
+derive_builder = "0.20.0"
 determinator = "0.12.0"
 derive_more = "0.99.11"
 diesel = "2.1"
@@ -619,7 +622,7 @@ libfuzzer-sys = "0.4.6"
 libsecp256k1 = "0.7.0"
 log = "0.4.17"
 lru = "0.7.5"
-lz4 = "1.24.0"
+lz4 = "1.25.0"
 maplit = "1.0.2"
 merlin = "3"
 memory-stats = "1.0.0"

Cargo.toml

@@ -0,0 +1,3 @@
+fixtures/compressed_files_lz4_00008bc1d5adcf862d3967c1410001fb_705101000.pb.lz4 filter=lfs diff=lfs merge=lfs -text
+fixtures/compressed_files_lz4_0013c194ec4fdbfb8db7306170aac083_445907000.pb.lz4 filter=lfs diff=lfs merge=lfs -text
+fixtures/compressed_files_lz4_f3d880d9700c70d71fefe71aa9218aa9_301616000.pb.lz4 filter=lfs diff=lfs merge=lfs -text

ecosystem/indexer-grpc/transaction-filter/.gitattributes

@@ -0,0 +1,31 @@
+[package]
+name = "aptos-transaction-filter"
+version = "0.1.0"
+
+# Workspace inherited keys
+authors = { workspace = true }
+edition = { workspace = true }
+homepage = { workspace = true }
+license = { workspace = true }
+publish = { workspace = true }
+repository = { workspace = true }
+rust-version = { workspace = true }
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = { workspace = true }
+aptos-protos = { workspace = true }
+
+derive_builder = { workspace = true }
+
+prost = { workspace = true }
+
+serde = { workspace = true }
+serde_json = { workspace = true }
+serde_yaml = { workspace = true }
+
+thiserror = { workspace = true }
+
+[dev-dependencies]
+# we only decompress the fixture protos in test
+lz4 = { workspace = true }

ecosystem/indexer-grpc/transaction-filter/Cargo.toml

@@ -0,0 +1,145 @@
+# Transaction Filter
+
+## Overview
+
+The goal of **transaction filtering** is to be able to save resources downstream of wherever filtering is used.
+For this to be true, the filtering itself must be **fast and use minimal resources**, and so we do a few things:
+
+1. We avoid clones, copies, etc as much as possible
+2. We do a single pass over the transaction data
+
+## Transaction Filtering
+
+There are a few different parts of a transaction that are queryable:
+
+1. The "root" level. This includes:
+    - Transaction type
+    - Success
+2. User Transactions. Each user transaction has:
+    - Sender
+    - Payload: we only support the entry function payload
+    - Entry function (address, module, name)
+    - Entry function ID string
+3. Events. Each event has:
+    - Key
+    - Type
+
+### Usage & Examples
+
+There are two different patterns for building a filter- you can either use the `TransactionFilterBuilder` or
+the `TransactionFilter` struct directly.
+
+The `TransactionFilterBuilder` is a more ergonomic way to build a filter, and is not significantly worse construction
+performance, assuming this is being done infrequently.
+
+```
+  use transaction_filter::filters::EventFilterBuilder;
+
+  let ef = EventFilterBuilder::default()
+      .data("spins")
+      .struct_type(
+          MoveStructTagFilterBuilder::default()
+              .address("0x0077")
+              .module("roulette")
+              .name("spin")
+              .build()?,
+      )
+      .build()?;
+```
+
+The `TransactionFilter` struct is also available, but requires direct construction of the structs.
+
+```
+  use transaction_filter::filters::EventFilter;
+
+  let ef = EventFilter {
+      data: Some("spins".into()),
+      struct_type: Some(MoveStructTagFilter {
+          address: Some("0x0077".into()),
+          module: Some("roulette".into()),
+          name: Some("spin".into()),
+      }),
+  };
+```
+
+Once you have some filters built, you can combine them with the boolean operators `and`, `or`, and `not`.
+
+```
+   let trf = TransactionRootFilterBuilder::default()
+      .success(true).build()?;
+     
+   let utf = UserTransactionFilterBuilder::default()
+      .sender("0x0011".into()).build()?;
+   
+   let ef = EventFilterBuilder::default()
+      .struct_type(
+          MoveStructTagFilterBuilder::default()
+              .address("0x0077")
+              .module("roulette")
+              .name("spin")
+              .build()?,
+      )
+      .build()?;
+   
+   // Combine filters using logical operators!
+   // (trf OR utf)
+   let trf_or_utf = BooleanTransactionFilter::from(trf).or(utf);
+   // ((trf OR utf) AND ef)
+   let query = trf_or_utf.and(ef);
+    
+   let transactions: Vec<Transaction> = transaction_stream.next().await;
+   let filtered_transactions = query.filter_vec(transactions);
+```
+
+## API & Serialization
+
+`BooleanTransactionFilter` is the top level filter struct, and it uses `serde` for serialization and deserialization.
+
+This means we can use it across all of our projects, whether they be GRPC services, REST services, or CLI tools.
+
+The above example can be serialized to JSON like so:
+
+```json
+{
+  "and": [
+    {
+      "or": [
+        {
+          "type": "TransactionRootFilter",
+          "success": true
+        },
+        {
+          "type": "UserTransactionFilter",
+          "sender": "0x0011"
+        }
+      ]
+    },
+    {
+      "type": "EventFilter",
+      "struct_type": {
+        "address": "0x0077",
+        "module": "roulette",
+        "name": "spin"
+      }
+    }
+  ]
+}
+```
+
+Or, if you prefer, as yaml:
+
+```yaml
+---
+and:
+  - or:
+      - type: TransactionRootFilter
+        success: true
+      - type: UserTransactionFilter
+        sender: '0x0011'
+  - type: EventFilter
+    struct_type:
+      address: '0x0077'
+      module: roulette
+      name: spin
+```
+

ecosystem/indexer-grpc/transaction-filter/README.md

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dedf23a08bc7bd9fc9c96dc5d45741fe52c225ccfc2f7c72030240585a531dc3
+size 465907

ecosystem/indexer-grpc/transaction-filter/fixtures/compressed_files_lz4_00008bc1d5adcf862d3967c1410001fb_705101000.pb.lz4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:596c2e0582123c36e06c3be8ded161afa63b39c742acdbf279da8d97bfa8ccd3
+size 3039165

ecosystem/indexer-grpc/transaction-filter/fixtures/compressed_files_lz4_0013c194ec4fdbfb8db7306170aac083_445907000.pb.lz4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55750a530523ed097005e2c95e832d2f1ef8c50f535a65f3d4d025073f635e9a
+size 40801877