Refactor Topology query for BanyanDB (apache#10181)

hutaishi · Dec 23, 2022 · 222bd4a · 222bd4a
1 parent 995d790
commit 222bd4a
Show file tree

Hide file tree

Showing 9 changed files with 158 additions and 130 deletions.
diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
@@ -57,6 +57,7 @@
 * Remove unnecessary additional columns when storage is in logical sharding mode.
 * The cluster coordinator support watch mechanism for notifying `RemoteClientManager` and `ServerStatusService`.
 * Fix ServiceMeshServiceDispatcher overwrite ServiceDispatcher debug file when open SW_OAL_ENGINE_DEBUG.
+* Use `groupBy` and `in` operators to optimize topology query for BanyanDB storage plugin.
 
 #### UI
 

diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml
@@ -73,7 +73,8 @@
         <awaitility.version>3.0.0</awaitility.version>
         <httpcore.version>4.4.13</httpcore.version>
         <commons-compress.version>1.21</commons-compress.version>
-        <banyandb-java-client.version>0.2.1</banyandb-java-client.version>
+        <!-- TODO: use 0.3.0 before OAP 9.4.0 release -->
+        <banyandb-java-client.version>0.3.0-SNAPSHOT</banyandb-java-client.version>
         <kafka-clients.version>2.8.1</kafka-clients.version>
         <spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version>
     </properties>

diff --git a/...ng/oap/server/core/analysis/manual/relation/process/ProcessRelationClientSideMetrics.java b/...ng/oap/server/core/analysis/manual/relation/process/ProcessRelationClientSideMetrics.java
@@ -30,6 +30,7 @@
 import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
 import org.apache.skywalking.oap.server.core.storage.ShardingAlgorithm;
 import org.apache.skywalking.oap.server.core.storage.StorageID;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
 import org.apache.skywalking.oap.server.core.storage.annotation.SQLDatabase;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@@ -67,10 +68,12 @@ public class ProcessRelationClientSideMetrics extends Metrics {
     @Setter
     @Getter
     @Column(columnName = ENTITY_ID, length = 512)
+    @BanyanDB.SeriesID(index = 0)
     private String entityId;
     @Setter
     @Getter
     @Column(columnName = COMPONENT_ID, storageOnly = true)
+    @BanyanDB.SeriesID(index = 1)
     private int componentId;
 
     @Override

diff --git a/...ng/oap/server/core/analysis/manual/relation/process/ProcessRelationServerSideMetrics.java b/...ng/oap/server/core/analysis/manual/relation/process/ProcessRelationServerSideMetrics.java
@@ -30,6 +30,7 @@
 import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
 import org.apache.skywalking.oap.server.core.storage.ShardingAlgorithm;
 import org.apache.skywalking.oap.server.core.storage.StorageID;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
 import org.apache.skywalking.oap.server.core.storage.annotation.SQLDatabase;
 import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
@@ -66,10 +67,12 @@ public class ProcessRelationServerSideMetrics extends Metrics {
     @Setter
     @Getter
     @Column(columnName = ENTITY_ID, length = 512)
+    @BanyanDB.SeriesID(index = 0)
     private String entityId;
     @Setter
     @Getter
     @Column(columnName = COMPONENT_ID, storageOnly = true)
+    @BanyanDB.SeriesID(index = 1)
     private int componentId;
 
     @Override

diff --git a/...ng/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java b/...ng/oap/server/core/analysis/manual/relation/service/ServiceRelationClientSideMetrics.java
@@ -29,6 +29,7 @@
 import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
 import org.apache.skywalking.oap.server.core.storage.ShardingAlgorithm;
 import org.apache.skywalking.oap.server.core.storage.StorageID;
+import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
 import org.apache.skywalking.oap.server.core.storage.annotation.Column;
 import org.apache.skywalking.oap.server.core.storage.annotation.ElasticSearch;
 import org.apache.skywalking.oap.server.core.storage.annotation.SQLDatabase;
@@ -64,10 +65,12 @@ public class ServiceRelationClientSideMetrics extends Metrics {
     @Getter
     @Column(columnName = COMPONENT_IDS, storageOnly = true)
     @ElasticSearch.Keyword
+    @BanyanDB.SeriesID(index = 1)
     private IntList componentIds = new IntList(3);
     @Setter
     @Getter
     @Column(columnName = ENTITY_ID, length = 512)
+    @BanyanDB.SeriesID(index = 0)
     private String entityId;
 
     @Override

diff --git a/...ng/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java b/...ng/oap/server/core/analysis/manual/relation/service/ServiceRelationServerSideMetrics.java
@@ -67,6 +67,7 @@ public class ServiceRelationServerSideMetrics extends Metrics {
     @Getter
     @Column(columnName = COMPONENT_IDS, storageOnly = true)
     @ElasticSearch.Keyword
+    @BanyanDB.SeriesID(index = 1)
     private IntList componentIds = new IntList(3);
     @Setter
     @Getter

diff --git a/...pache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java b/...pache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBTopologyQueryDAO.java
diff --git a/.../org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java b/.../org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java
@@ -134,6 +134,14 @@ protected PairQueryCondition<String> eq(String name, String value) {
             return PairQueryCondition.StringQueryCondition.eq(name, value);
         }
 
+        protected PairQueryCondition<List<String>> in(String name, List<String> values) {
+            return PairQueryCondition.StringArrayQueryCondition.in(name, values);
+        }
+
+        protected PairQueryCondition<List<String>> notIn(String name, List<String> values) {
+            return PairQueryCondition.StringArrayQueryCondition.in(name, values);
+        }
+
         protected PairQueryCondition<Long> ne(String name, long value) {
             return PairQueryCondition.LongQueryCondition.ne(name, value);
         }

diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env
@@ -23,6 +23,6 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449
 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016
 SW_KUBERNETES_COMMIT_SHA=b670c41d94a82ddefcf466d54bab5c492d88d772
 SW_ROVER_COMMIT=d956eaede57b62108b78bca48045bd09ba88e653
-SW_BANYANDB_COMMIT=005b02210caacee0141de8085edebed367ef5a6f
+SW_BANYANDB_COMMIT=e7b08bea242e76c68950509529339995ac0646df
 
 SW_CTL_COMMIT=e684fae0107045fc23799146d62f04cb68bd5a3b