[IcebergIO] load tests

.github/trigger_files/IO_Iceberg_Performance_Tests.json

@@ -0,0 +1,4 @@
+{
+  "comment": "Modify this file in a trivial way to cause this test suite to run",
+  "modification": 1
+}

.github/workflows/IO_Iceberg_Performance_Tests.yml

@@ -75,4 +75,4 @@ jobs:
       - name: Run IcebergIO Performance Test
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sdks:java:io:iceberg:loadTest
+          gradle-command: :it:iceberg:IcebergBQMSLoadTestLarge --info -DinfluxHost="http://10.128.0.96:8086" -DinfluxDatabase="beam_test_metrics" -DinfluxMeasurement="java_stress_test_iceberg_bqms"

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/IOStressTestBase.java

@@ -22,6 +22,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.MoreObjects;
 import org.joda.time.Instant;
 
 /** Base class for IO Stress tests. */
@@ -83,6 +84,15 @@ public long getPeriodStartMillis() {
     public long getPeriodEndMillis() {
       return periodEndMillis;
     }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(LoadPeriod.class)
+          .add("start_millis", periodStartMillis)
+          .add("end_millis", periodEndMillis)
+          .add("multiplier", loadIncreaseMultiplier)
+          .toString();
+    }
   }
 
   /**
@@ -92,7 +102,7 @@ public long getPeriodEndMillis() {
    */
   protected static class MultiplierDoFn<T> extends DoFn<T, T> {
     private final int startMultiplier;
-    private final long startTimesMillis;
+    private long startTimesMillis;
     private final List<LoadPeriod> loadPeriods;
 
     public MultiplierDoFn(int startMultiplier, List<LoadPeriod> loadPeriods) {
@@ -101,12 +111,16 @@ public MultiplierDoFn(int startMultiplier, List<LoadPeriod> loadPeriods) {
       this.loadPeriods = loadPeriods;
     }
 
+    @Setup
+    public void start() {
+      this.startTimesMillis = Instant.now().getMillis();
+    }
+
     @DoFn.ProcessElement
-    public void processElement(
-        @Element T element, OutputReceiver<T> outputReceiver, @DoFn.Timestamp Instant timestamp) {
+    public void processElement(@Element T element, OutputReceiver<T> outputReceiver) {
 
       int multiplier = this.startMultiplier;
-      long elapsedTimeMillis = timestamp.getMillis() - startTimesMillis;
+      long elapsedTimeMillis = Instant.now().getMillis() - startTimesMillis;
 
       for (LoadPeriod loadPeriod : loadPeriods) {
         if (elapsedTimeMillis >= loadPeriod.getPeriodStartMillis()

it/iceberg/build.gradle

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import org.apache.beam.gradle.IoPerformanceTestUtilities
+
+plugins { id 'org.apache.beam.module' }
+applyJavaNature(
+        automaticModuleName: 'org.apache.beam.it.iceberg',
+        exportJavadoc: false,
+)
+
+description = "Apache Beam :: IT :: Iceberg"
+ext.summary = "Integration test utilities for Iceberg."
+
+dependencies {
+    testImplementation project(path: ":it:common")
+    testImplementation project(path: ":it:google-cloud-platform")
+    testImplementation project(path: ":sdks:java:io:synthetic")
+    testImplementation project(path: ":sdks:java:managed")
+    testImplementation library.java.google_api_services_bigquery
+
+    testImplementation project(path: ":sdks:java:io:iceberg")
+    testImplementation "org.apache.iceberg:iceberg-core:1.6.1"
+    testImplementation library.java.hadoop_common
+    testRuntimeOnly project(path: ":sdks:java:io:iceberg:bqms", configuration: "shadow")
+    testRuntimeOnly library.java.hadoop_auth
+    testRuntimeOnly library.java.hadoop_client
+    testRuntimeOnly library.java.bigdataoss_gcs_connector
+
+    testImplementation project(path: ":runners:direct-java")
+    testImplementation project(path: ":runners:google-cloud-dataflow-java")
+    testImplementation project(path: ":sdks:java:testing:test-utils")
+    testImplementation project(path: ":sdks:java:extensions:google-cloud-platform-core", configuration: "testRuntimeMigration")
+    testImplementation project(":sdks:java:io:google-cloud-platform")
+    testImplementation library.java.google_api_services_dataflow
+}
+
+tasks.register(
+        "IcebergBQMSLoadTestMedium", IoPerformanceTestUtilities.IoPerformanceTest, project, 'iceberg', 'IcebergIOBigQueryMetastoreST',
+        ['configuration':'medium','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
+                + System.properties
+)
+
+tasks.register(
+        "IcebergBQMSLoadTestLarge", IoPerformanceTestUtilities.IoPerformanceTest, project, 'iceberg', 'IcebergIOBigQueryMetastoreST',
+        ['configuration':'large','project':'apache-beam-testing', 'artifactBucket':'io-performance-temp']
+                + System.properties
+)
+