Skip to content

Commit 0950492

Browse files
author
Arunodoy18
committed
Fix triggeringFrequency documentation and validation for STORAGE_API_AT_LEAST_ONCE
- Updated JavaDoc to reflect that triggeringFrequency applies to FILE_LOADS, STORAGE_WRITE_API, and STORAGE_API_AT_LEAST_ONCE methods - Fixed validation logic in expand() to require triggeringFrequency for STORAGE_API_AT_LEAST_ONCE when writing unbounded PCollections - Removed conflicting warning that incorrectly stated STORAGE_API_AT_LEAST_ONCE ignores triggeringFrequency - Added comprehensive test cases to verify proper validation behavior: * Test failure when triggeringFrequency is missing for unbounded collections * Test success when triggeringFrequency is provided for unbounded collections * Test success for bounded collections without triggeringFrequency requirement - Updated error messages to include all three supported methods This ensures consistent behavior across all BigQuery write methods that support triggered writes for unbounded collections.
1 parent bf37c0f commit 0950492

2 files changed

Lines changed: 79 additions & 14 deletions

File tree

sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIO.java

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3330,8 +3330,9 @@ public Write<T> withLoadJobProjectId(ValueProvider<String> loadJobProjectId) {
33303330
/**
33313331
* Choose the frequency at which file writes are triggered.
33323332
*
3333-
* <p>This is only applicable when the write method is set to {@link Method#FILE_LOADS} or
3334-
* {@link Method#STORAGE_WRITE_API}, and only when writing an unbounded {@link PCollection}.
3333+
* <p>This is only applicable when the write method is set to {@link Method#FILE_LOADS}, {@link
3334+
* Method#STORAGE_WRITE_API}, or {@link Method#STORAGE_API_AT_LEAST_ONCE}, and only when writing
3335+
* an unbounded {@link PCollection}.
33353336
*
33363337
* <p>Every triggeringFrequency duration, a BigQuery load job will be generated for all the data
33373338
* written since the last load job. BigQuery has limits on how many load jobs can be triggered
@@ -3736,19 +3737,22 @@ public WriteResult expand(PCollection<T> input) {
37363737
BigQueryOptions bqOptions = input.getPipeline().getOptions().as(BigQueryOptions.class);
37373738
Write.Method method = resolveMethod(input);
37383739
if (input.isBounded() == IsBounded.UNBOUNDED) {
3739-
if (method == Write.Method.FILE_LOADS || method == Write.Method.STORAGE_WRITE_API) {
3740+
if (method == Write.Method.FILE_LOADS
3741+
|| method == Write.Method.STORAGE_WRITE_API
3742+
|| method == Write.Method.STORAGE_API_AT_LEAST_ONCE) {
37403743
Duration triggeringFrequency =
3741-
(method == Write.Method.STORAGE_WRITE_API)
3744+
(method == Write.Method.STORAGE_WRITE_API
3745+
|| method == Write.Method.STORAGE_API_AT_LEAST_ONCE)
37423746
? getStorageApiTriggeringFrequency(bqOptions)
37433747
: getTriggeringFrequency();
37443748
checkArgument(
37453749
triggeringFrequency != null,
3746-
"When writing an unbounded PCollection via FILE_LOADS or STORAGE_WRITE_API, "
3750+
"When writing an unbounded PCollection via FILE_LOADS, STORAGE_WRITE_API, or STORAGE_API_AT_LEAST_ONCE, "
37473751
+ "triggering frequency must be specified");
37483752
} else {
37493753
checkArgument(
37503754
getTriggeringFrequency() == null,
3751-
"Triggering frequency can be specified only when writing via FILE_LOADS or STORAGE_WRITE_API, but the method was %s.",
3755+
"Triggering frequency can be specified only when writing via FILE_LOADS, STORAGE_WRITE_API, or STORAGE_API_AT_LEAST_ONCE, but the method was %s.",
37523756
method);
37533757
}
37543758
if (method != Method.FILE_LOADS) {
@@ -3757,13 +3761,7 @@ public WriteResult expand(PCollection<T> input) {
37573761
"Number of file shards can be specified only when writing via FILE_LOADS, but the method was %s.",
37583762
method);
37593763
}
3760-
if (method == Method.STORAGE_API_AT_LEAST_ONCE
3761-
&& getStorageApiTriggeringFrequency(bqOptions) != null) {
3762-
LOG.warn(
3763-
"Storage API triggering frequency option will be ignored is it can only be specified only "
3764-
+ "when writing via STORAGE_WRITE_API, but the method was {}.",
3765-
method);
3766-
}
3764+
37673765
if (getAutoSharding()) {
37683766
if (method == Method.STORAGE_WRITE_API && getStorageApiNumStreams(bqOptions) > 0) {
37693767
LOG.warn(

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryIOWriteTest.java

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2632,7 +2632,8 @@ public void testStreamingWriteValidateFailsWithoutTriggeringFrequency() {
26322632
Method method = useStorageApi ? Method.STORAGE_WRITE_API : Method.FILE_LOADS;
26332633

26342634
thrown.expect(IllegalArgumentException.class);
2635-
thrown.expectMessage("unbounded PCollection via FILE_LOADS or STORAGE_WRITE_API");
2635+
thrown.expectMessage(
2636+
"unbounded PCollection via FILE_LOADS, STORAGE_WRITE_API, or STORAGE_API_AT_LEAST_ONCE");
26362637
thrown.expectMessage("triggering frequency must be specified");
26372638

26382639
p.getOptions().as(BigQueryOptions.class).setStorageWriteApiTriggeringFrequencySec(null);
@@ -2646,6 +2647,72 @@ public void testStreamingWriteValidateFailsWithoutTriggeringFrequency() {
26462647
.withCreateDisposition(CreateDisposition.CREATE_NEVER));
26472648
}
26482649

2650+
@Test
2651+
public void testStreamingWriteValidateFailsWithoutTriggeringFrequencyForStorageApiAtLeastOnce() {
2652+
assumeTrue(useStreaming);
2653+
assumeTrue(useStorageApiApproximate); // Test STORAGE_API_AT_LEAST_ONCE specifically
2654+
p.enableAbandonedNodeEnforcement(false);
2655+
2656+
thrown.expect(IllegalArgumentException.class);
2657+
thrown.expectMessage(
2658+
"unbounded PCollection via FILE_LOADS, STORAGE_WRITE_API, or STORAGE_API_AT_LEAST_ONCE");
2659+
thrown.expectMessage("triggering frequency must be specified");
2660+
2661+
p.getOptions().as(BigQueryOptions.class).setStorageWriteApiTriggeringFrequencySec(null);
2662+
p.apply(Create.empty(INPUT_RECORD_CODER))
2663+
.setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED)
2664+
.apply(
2665+
BigQueryIO.<InputRecord>write()
2666+
.withAvroFormatFunction(r -> new GenericData.Record(r.getSchema()))
2667+
.to("dataset.table")
2668+
.withMethod(Method.STORAGE_API_AT_LEAST_ONCE)
2669+
.withCreateDisposition(CreateDisposition.CREATE_NEVER));
2670+
}
2671+
2672+
@Test
2673+
public void testStreamingWriteValidateSucceedsWithTriggeringFrequencyForStorageApiAtLeastOnce() {
2674+
assumeTrue(useStreaming);
2675+
assumeTrue(useStorageApiApproximate); // Test STORAGE_API_AT_LEAST_ONCE specifically
2676+
p.enableAbandonedNodeEnforcement(false);
2677+
2678+
// This should not throw - STORAGE_API_AT_LEAST_ONCE with triggering frequency should be valid
2679+
p.getOptions().as(BigQueryOptions.class).setStorageWriteApiTriggeringFrequencySec(30);
2680+
p.apply(Create.empty(INPUT_RECORD_CODER))
2681+
.setIsBoundedInternal(PCollection.IsBounded.UNBOUNDED)
2682+
.apply(
2683+
BigQueryIO.<InputRecord>write()
2684+
.withAvroFormatFunction(r -> new GenericData.Record(r.getSchema()))
2685+
.to("dataset.table")
2686+
.withMethod(Method.STORAGE_API_AT_LEAST_ONCE)
2687+
.withCreateDisposition(CreateDisposition.CREATE_NEVER)
2688+
.withTestServices(fakeBqServices)
2689+
.withoutValidation());
2690+
// Should validate without throwing
2691+
p.run();
2692+
}
2693+
2694+
@Test
2695+
public void testBoundedWriteValidateSucceedsWithoutTriggeringFrequencyForStorageApiAtLeastOnce() {
2696+
assumeTrue(!useStreaming); // Test bounded PCollection
2697+
assumeTrue(useStorageApiApproximate); // Test STORAGE_API_AT_LEAST_ONCE specifically
2698+
2699+
// Bounded collections should not require triggering frequency even for
2700+
// STORAGE_API_AT_LEAST_ONCE
2701+
p.getOptions().as(BigQueryOptions.class).setStorageWriteApiTriggeringFrequencySec(null);
2702+
p.apply(Create.empty(INPUT_RECORD_CODER))
2703+
.setIsBoundedInternal(PCollection.IsBounded.BOUNDED)
2704+
.apply(
2705+
BigQueryIO.<InputRecord>write()
2706+
.withAvroFormatFunction(r -> new GenericData.Record(r.getSchema()))
2707+
.to("dataset.table")
2708+
.withMethod(Method.STORAGE_API_AT_LEAST_ONCE)
2709+
.withCreateDisposition(CreateDisposition.CREATE_NEVER)
2710+
.withTestServices(fakeBqServices)
2711+
.withoutValidation());
2712+
// Should validate without throwing
2713+
p.run();
2714+
}
2715+
26492716
@Test
26502717
public void testBigQueryIOGetName() {
26512718
assertEquals(

0 commit comments

Comments
 (0)