Skip to content

Commit 84b467b

Browse files
derrickawderrickaw
authored
Add Datadog IO (#37319)
* porting and some improvements * fix some publisher issues * checkstyle, import, etc changes * add changes info on datadog * updated formatting and correct issue link * fix format issue * fix comments * add precommit for datadog * fix nullable annotation
1 parent e7c80cd commit 84b467b

23 files changed

+2966
-5
lines changed

‎.github/workflows/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,7 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour
235235
| [ PreCommit Java Cdap IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml) | N/A |`Run Java_Cdap_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Cdap_IO_Direct.yml?query=event%3Aschedule) |
236236
| [ PreCommit Java Clickhouse IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml) | N/A |`Run Java_Clickhouse_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Clickhouse_IO_Direct.yml?query=event%3Aschedule) |
237237
| [ PreCommit Java Csv IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml) | N/A |`Run Java_Csv_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Csv_IO_Direct.yml?query=event%3Aschedule) |
238+
| [ PreCommit Java Datadog IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml) | N/A |`Run Java_Datadog_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml?query=event%3Aschedule) |
238239
| [ PreCommit Java Debezium IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml) | N/A |`Run Java_Debezium_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Debezium_IO_Direct.yml?query=event%3Aschedule) |
239240
| [ PreCommit Java ElasticSearch IO Direct ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml) | N/A |`Run Java_ElasticSearch_IO_Direct PreCommit`| [![.github/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_ElasticSearch_IO_Direct.yml?query=event%3Aschedule) |
240241
| [ PreCommit Java Examples Dataflow ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml) | N/A |`Run Java_Examples_Dataflow PreCommit`| [![.github/workflows/beam_PreCommit_Java_Examples_Dataflow.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Java_Examples_Dataflow.yml?query=event%3Aschedule) |

‎.github/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: PreCommit Java Datadog IO Direct
17+
18+
on:
19+
push:
20+
tags: ['v*']
21+
branches: ['master', 'release-*']
22+
paths:
23+
- "sdks/java/io/datadog/**"
24+
- ".github/workflows/beam_PreCommit_Java_Datadog_IO_Direct.yml"
25+
pull_request_target:
26+
branches: ['master', 'release-*']
27+
paths:
28+
- "sdks/java/io/datadog/**"
29+
- 'release/trigger_all_tests.json'
30+
- '.github/trigger_files/beam_PreCommit_Java_Datadog_IO_Direct.json'
31+
issue_comment:
32+
types: [created]
33+
schedule:
34+
- cron: '15 1/6 * * *'
35+
workflow_dispatch:
36+
37+
#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
38+
permissions:
39+
actions: write
40+
pull-requests: write
41+
checks: write
42+
contents: read
43+
deployments: read
44+
id-token: none
45+
issues: write
46+
discussions: read
47+
packages: read
48+
pages: read
49+
repository-projects: read
50+
security-events: read
51+
statuses: read
52+
53+
# This allows a subsequently queued workflow run to interrupt previous runs
54+
concurrency:
55+
group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.event.pull_request.head.label || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
56+
cancel-in-progress: true
57+
58+
env:
59+
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
60+
GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
61+
GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
62+
63+
jobs:
64+
beam_PreCommit_Java_Datadog_IO_Direct:
65+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
66+
strategy:
67+
matrix:
68+
job_name: ["beam_PreCommit_Java_Datadog_IO_Direct"]
69+
job_phrase: ["Run Java_Datadog_IO_Direct PreCommit"]
70+
timeout-minutes: 60
71+
if: |
72+
github.event_name == 'push' ||
73+
github.event_name == 'pull_request_target' ||
74+
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
75+
github.event_name == 'workflow_dispatch' ||
76+
github.event.comment.body == 'Run Java_Datadog_IO_Direct PreCommit'
77+
runs-on: [self-hosted, ubuntu-20.04, main]
78+
steps:
79+
- uses: actions/checkout@v4
80+
- name: Setup repository
81+
uses: ./.github/actions/setup-action
82+
with:
83+
comment_phrase: ${{ matrix.job_phrase }}
84+
github_token: ${{ secrets.GITHUB_TOKEN }}
85+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
86+
- name: Setup environment
87+
uses: ./.github/actions/setup-environment-action
88+
- name: run Datadog IO build script
89+
uses: ./.github/actions/gradle-command-self-hosted-action
90+
with:
91+
gradle-command: :sdks:java:io:datadog:build
92+
arguments: |
93+
-PdisableSpotlessCheck=true \
94+
-PdisableCheckStyle=true \
95+
- name: Archive JUnit Test Results
96+
uses: actions/upload-artifact@v4
97+
if: ${{ !success() }}
98+
with:
99+
name: JUnit Test Results
100+
path: "**/build/reports/tests/"
101+
- name: Publish JUnit Test Results
102+
uses: EnricoMi/publish-unit-test-result-action@v2
103+
if: always()
104+
with:
105+
commit: '${{ env.prsha || env.GITHUB_SHA }}'
106+
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
107+
files: '**/build/test-results/**/*.xml'
108+
large_files: true
109+
- name: Archive SpotBugs Results
110+
uses: actions/upload-artifact@v4
111+
if: always()
112+
with:
113+
name: SpotBugs Results
114+
path: '**/build/reports/spotbugs/*.html'
115+
- name: Publish SpotBugs Results
116+
uses: jwgmeligmeyling/[email protected]
117+
if: always()
118+
with:
119+
name: Publish SpotBugs
120+
path: '**/build/reports/spotbugs/*.html'

‎CHANGES.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@
5555
* ([#X](https://github.com/apache/beam/issues/X)).
5656
-->
5757

58-
5958
# [2.72.0] - Unreleased
6059

6160
## Highlights
@@ -65,7 +64,7 @@
6564

6665
## I/Os
6766

68-
* Support for X source added (Java/Python) ([#X](https://github.com/apache/beam/issues/X)).
67+
* Add support for Datadog IO (Java) ([#37318](https://github.com/apache/beam/issues/37318)).
6968

7069
## New Features / Improvements
7170

@@ -113,6 +112,7 @@
113112

114113
## Known Issues
115114

115+
116116
# [2.70.0] - 2025-12-16
117117

118118
## Highlights
@@ -196,7 +196,7 @@ Now Beam has full support for Milvus integration including Milvus enrichment and
196196

197197
## Highlights
198198

199-
* [Python] Prism runner now enabled by default for most Python pipelines using the direct runner ([#34612](https://github.com/apache/beam/pull/34612)). This may break some tests, see https://github.com/apache/beam/pull/34612 for details on how to handle issues.
199+
* (Python) Prism runner now enabled by default for most Python pipelines using the direct runner ([#34612](https://github.com/apache/beam/pull/34612)). This may break some tests, see https://github.com/apache/beam/pull/34612 for details on how to handle issues.
200200

201201
## I/Os
202202

@@ -212,7 +212,7 @@ Now Beam has full support for Milvus integration including Milvus enrichment and
212212
Beam now supports data enrichment capabilities using SQL databases, with built-in support for:
213213
- Managed PostgreSQL, MySQL, and Microsoft SQL Server instances on CloudSQL
214214
- Unmanaged SQL database instances not hosted on CloudSQL (e.g., self-hosted or on-premises databases)
215-
* [Python] Added the `ReactiveThrottler` and `ThrottlingSignaler` classes to streamline throttling behavior in DoFns, expose throttling mechanisms for users ([#35984](https://github.com/apache/beam/pull/35984))
215+
* (Python) Added the `ReactiveThrottler` and `ThrottlingSignaler` classes to streamline throttling behavior in DoFns, expose throttling mechanisms for users ([#35984](https://github.com/apache/beam/pull/35984))
216216
* Added a pipeline option to specify the processing timeout for a single element by any PTransform (Java/Python/Go) ([#35174](https://github.com/apache/beam/issues/35174)).
217217
- When specified, the SDK harness automatically restarts if an element takes too long to process. Beam runner may then retry processing of the same work item.
218218
- Use the `--element_processing_timeout_minutes` option to reduce the chance of having stalled pipelines due to unexpected cases of slow processing, where slowness might not happen again if processing of the same element is retried.
@@ -2351,4 +2351,4 @@ Schema Options, it will be removed in version `2.23.0`. ([BEAM-9704](https://iss
23512351

23522352
## Highlights
23532353

2354-
- For versions 2.19.0 and older release notes are available on [Apache Beam Blog](https://beam.apache.org/blog/).
2354+
- For versions 2.19.0 and older release notes are available on [Apache Beam Blog](https://beam.apache.org/blog/).

‎build.gradle.kts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,7 @@ tasks.register("javaioPreCommit") {
344344
dependsOn(":sdks:java:io:csv:build")
345345
dependsOn(":sdks:java:io:cdap:build")
346346
dependsOn(":sdks:java:io:clickhouse:build")
347+
dependsOn(":sdks:java:io:datadog:build")
347348
dependsOn(":sdks:java:io:debezium:expansion-service:build")
348349
dependsOn(":sdks:java:io:debezium:build")
349350
dependsOn(":sdks:java:io:elasticsearch:build")

‎sdks/java/io/datadog/build.gradle‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* License); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an AS IS BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
plugins { id 'org.apache.beam.module' }
20+
applyJavaNature(
21+
automaticModuleName: 'org.apache.beam.sdk.io.datadog'
22+
)
23+
24+
description = "Apache Beam :: SDKs :: Java :: IO :: Datadog"
25+
ext.summary = "IO to read and write to Datadog."
26+
27+
dependencies {
28+
implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
29+
implementation project(path: ":sdks:java:core", configuration: "shadow")
30+
implementation library.java.vendored_guava_32_1_2_jre
31+
implementation library.java.joda_time
32+
implementation library.java.slf4j_api
33+
implementation library.java.google_http_client
34+
implementation library.java.google_code_gson
35+
implementation library.java.auto_value_annotations
36+
testImplementation project(path: ":sdks:java:core", configuration: "shadowTest")
37+
testImplementation library.java.jupiter_api
38+
testRuntimeOnly library.java.jupiter_engine
39+
testImplementation library.java.jupiter_params
40+
testImplementation library.java.truth
41+
testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
42+
testImplementation project(path: ":sdks:java:io:common")
43+
testImplementation group: 'org.mock-server', name: 'mockserver-client-java', version: '5.10.0'
44+
testImplementation group: 'org.mock-server', name: 'mockserver-junit-rule', version: '5.10.0'
45+
implementation library.java.google_http_client_apache_v2
46+
implementation library.java.http_client
47+
implementation library.java.http_core
48+
}

‎sdks/java/io/datadog/src/main/java/org/apache/beam/sdk/io/datadog/DatadogEvent.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.beam.sdk.io.datadog;
19+
20+
import static org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Preconditions.checkNotNull;
21+
22+
import com.google.auto.value.AutoValue;
23+
import org.checkerframework.checker.nullness.qual.Nullable;
24+
25+
/** A class for Datadog events. */
26+
@AutoValue
27+
public abstract class DatadogEvent {
28+
29+
public static Builder newBuilder() {
30+
return new AutoValue_DatadogEvent.Builder();
31+
}
32+
33+
public abstract @Nullable String ddsource();
34+
35+
public abstract @Nullable String ddtags();
36+
37+
public abstract @Nullable String hostname();
38+
39+
public abstract @Nullable String service();
40+
41+
public abstract @Nullable String message();
42+
43+
/** A builder class for creating {@link DatadogEvent} objects. */
44+
@AutoValue.Builder
45+
public abstract static class Builder {
46+
47+
abstract Builder setDdsource(String source);
48+
49+
abstract Builder setDdtags(String tags);
50+
51+
abstract Builder setHostname(String hostname);
52+
53+
abstract Builder setService(String service);
54+
55+
abstract Builder setMessage(String message);
56+
57+
abstract String message();
58+
59+
abstract DatadogEvent autoBuild();
60+
61+
public Builder withSource(String source) {
62+
checkNotNull(source, "withSource(source) called with null input.");
63+
64+
return setDdsource(source);
65+
}
66+
67+
public Builder withTags(String tags) {
68+
checkNotNull(tags, "withTags(tags) called with null input.");
69+
70+
return setDdtags(tags);
71+
}
72+
73+
public Builder withHostname(String hostname) {
74+
checkNotNull(hostname, "withHostname(hostname) called with null input.");
75+
76+
return setHostname(hostname);
77+
}
78+
79+
public Builder withService(String service) {
80+
checkNotNull(service, "withService(service) called with null input.");
81+
82+
return setService(service);
83+
}
84+
85+
public Builder withMessage(String message) {
86+
checkNotNull(message, "withMessage(message) called with null input.");
87+
88+
return setMessage(message);
89+
}
90+
91+
public DatadogEvent build() {
92+
checkNotNull(message(), "Message is required.");
93+
94+
return autoBuild();
95+
}
96+
}
97+
}

0 commit comments

Comments
 (0)