Skip to content

Commit b5bec56

Browse files
edmocostaedmocosta
committed
Merge branch 'main' of https://github.com/edmocosta/logstash into fix-jackson-json-log-format-issue
2 parents b45acec + 5af14f4 commit b5bec56

File tree

114 files changed

+6770
-1829
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

114 files changed

+6770
-1829
lines changed

‎.buildkite/aarch64_pipeline.yml

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
agents:
2+
provider: aws
3+
imagePrefix: platform-ingest-logstash-ubuntu-2204-aarch64
4+
instanceType: "m6g.4xlarge"
5+
diskSizeGb: 200
6+
7+
steps:
8+
- group: "Testing Phase"
9+
key: "testing-phase"
10+
steps:
11+
- label: ":rspec: Ruby unit tests"
12+
key: "ruby-unit-tests"
13+
command: |
14+
set -euo pipefail
15+
16+
source .buildkite/scripts/common/vm-agent.sh
17+
ci/unit_tests.sh ruby
18+
19+
- label: ":java: Java unit tests"
20+
key: "java-unit-tests"
21+
env:
22+
# https://github.com/elastic/logstash/pull/15486 for background
23+
ENABLE_SONARQUBE: "false"
24+
command: |
25+
set -euo pipefail
26+
27+
source .buildkite/scripts/common/vm-agent.sh
28+
ci/unit_tests.sh java
29+
30+
- label: ":lab_coat: Integration Tests / part 1"
31+
key: "integration-tests-part-1"
32+
command: |
33+
set -euo pipefail
34+
35+
source .buildkite/scripts/common/vm-agent.sh
36+
ci/integration_tests.sh split 0
37+
38+
- label: ":lab_coat: Integration Tests / part 2"
39+
key: "integration-tests-part-2"
40+
command: |
41+
set -euo pipefail
42+
43+
source .buildkite/scripts/common/vm-agent.sh
44+
ci/integration_tests.sh split 1
45+
46+
- label: ":lab_coat: IT Persistent Queues / part 1"
47+
key: "integration-tests-qa-part-1"
48+
command: |
49+
set -euo pipefail
50+
51+
source .buildkite/scripts/common/vm-agent.sh
52+
export FEATURE_FLAG=persistent_queues
53+
ci/integration_tests.sh split 0
54+
55+
- label: ":lab_coat: IT Persistent Queues / part 2"
56+
key: "integration-tests-qa-part-2"
57+
command: |
58+
set -euo pipefail
59+
60+
source .buildkite/scripts/common/vm-agent.sh
61+
export FEATURE_FLAG=persistent_queues
62+
ci/integration_tests.sh split 1
63+
64+
- label: ":lab_coat: x-pack unit tests"
65+
key: "x-pack-unit-tests"
66+
command: |
67+
set -euo pipefail
68+
69+
source .buildkite/scripts/common/vm-agent.sh
70+
x-pack/ci/unit_tests.sh
71+
72+
- label: ":lab_coat: x-pack integration"
73+
key: "integration-tests-x-pack"
74+
command: |
75+
set -euo pipefail
76+
77+
source .buildkite/scripts/common/vm-agent.sh
78+
x-pack/ci/integration_tests.sh
79+
80+
- group: "Acceptance Phase"
81+
depends_on: "testing-phase"
82+
key: "acceptance-phase"
83+
steps:
84+
- label: "Docker [{{matrix}}] flavor acceptance"
85+
command:
86+
set -euo pipefail
87+
88+
source .buildkite/scripts/common/vm-agent.sh && ci/docker_acceptance_tests.sh {{matrix}}
89+
matrix:
90+
- "full"
91+
- "oss"
92+
93+
# *** TODO: enable after clarifying if acceptance tests really need vagrant on aarch64
94+
# - label: "Acceptance tests on {{matrix.distribution}}"
95+
# agents:
96+
# provider: aws
97+
# imagePrefix: platform-ingest-logstash-{{matrix.distribution}}-aarch64
98+
# instanceType: "m6g.4xlarge"
99+
# diskSizeGb: 200
100+
# command:
101+
# set -euo pipefail
102+
103+
# source .buildkite/scripts/common/vm-agent.sh && ci/acceptance_tests.sh {{matrix.suite}}
104+
# matrix:
105+
# setup:
106+
# suite:
107+
# - "debian"
108+
# distribution:
109+
# - "ubuntu-2204"

‎.buildkite/dra_pipeline.yml

+8-9
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
22

3-
agents:
4-
cpu: "2"
5-
memory: "4Gi"
6-
ephemeralStorage: "20Gi"
7-
83
steps:
9-
- label: ":wave: Greetings"
10-
command: "echo 'TODO: run DRA Logstash'"
11-
agents:
12-
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.1"
4+
- label: ":pipeline: Generate steps"
5+
command: |
6+
set -euo pipefail
7+
8+
echo "--- Building [${WORKFLOW_TYPE}] artifacts"
9+
python3 -m pip install pyyaml
10+
echo "--- Building dynamic pipeline steps"
11+
python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload

‎.buildkite/linux_jdk_matrix_pipeline.yml

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
steps:
2+
- input: "Test Parameters"
3+
if: build.source != "schedule"
4+
fields:
5+
- select: "Operating System"
6+
key: "matrix-os"
7+
hint: "The operating system variant(s) to run on:"
8+
required: true
9+
multiple: true
10+
default: "ubuntu-2204"
11+
options:
12+
- label: "Ubuntu 22.04"
13+
value: "ubuntu-2204"
14+
- label: "Ubuntu 20.04"
15+
value: "ubuntu-2004"
16+
- label: "Ubuntu 18.04"
17+
value: "ubuntu-1804"
18+
- label: "Debian 11"
19+
value: "debian-11"
20+
- label: "Debian 10"
21+
value: "debian-10"
22+
- label: "RHEL 9"
23+
value: "rhel-9"
24+
- label: "RHEL 8"
25+
value: "rhel-8"
26+
- label: "CentOS 8"
27+
value: "centos-8"
28+
- label: "CentOS 7"
29+
value: "centos-7"
30+
- label: "Oracle Linux 8"
31+
value: "oraclelinux-8"
32+
- label: "Oracle Linux 7"
33+
value: "oraclelinux-7"
34+
- label: "Rocky Linux 8"
35+
value: "rocky-8"
36+
- label: "Amazon Linux"
37+
value: "amazonlinux-2023"
38+
- label: "OpenSUSE Leap 15"
39+
value: "opensuse-leap-15"
40+
41+
- select: "Java"
42+
key: "matrix-jdk"
43+
hint: "The JDK to test with:"
44+
required: true
45+
multiple: true
46+
default: "adoptiumjdk_17"
47+
options:
48+
- label: "Adoptium JDK 17 (Eclipse Temurin)"
49+
value: "adoptiumjdk_17"
50+
- label: "Adopt OpenJDK 11"
51+
value: "adoptopenjdk_11"
52+
- label: "OpenJDK 17"
53+
value: "openjdk_17"
54+
- label: "OpenJDK 11"
55+
value: "openjdk_11"
56+
- label: "Zulu 17"
57+
value: "zulu_17"
58+
- label: "Zulu 11"
59+
value: "zulu_11"
60+
61+
- wait: ~
62+
if: build.source != "schedule"
63+
64+
- command: |
65+
set -euo pipefail
66+
67+
echo "--- Downloading prerequisites"
68+
python3 -m pip install ruamel.yaml
69+
70+
echo "--- Printing generated dynamic steps"
71+
export MATRIX_OSES="$(buildkite-agent meta-data get matrix-os)"
72+
export MATRIX_JDKS="$(buildkite-agent meta-data get matrix-jdk)"
73+
set +eo pipefail
74+
python3 .buildkite/scripts/jdk-matrix-tests/generate-steps.py >pipeline_steps.yml
75+
if [[ $$? -ne 0 ]]; then
76+
echo "^^^ +++"
77+
echo "There was a problem rendering the pipeline steps."
78+
cat pipeline_steps.yml
79+
echo "Exiting now."
80+
exit 1
81+
else
82+
set -eo pipefail
83+
cat pipeline_steps.yml
84+
fi
85+
86+
echo "--- Uploading steps to buildkite"
87+
cat pipeline_steps.yml | buildkite-agent pipeline upload

‎.buildkite/pull-requests.json

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"jobs": [
3+
{
4+
"enabled": true,
5+
"pipeline_slug": "logstash-pull-request-pipeline",
6+
"allow_org_users": true,
7+
"allowed_repo_permissions": ["admin", "write"],
8+
"allowed_list": ["github-actions[bot]"],
9+
"set_commit_status": true,
10+
"build_on_commit": true,
11+
"build_on_comment": true,
12+
"trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
13+
"always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))",
14+
"skip_ci_labels": [ ],
15+
"skip_target_branches": [ ],
16+
"skip_ci_on_only_changed": [ ],
17+
"always_require_ci_on_changed": [ ]
18+
}
19+
]
20+
}

‎.buildkite/pull_request_pipeline.yml

+136
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
2+
3+
steps:
4+
- label: ":passport_control: License check"
5+
key: "license-check"
6+
agents:
7+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
8+
cpu: "4"
9+
memory: "6Gi"
10+
ephemeralStorage: "100Gi"
11+
command: |
12+
set -euo pipefail
13+
14+
source .buildkite/scripts/common/container-agent.sh
15+
export JRUBY_OPTS="-J-Xmx1g"
16+
export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info"
17+
ci/license_check.sh -m 4G
18+
19+
- label: ":rspec: Ruby unit tests"
20+
key: "ruby-unit-tests"
21+
agents:
22+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
23+
cpu: "4"
24+
memory: "8Gi"
25+
ephemeralStorage: "100Gi"
26+
command: |
27+
set -euo pipefail
28+
29+
source .buildkite/scripts/common/container-agent.sh
30+
ci/unit_tests.sh ruby
31+
32+
- label: ":java: Java unit tests"
33+
key: "java-unit-tests"
34+
agents:
35+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
36+
cpu: "8"
37+
memory: "16Gi"
38+
ephemeralStorage: "100Gi"
39+
command: |
40+
set -euo pipefail
41+
42+
source .buildkite/scripts/common/container-agent.sh
43+
source .buildkite/scripts/pull-requests/sonar-env.sh
44+
ci/unit_tests.sh java
45+
46+
- label: ":lab_coat: Integration Tests / part 1"
47+
key: "integration-tests-part-1"
48+
agents:
49+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
50+
cpu: "8"
51+
memory: "16Gi"
52+
ephemeralStorage: "100Gi"
53+
# Run as a non-root user
54+
imageUID: "1002"
55+
command: |
56+
set -euo pipefail
57+
58+
source .buildkite/scripts/common/container-agent.sh
59+
ci/integration_tests.sh split 0
60+
61+
- label: ":lab_coat: Integration Tests / part 2"
62+
key: "integration-tests-part-2"
63+
agents:
64+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
65+
cpu: "8"
66+
memory: "16Gi"
67+
ephemeralStorage: "100Gi"
68+
# Run as a non-root user
69+
imageUID: "1002"
70+
command: |
71+
set -euo pipefail
72+
73+
source .buildkite/scripts/common/container-agent.sh
74+
ci/integration_tests.sh split 1
75+
76+
- label: ":lab_coat: IT Persistent Queues / part 1"
77+
key: "integration-tests-qa-part-1"
78+
agents:
79+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
80+
cpu: "8"
81+
memory: "16Gi"
82+
ephemeralStorage: "100Gi"
83+
# Run as non root (logstash) user. UID is hardcoded in image.
84+
imageUID: "1002"
85+
command: |
86+
set -euo pipefail
87+
88+
source .buildkite/scripts/common/container-agent.sh
89+
export FEATURE_FLAG=persistent_queues
90+
ci/integration_tests.sh split 0
91+
92+
- label: ":lab_coat: IT Persistent Queues / part 2"
93+
key: "integration-tests-qa-part-2"
94+
agents:
95+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
96+
cpu: "8"
97+
memory: "16Gi"
98+
ephemeralStorage: "100Gi"
99+
# Run as non root (logstash) user. UID is hardcoded in image.
100+
imageUID: "1002"
101+
command: |
102+
set -euo pipefail
103+
104+
source .buildkite/scripts/common/container-agent.sh
105+
export FEATURE_FLAG=persistent_queues
106+
ci/integration_tests.sh split 1
107+
108+
- label: ":lab_coat: x-pack unit tests"
109+
key: "x-pack-unit-tests"
110+
agents:
111+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
112+
cpu: "8"
113+
memory: "16Gi"
114+
ephemeralStorage: "100Gi"
115+
# Run as non root (logstash) user. UID is hardcoded in image.
116+
imageUID: "1002"
117+
command: |
118+
set -euo pipefail
119+
120+
source .buildkite/scripts/common/container-agent.sh
121+
x-pack/ci/unit_tests.sh
122+
123+
- label: ":lab_coat: x-pack integration"
124+
key: "integration-tests-x-pack"
125+
agents:
126+
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
127+
cpu: "8"
128+
memory: "16Gi"
129+
ephemeralStorage: "100Gi"
130+
# Run as non root (logstash) user. UID is hardcoded in image.
131+
imageUID: "1002"
132+
command: |
133+
set -euo pipefail
134+
135+
source .buildkite/scripts/common/container-agent.sh
136+
x-pack/ci/integration_tests.sh

‎.buildkite/scripts/common/container-agent.sh

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/usr/bin/env bash
2+
3+
# ********************************************************
4+
# This file contains prerequisite bootstrap invocations
5+
# required for Logstash CI when using containerized agents
6+
# ********************************************************
7+
8+
set -euo pipefail
9+
10+
if [[ $(whoami) == "logstash" ]]
11+
then
12+
export PATH="/home/logstash/.rbenv/bin:$PATH"
13+
eval "$(rbenv init -)"
14+
else
15+
export PATH="/usr/local/rbenv/bin:$PATH"
16+
eval "$(rbenv init -)"
17+
fi

0 commit comments

Comments
 (0)