Skip to content

Commit e05fdd7

Browse files
VihasMakwanamergify[bot]
VihasMakwana
authored and
mergify[bot]
committed
[samples] remove unneeded file_storage and fix a bug (#10856)
* fix: add flush_interval * couple of enhancements * test samples * lint * use observer and t.Setenv * lint * fix profiling * fix unit test race * more time * comment * enable profiling for linux * Add comment * use DryRun to validate the config --------- Co-authored-by: Fabrizio Ferri-Benedetti <[email protected]> (cherry picked from commit 49af983) # Conflicts: # internal/pkg/otel/samples/windows/managed_otlp/logs_metrics_traces.yml # internal/pkg/otel/samples/windows/managed_otlp/platformlogs.yml # internal/pkg/otel/samples/windows/managed_otlp/platformlogs_hostmetrics.yml
1 parent da89fed commit e05fdd7

File tree

6 files changed

+343
-6
lines changed

6 files changed

+343
-6
lines changed

‎internal/pkg/otel/samples/linux/profiles_metrics.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
receivers:
22
profiling:
3-
SamplesPerSecond: 19
3+
samples_per_second: 19
44

55
connectors:
66
profilingmetrics:

‎internal/pkg/otel/samples/windows/logs_metrics_traces.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,6 @@ receivers:
3636
grpc:
3737
http:
3838

39-
extensions:
40-
file_storage:
41-
directory: ${env:STORAGE_DIR}
42-
4339
connectors:
4440
elasticapm:
4541

@@ -85,7 +81,6 @@ exporters:
8581
mode: otel
8682

8783
service:
88-
extensions: [file_storage]
8984
pipelines:
9085
traces/fromsdk:
9186
receivers: [otlp/fromsdk]

‎internal/pkg/otel/samples/windows/managed_otlp/logs_metrics_traces.yml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
receivers:
2+
windowseventlog:
3+
channel: application
4+
5+
# Receiver for CPU, Disk, Memory, and Filesystem metrics
6+
hostmetrics/system:
7+
collection_interval: 60s
8+
scrapers:
9+
filesystem:
10+
memory:
11+
metrics:
12+
system.memory.utilization:
13+
enabled: true
14+
# process scraper is disabled for now: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/39423
15+
#process:
16+
# mute_process_exe_error: true
17+
# mute_process_io_error: true
18+
# mute_process_user_error: true
19+
# metrics:
20+
# process.threads:
21+
# enabled: true
22+
# process.open_file_descriptors:
23+
# enabled: true
24+
# process.memory.utilization:
25+
# enabled: true
26+
# process.disk.operations:
27+
# enabled: true
28+
network:
29+
processes:
30+
load:
31+
32+
# Receiver for logs, traces, and metrics from SDKs
33+
otlp/fromsdk:
34+
protocols:
35+
grpc:
36+
http:
37+
38+
processors:
39+
resourcedetection:
40+
detectors: ["system"]
41+
system:
42+
hostname_sources: ["os"]
43+
resource_attributes:
44+
host.name:
45+
enabled: true
46+
host.id:
47+
enabled: false
48+
host.arch:
49+
enabled: true
50+
host.ip:
51+
enabled: true
52+
host.mac:
53+
enabled: true
54+
host.cpu.vendor.id:
55+
enabled: true
56+
host.cpu.family:
57+
enabled: true
58+
host.cpu.model.id:
59+
enabled: true
60+
host.cpu.model.name:
61+
enabled: true
62+
host.cpu.stepping:
63+
enabled: true
64+
host.cpu.cache.l2.size:
65+
enabled: true
66+
os.description:
67+
enabled: true
68+
os.type:
69+
enabled: true
70+
71+
exporters:
72+
otlp/ingest_logs:
73+
endpoint: ${env:ELASTIC_OTLP_ENDPOINT}
74+
headers:
75+
Authorization: ApiKey ${env:ELASTIC_API_KEY}
76+
sending_queue:
77+
enabled: true
78+
sizer: bytes
79+
queue_size: 50000000 # 50MB uncompressed
80+
block_on_overflow: true
81+
batch:
82+
flush_timeout: 1s
83+
min_size: 1_000_000 # 1MB uncompressed
84+
max_size: 4_000_000 # 4MB uncompressed
85+
otlp/ingest_metrics_traces:
86+
endpoint: ${env:ELASTIC_OTLP_ENDPOINT}
87+
headers:
88+
Authorization: ApiKey ${env:ELASTIC_API_KEY}
89+
sending_queue:
90+
enabled: true
91+
sizer: bytes
92+
queue_size: 50000000 # 50MB uncompressed
93+
block_on_overflow: true
94+
batch:
95+
flush_timeout: 1s
96+
min_size: 1_000_000 # 1MB uncompressed
97+
max_size: 4_000_000 # 4MB uncompressed
98+
99+
service:
100+
pipelines:
101+
traces/fromsdk:
102+
receivers: [otlp/fromsdk]
103+
processors: []
104+
exporters: [otlp/ingest_metrics_traces]
105+
106+
metrics/fromsdk:
107+
receivers: [otlp/fromsdk]
108+
processors: []
109+
exporters: [otlp/ingest_metrics_traces]
110+
111+
logs/fromsdk:
112+
receivers: [otlp/fromsdk]
113+
processors: []
114+
exporters: [otlp/ingest_logs]
115+
116+
metrics/hostmetrics:
117+
receivers: [hostmetrics/system]
118+
processors: [resourcedetection]
119+
exporters: [otlp/ingest_metrics_traces]
120+
121+
logs/platformlogs:
122+
receivers: [windowseventlog]
123+
processors: [resourcedetection]
124+
exporters: [otlp/ingest_logs]

‎internal/pkg/otel/samples/windows/managed_otlp/platformlogs.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
receivers:
2+
windowseventlog:
3+
channel: application
4+
5+
processors:
6+
resourcedetection:
7+
detectors: ["system"]
8+
system:
9+
hostname_sources: ["os"]
10+
resource_attributes:
11+
host.name:
12+
enabled: true
13+
host.id:
14+
enabled: false
15+
host.arch:
16+
enabled: true
17+
host.ip:
18+
enabled: true
19+
host.mac:
20+
enabled: true
21+
host.cpu.vendor.id:
22+
enabled: true
23+
host.cpu.family:
24+
enabled: true
25+
host.cpu.model.id:
26+
enabled: true
27+
host.cpu.model.name:
28+
enabled: true
29+
host.cpu.stepping:
30+
enabled: true
31+
host.cpu.cache.l2.size:
32+
enabled: true
33+
os.description:
34+
enabled: true
35+
os.type:
36+
enabled: true
37+
38+
exporters:
39+
# Exporter to print the first 5 logs/metrics and then every 1000th
40+
debug:
41+
verbosity: detailed
42+
sampling_initial: 5
43+
sampling_thereafter: 1000
44+
45+
# Exporter to send logs and metrics to Elasticsearch Managed OTLP Input
46+
otlp/ingest:
47+
endpoint: ${env:ELASTIC_OTLP_ENDPOINT}
48+
headers:
49+
Authorization: ApiKey ${env:ELASTIC_API_KEY}
50+
sending_queue:
51+
enabled: true
52+
sizer: bytes
53+
queue_size: 50000000 # 50MB uncompressed
54+
block_on_overflow: true
55+
batch:
56+
flush_timeout: 1s
57+
min_size: 1_000_000 # 1MB uncompressed
58+
max_size: 4_000_000 # 4MB uncompressed
59+
60+
service:
61+
pipelines:
62+
logs/platformlogs:
63+
receivers: [windowseventlog]
64+
processors: [resourcedetection]
65+
exporters: [debug, otlp/ingest]

‎internal/pkg/otel/samples/windows/managed_otlp/platformlogs_hostmetrics.yml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
receivers:
2+
windowseventlog:
3+
channel: application
4+
5+
# Receiver for CPU, Disk, Memory, and Filesystem metrics
6+
hostmetrics/system:
7+
collection_interval: 60s
8+
scrapers:
9+
filesystem:
10+
memory:
11+
metrics:
12+
system.memory.utilization:
13+
enabled: true
14+
# process scraper is disabled for now: https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/39423
15+
#process:
16+
# mute_process_exe_error: true
17+
# mute_process_io_error: true
18+
# mute_process_user_error: true
19+
# metrics:
20+
# process.threads:
21+
# enabled: true
22+
# process.open_file_descriptors:
23+
# enabled: true
24+
# process.memory.utilization:
25+
# enabled: true
26+
# process.disk.operations:
27+
# enabled: true
28+
network:
29+
processes:
30+
load:
31+
32+
processors:
33+
resourcedetection:
34+
detectors: ["system"]
35+
system:
36+
hostname_sources: ["os"]
37+
resource_attributes:
38+
host.name:
39+
enabled: true
40+
host.id:
41+
enabled: false
42+
host.arch:
43+
enabled: true
44+
host.ip:
45+
enabled: true
46+
host.mac:
47+
enabled: true
48+
host.cpu.vendor.id:
49+
enabled: true
50+
host.cpu.family:
51+
enabled: true
52+
host.cpu.model.id:
53+
enabled: true
54+
host.cpu.model.name:
55+
enabled: true
56+
host.cpu.stepping:
57+
enabled: true
58+
host.cpu.cache.l2.size:
59+
enabled: true
60+
os.description:
61+
enabled: true
62+
os.type:
63+
enabled: true
64+
65+
exporters:
66+
# Exporter to send logs and metrics to Elasticsearch Managed OTLP Input
67+
otlp/ingest:
68+
endpoint: ${env:ELASTIC_OTLP_ENDPOINT}
69+
headers:
70+
Authorization: ApiKey ${env:ELASTIC_API_KEY}
71+
sending_queue:
72+
enabled: true
73+
sizer: bytes
74+
queue_size: 50000000 # 50MB uncompressed
75+
block_on_overflow: true
76+
batch:
77+
flush_timeout: 1s
78+
min_size: 1_000_000 # 1MB uncompressed
79+
max_size: 4_000_000 # 4MB uncompressed
80+
81+
service:
82+
pipelines:
83+
metrics/hostmetrics:
84+
receivers: [hostmetrics/system]
85+
processors: [resourcedetection]
86+
exporters: [otlp/ingest]
87+
logs/platformlogs:
88+
receivers: [windowseventlog]
89+
processors: [resourcedetection]
90+
exporters: [otlp/ingest]

‎internal/pkg/otel/samples_test.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
2+
// or more contributor license agreements. Licensed under the Elastic License 2.0;
3+
// you may not use this file except in compliance with the Elastic License 2.0.
4+
5+
package otel
6+
7+
import (
8+
"os"
9+
"path/filepath"
10+
"runtime"
11+
"testing"
12+
13+
"github.com/stretchr/testify/assert"
14+
"go.opentelemetry.io/collector/featuregate"
15+
"go.opentelemetry.io/collector/otelcol"
16+
"go.uber.org/zap"
17+
"go.uber.org/zap/zapcore"
18+
)
19+
20+
func TestSamples(t *testing.T) {
21+
// Set environment variables required by the sample configurations.
22+
// These are just sample values to allow the collector to start.
23+
t.Setenv("STORAGE_DIR", t.TempDir())
24+
t.Setenv("ELASTIC_ENDPOINT", "http://localhost:9200")
25+
t.Setenv("ELASTIC_API_KEY", "test_api_key")
26+
t.Setenv("ELASTIC_OTLP_ENDPOINT", "http://localhost:4318")
27+
t.Setenv("AUTOOPS_ES_URL", "http://localhost:9200")
28+
t.Setenv("AUTOOPS_TOKEN", "token")
29+
t.Setenv("AUTOOPS_TEMP_RESOURCE_ID", "temp")
30+
t.Setenv("AUTOOPS_OTEL_URL", "http://localhost:4318")
31+
32+
// Enable service.profilesSupport featuregate to test the profiling samples.
33+
assert.NoError(t, featuregate.GlobalRegistry().Set("service.profilesSupport", true))
34+
defer func() {
35+
assert.NoError(t, featuregate.GlobalRegistry().Set("service.profilesSupport", false))
36+
}()
37+
38+
err := filepath.WalkDir(filepath.Join(".", "samples", runtime.GOOS), func(path string, d os.DirEntry, err error) error {
39+
if err != nil {
40+
return err
41+
}
42+
if !d.IsDir() && (filepath.Ext(d.Name()) == ".yaml" || filepath.Ext(d.Name()) == ".yml") {
43+
t.Run(d.Name(), func(t *testing.T) {
44+
testSample(t, filepath.Join(".", "samples", runtime.GOOS, d.Name()))
45+
})
46+
}
47+
return nil
48+
})
49+
assert.NoError(t, err)
50+
}
51+
52+
func testSample(t *testing.T, configFile string) {
53+
settings := NewSettings("test", []string{configFile})
54+
settings.LoggingOptions = []zap.Option{zap.WrapCore(func(zapcore.Core) zapcore.Core {
55+
// TODO: Replace with observer core. Right now, it results in a race condition.
56+
return zapcore.NewNopCore()
57+
})}
58+
collector, err := otelcol.NewCollector(*settings)
59+
assert.NoError(t, err)
60+
assert.NotNil(t, collector)
61+
assert.NoError(t, collector.DryRun(t.Context()))
62+
collector.Shutdown()
63+
}

0 commit comments

Comments
 (0)