diff --git a/apps/otlp.go b/apps/otlp.go index 6ceb70e637..1ccc25e152 100644 --- a/apps/otlp.go +++ b/apps/otlp.go @@ -141,7 +141,7 @@ func (r ReceiverOTLP) Pipelines(ctx context.Context) ([]otel.ReceiverPipeline, e ExporterTypes: map[string]otel.ExporterType{ "metrics": receiverPipelineType, "traces": otel.OTel, - "logs": otel.OTel, + "logs": otel.Logs, }, Receiver: otel.Component{ Type: "otlp", diff --git a/cmd/ops_agent_uap_plugin/service_linux.go b/cmd/ops_agent_uap_plugin/service_linux.go index bdf08935aa..b369fb22d4 100644 --- a/cmd/ops_agent_uap_plugin/service_linux.go +++ b/cmd/ops_agent_uap_plugin/service_linux.go @@ -43,6 +43,7 @@ const ( LogsDirectory = "log/google-cloud-ops-agent" FluentBitStateDiectory = "state/fluent-bit" FluentBitRuntimeDirectory = "run/google-cloud-ops-agent-fluent-bit" + OtelStateDiectory = "state/opentelemetry-collector" OtelRuntimeDirectory = "run/google-cloud-ops-agent-opentelemetry-collector" DefaultPluginStateDirectory = "/var/lib/google-guest-agent/agent_state/plugins/ops-agent-plugin" ) @@ -209,7 +210,8 @@ func generateSubagentConfigs(ctx context.Context, runCommand RunCommandFunc, plu "-service", "otel", "-in", OpsAgentConfigLocationLinux, "-out", path.Join(pluginStateDirectory, OtelRuntimeDirectory), - "-logs", path.Join(pluginStateDirectory, LogsDirectory)) + "-logs", path.Join(pluginStateDirectory, LogsDirectory), + "-state", path.Join(pluginStateDirectory, OtelStateDiectory)) if output, err := runCommand(otelConfigGenerationCmd); err != nil { return fmt.Errorf("failed to generate Otel config:\ncommand output: %s\ncommand error: %s", output, err) @@ -220,7 +222,8 @@ func generateSubagentConfigs(ctx context.Context, runCommand RunCommandFunc, plu "-service", "fluentbit", "-in", OpsAgentConfigLocationLinux, "-out", path.Join(pluginStateDirectory, FluentBitRuntimeDirectory), - "-logs", path.Join(pluginStateDirectory, LogsDirectory), "-state", path.Join(pluginStateDirectory, FluentBitStateDiectory)) + "-logs", path.Join(pluginStateDirectory, LogsDirectory), + "-state", path.Join(pluginStateDirectory, FluentBitStateDiectory)) if output, err := runCommand(fluentBitConfigGenerationCmd); err != nil { return fmt.Errorf("failed to generate Fluntbit config:\ncommand output: %s\ncommand error: %s", output, err) diff --git a/cmd/ops_agent_windows/main_windows.go b/cmd/ops_agent_windows/main_windows.go index 0ddcfa7a50..8b7c03c579 100644 --- a/cmd/ops_agent_windows/main_windows.go +++ b/cmd/ops_agent_windows/main_windows.go @@ -106,6 +106,10 @@ func initServices() error { if err := os.MkdirAll(fluentbitStoragePath, 0644); err != nil { return err } + otelStoragePath := filepath.Join(os.Getenv("PROGRAMDATA"), dataDirectory, `run\file_storage`) + if err := os.MkdirAll(otelStoragePath, 0644); err != nil { + return err + } logDirectory := filepath.Join(os.Getenv("PROGRAMDATA"), dataDirectory, "log") if err := os.MkdirAll(logDirectory, 0644); err != nil { return err diff --git a/confgenerator/confgenerator.go b/confgenerator/confgenerator.go index 850421f00a..c829076d4b 100644 --- a/confgenerator/confgenerator.go +++ b/confgenerator/confgenerator.go @@ -34,27 +34,42 @@ import ( "github.com/GoogleCloudPlatform/ops-agent/internal/platform" ) -func googleCloudExporter(userAgent string, instrumentationLabels bool, serviceResourceLabels bool) otel.Component { - return otel.Component{ - Type: "googlecloud", - Config: map[string]interface{}{ - "user_agent": userAgent, - "metric": map[string]interface{}{ - // Receivers are responsible for sending fully-qualified metric names. - // NB: If a receiver fails to send a full URL, OT will add the prefix `workload.googleapis.com/{metric_name}`. - // TODO(b/197129428): Write a test to make sure this doesn't happen. - "prefix": "", - // OT calls CreateMetricDescriptor by default. Skip because we want - // descriptors to be created implicitly with new time series. - "skip_create_descriptor": true, - // Omit instrumentation labels, which break agent metrics. - "instrumentation_library_labels": instrumentationLabels, - // Omit service labels, which break agent metrics. - "service_resource_labels": serviceResourceLabels, - "resource_filters": []map[string]interface{}{}, - }, +func googleCloudExporter(userAgent string, instrumentationLabels, serviceResourceLabels, logBuffering bool) otel.Component { + config := map[string]interface{}{ + "user_agent": userAgent, + "metric": map[string]interface{}{ + // Receivers are responsible for sending fully-qualified metric names. + // NB: If a receiver fails to send a full URL, OT will add the prefix `workload.googleapis.com/{metric_name}`. + // TODO(b/197129428): Write a test to make sure this doesn't happen. + "prefix": "", + // OT calls CreateMetricDescriptor by default. Skip because we want + // descriptors to be created implicitly with new time series. + "skip_create_descriptor": true, + // Omit instrumentation labels, which break agent metrics. + "instrumentation_library_labels": instrumentationLabels, + // Omit service labels, which break agent metrics. + "service_resource_labels": serviceResourceLabels, + "resource_filters": []map[string]interface{}{}, }, } + if logBuffering { + config["log"] = map[string]any{ + "grpc_pool_size": 20, + } + config["sending_queue"] = map[string]any{ + "enabled": true, + "num_consumers": 40, + "storage": FileStorageExtensionID(), + "sizer": "bytes", + "queue_size": 50000000, //50M + } + config["timeout"] = "60s" + } + + return otel.Component{ + Type: "googlecloud", + Config: config, + } } func ConvertPrometheusExporterToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline { @@ -122,7 +137,17 @@ func (uc *UnifiedConfig) getOTelLogLevel() string { return logLevel } -func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir string) (string, error) { +// FileStorageExtensionID returns the file_storage extension used by all receivers and exporters. +func FileStorageExtensionID() string { + return "file_storage" +} + +// FileStorageExtensionDirectoryPath returns the directory path for the file_storage extension. +func FileStorageExtensionDirectoryPath(stateDir string) string { + return path.Join(stateDir, "file_storage") +} + +func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir, stateDir string) (string, error) { p := platform.FromContext(ctx) userAgent, _ := p.UserAgent("Google-Cloud-Ops-Agent-Metrics") metricVersionLabel, _ := p.VersionLabel("google-cloud-ops-agent-metrics") @@ -148,6 +173,12 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir string) if expOtlpExporter { extensions["googleclientauth"] = map[string]interface{}{} } + if uc.Logging.Service.OTelLogging { + extensions["file_storage"] = map[string]interface{}{ + "directory": FileStorageExtensionDirectoryPath(stateDir), + "create_directory": true, + } + } otelConfig, err := otel.ModularConfig{ LogLevel: uc.getOTelLogLevel(), @@ -155,8 +186,9 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir string) Pipelines: pipelines, Extensions: extensions, Exporters: map[otel.ExporterType]otel.Component{ - otel.System: googleCloudExporter(userAgent, false, false), - otel.OTel: googleCloudExporter(userAgent, true, true), + otel.System: googleCloudExporter(userAgent, false, false, false), + otel.OTel: googleCloudExporter(userAgent, true, true, false), + otel.Logs: googleCloudExporter(userAgent, true, true, true), otel.GMP: googleManagedPrometheusExporter(userAgent), otel.OTLP: otlpExporter(userAgent), }, diff --git a/confgenerator/confgenerator_test.go b/confgenerator/confgenerator_test.go index afb6dc9381..070fe802fe 100644 --- a/confgenerator/confgenerator_test.go +++ b/confgenerator/confgenerator_test.go @@ -279,7 +279,7 @@ func generateConfigs(pc platformConfig, testDir string) (got map[string]string, } // Otel configs - otelGeneratedConfig, err := mergedUc.GenerateOtelConfig(ctx, "") + otelGeneratedConfig, err := mergedUc.GenerateOtelConfig(ctx, "", "") if err != nil { return } diff --git a/confgenerator/config.go b/confgenerator/config.go index 8162587599..9515a7975e 100644 --- a/confgenerator/config.go +++ b/confgenerator/config.go @@ -1179,7 +1179,7 @@ func (uc *UnifiedConfig) OTelLoggingSupported(ctx context.Context) bool { ucLoggingCopy.Logging.Service = &LoggingService{} } ucLoggingCopy.Logging.Service.OTelLogging = true - _, err = ucLoggingCopy.GenerateOtelConfig(ctx, "") + _, err = ucLoggingCopy.GenerateOtelConfig(ctx, "", "") return err == nil } diff --git a/confgenerator/files.go b/confgenerator/files.go index 5ed4be6eff..23afc6b715 100644 --- a/confgenerator/files.go +++ b/confgenerator/files.go @@ -59,7 +59,7 @@ func (uc *UnifiedConfig) GenerateFilesFromConfig(ctx context.Context, service, l } } case "otel": - otelConfig, err := uc.GenerateOtelConfig(ctx, outDir) + otelConfig, err := uc.GenerateOtelConfig(ctx, outDir, stateDir) if err != nil { return fmt.Errorf("can't parse configuration: %w", err) } diff --git a/confgenerator/logging_receivers.go b/confgenerator/logging_receivers.go index 6bece3f0b3..f1af9ef2f1 100644 --- a/confgenerator/logging_receivers.go +++ b/confgenerator/logging_receivers.go @@ -207,6 +207,9 @@ func (r LoggingReceiverFilesMixin) Pipelines(ctx context.Context) ([]otel.Receiv "preserve_leading_whitespaces": true, "preserve_trailing_whitespaces": true, } + if !r.TransformationTest { + receiver_config["storage"] = FileStorageExtensionID() + } if i := r.WildcardRefreshInterval; i != nil { receiver_config["poll_interval"] = i.String() } @@ -241,7 +244,7 @@ func (r LoggingReceiverFilesMixin) Pipelines(ctx context.Context) ([]otel.Receiv "logs": nil, }, ExporterTypes: map[string]otel.ExporterType{ - "logs": otel.OTel, + "logs": otel.Logs, }, }}, nil } @@ -366,7 +369,7 @@ func (r LoggingReceiverSyslog) Pipelines(ctx context.Context) ([]otel.ReceiverPi }, ExporterTypes: map[string]otel.ExporterType{ - "logs": otel.OTel, + "logs": otel.Logs, }, }}, nil } @@ -606,7 +609,7 @@ func (r LoggingReceiverWindowsEventLog) Pipelines(ctx context.Context) ([]otel.R "start_at": "beginning", "poll_interval": "1s", "ignore_channel_errors": true, - // TODO: Configure storage + "storage": FileStorageExtensionID(), } var p []otel.Component @@ -633,7 +636,7 @@ func (r LoggingReceiverWindowsEventLog) Pipelines(ctx context.Context) ([]otel.R "logs": p, }, ExporterTypes: map[string]otel.ExporterType{ - "logs": otel.OTel, + "logs": otel.Logs, }, }) } @@ -966,6 +969,7 @@ func (r LoggingReceiverSystemd) Pipelines(ctx context.Context) ([]otel.ReceiverP receiver_config := map[string]any{ "start_at": "beginning", "priority": "debug", + "storage": FileStorageExtensionID(), } modify_fields_processors, err := LoggingProcessorModifyFields{ @@ -1011,7 +1015,7 @@ func (r LoggingReceiverSystemd) Pipelines(ctx context.Context) ([]otel.ReceiverP }, ExporterTypes: map[string]otel.ExporterType{ - "logs": otel.OTel, + "logs": otel.Logs, }, }}, nil } diff --git a/confgenerator/otel/modular.go b/confgenerator/otel/modular.go index 212b62b35d..0f53d92d27 100644 --- a/confgenerator/otel/modular.go +++ b/confgenerator/otel/modular.go @@ -36,6 +36,7 @@ const ( // another exporter type. OTel ExporterType = iota System + Logs GMP OTLP ) @@ -51,6 +52,8 @@ func (t ExporterType) Name() string { return "" } else if t == OTel { return "otel" + } else if t == Logs { + return "logs" } else if t == OTLP { return "otlp" } else { @@ -221,6 +224,15 @@ func (c ModularConfig) Generate(ctx context.Context, expOtlpExporter bool) (stri SetIfMissing: resourceDetectionProcessors[SetIfMissing].name("_global_1"), } + exporterTypeProcessors := map[ExporterType]Component{ + Logs: BatchProcessor(), + OTLP: CopyHostIDToInstanceID(), // b/459468648 + } + exporterTypeProcessorNames := map[ExporterType]string{ + Logs: exporterTypeProcessors[Logs].name("_global_2"), + OTLP: exporterTypeProcessors[OTLP].name("_global_3"), + } + for prefix, pipeline := range c.Pipelines { // Receiver pipelines need to be instantiated once, since they might have more than one type. // We do this work more than once if it's in more than one pipeline, but it should just overwrite the same names. @@ -251,14 +263,12 @@ func (c ModularConfig) Generate(ctx context.Context, expOtlpExporter bool) (stri if name, ok := resourceDetectionProcessorNames[rdm]; ok { processorNames = append(processorNames, name) processors[name] = resourceDetectionProcessors[rdm].Config - // b/459468648 - if expOtlpExporter { - copyProcessor := CopyHostIDToInstanceID() - processorNames = append(processorNames, copyProcessor.name("_global_0")) - processors[copyProcessor.name("_global_0")] = copyProcessor.Config - } } exporterType := receiverPipeline.ExporterTypes[pipeline.Type] + if name, ok := exporterTypeProcessorNames[exporterType]; ok { + processorNames = append(processorNames, name) + processors[name] = exporterTypeProcessors[exporterType].Config + } if _, ok := exporterNames[exporterType]; !ok { exporter := c.Exporters[exporterType] name := exporter.name(exporterType.Name()) diff --git a/confgenerator/otel/processors.go b/confgenerator/otel/processors.go index b6cb6bc4b5..858cf3a18b 100644 --- a/confgenerator/otel/processors.go +++ b/confgenerator/otel/processors.go @@ -291,6 +291,17 @@ func MetricsRemoveServiceAttributes() Component { } } +func BatchProcessor() Component { + return Component{ + Type: "batch", + Config: map[string]any{ + "send_batch_max_size": 1000, + "send_batch_size": 1000, + "timeout": "200s", + }, + } +} + func CopyHostIDToInstanceID() Component { return Component{ Type: "transform", diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml index 2e53b98f54..275417c2c6 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux-gpu/otel.yaml @@ -7,6 +7,23 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + googlecloud/logs: + log: + grpc_pool_size: 20 + metric: + instrumentation_library_labels: true + prefix: "" + resource_filters: [] + service_resource_labels: true + skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) googlecloud/otel: metric: instrumentation_library_labels: true @@ -23,6 +40,10 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -732,10 +753,11 @@ service: pipelines: logs/logs_otlp_otlp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/otlp_0 - resourcedetection/_global_1 + - batch/_global_2 receivers: - otlp/otlp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml index ba6de2ec0d..df27fd181e 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/linux/otel.yaml @@ -7,6 +7,23 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) + googlecloud/logs: + log: + grpc_pool_size: 20 + metric: + instrumentation_library_labels: true + prefix: "" + resource_filters: [] + service_resource_labels: true + skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) googlecloud/otel: metric: instrumentation_library_labels: true @@ -23,6 +40,10 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -701,10 +722,11 @@ service: pipelines: logs/logs_otlp_otlp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/otlp_0 - resourcedetection/_global_1 + - batch/_global_2 receivers: - otlp/otlp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml index eb7a638689..cec9207fc6 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows-2012/otel.yaml @@ -7,6 +7,23 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + googlecloud/logs: + log: + grpc_pool_size: 20 + metric: + instrumentation_library_labels: true + prefix: "" + resource_filters: [] + service_resource_labels: true + skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) googlecloud/otel: metric: instrumentation_library_labels: true @@ -23,6 +40,10 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -814,10 +835,11 @@ service: pipelines: logs/logs_otlp_otlp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/otlp_0 - resourcedetection/_global_1 + - batch/_global_2 receivers: - otlp/otlp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml index eb7a638689..cec9207fc6 100644 --- a/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/combined-receiver_otlp/golden/windows/otel.yaml @@ -7,6 +7,23 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) + googlecloud/logs: + log: + grpc_pool_size: 20 + metric: + instrumentation_library_labels: true + prefix: "" + resource_filters: [] + service_resource_labels: true + skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s + user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) googlecloud/otel: metric: instrumentation_library_labels: true @@ -23,6 +40,10 @@ processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -814,10 +835,11 @@ service: pipelines: logs/logs_otlp_otlp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/otlp_0 - resourcedetection/_global_1 + - batch/_global_2 receivers: - otlp/otlp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml index e00dd34e4e..10f7cfba1b 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -818,6 +835,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -832,6 +850,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -870,18 +889,21 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - filter/logs_p1_sample__logs_0 @@ -908,6 +930,7 @@ service: - filter/logs_p1_sample__logs_21 - filter/logs_p1_sample__logs_22 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml index 36d3e400df..ad67081727 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -789,6 +806,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -803,6 +821,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -839,18 +858,21 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - filter/logs_p1_sample__logs_0 @@ -877,6 +899,7 @@ service: - filter/logs_p1_sample__logs_21 - filter/logs_p1_sample__logs_22 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml index e9cfb7c108..d5290eeab6 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1180,6 +1197,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1223,16 +1241,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1266,37 +1287,42 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - filter/logs_p1_sample__logs_0 @@ -1323,6 +1349,7 @@ service: - filter/logs_p1_sample__logs_21 - filter/logs_p1_sample__logs_22 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml index e9cfb7c108..d5290eeab6 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_exclude_logs/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1180,6 +1197,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1223,16 +1241,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1266,37 +1287,42 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - filter/logs_p1_sample__logs_0 @@ -1323,6 +1349,7 @@ service: - filter/logs_p1_sample__logs_21 - filter/logs_p1_sample__logs_22 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml index 96803db406..44cc303d7d 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -791,6 +808,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -805,6 +823,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -843,22 +862,26 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml index 0c10a951c4..f92473bbb4 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -762,6 +779,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -776,6 +794,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -812,22 +831,26 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml index b955271d73..d4c1f2c2bb 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1153,6 +1170,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,16 +1214,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1239,41 +1260,47 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml index b955271d73..d4c1f2c2bb 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1153,6 +1170,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1196,16 +1214,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1239,41 +1260,47 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml index 0806e44a90..745d5ed7ae 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -718,6 +735,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -760,22 +778,26 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_1 + - batch/_global_2 receivers: - otlp/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml index c81f504e6d..1ef8e42013 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -689,6 +706,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -729,22 +747,26 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_1 + - batch/_global_2 receivers: - otlp/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml index 66ab65f1dc..6477a91130 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1113,16 +1130,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1156,41 +1176,47 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_1 + - batch/_global_2 receivers: - otlp/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml index 66ab65f1dc..6477a91130 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_modify_fields_ruby_regex/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1113,16 +1130,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1156,41 +1176,47 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_p1_sample__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/sample__logs_0 - transform/logs_p1_sample__logs_0 - resourcedetection/_global_1 + - batch/_global_2 receivers: - otlp/sample__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml index baca164dcb..08a8755109 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -758,6 +775,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -772,6 +790,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -810,22 +829,26 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/log__source__id1_0 - transform/logs_pipeline1_log__source__id1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml index 4854afdaa0..acfefb259c 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -729,6 +746,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -743,6 +761,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -779,22 +798,26 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/log__source__id1_0 - transform/logs_pipeline1_log__source__id1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml index 1a47ab8591..f9d74eedee 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1120,6 +1137,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1163,16 +1181,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1206,41 +1227,47 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/log__source__id1_0 - transform/logs_pipeline1_log__source__id1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml index 1a47ab8591..f9d74eedee 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_json/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1120,6 +1137,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1163,16 +1181,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1206,41 +1227,47 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/log__source__id1_0 - transform/logs_pipeline1_log__source__id1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml index fe1b71d191..cdcb13528f 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -729,6 +746,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -767,14 +785,17 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - transform/logs_default__pipeline_syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml index 03f6c74119..ae02bbe8e4 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -700,6 +717,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -736,14 +754,17 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - transform/logs_default__pipeline_syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml index 75604f1bf0..592b37f441 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1230,16 +1247,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1273,35 +1293,40 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/logs_default__pipeline_windows__event__log_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/logs_default__pipeline_windows__event__log_1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/logs_default__pipeline_windows__event__log_2_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml index 75604f1bf0..592b37f441 100644 --- a/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-processor_parse_regex_type_on_default_pipeline/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1230,16 +1247,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1273,35 +1293,40 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - transform/logs_default__pipeline_windows__event__log_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - transform/logs_default__pipeline_windows__event__log_1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - transform/logs_default__pipeline_windows__event__log_2_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml index 2562f7a162..b323d8e7e7 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -703,6 +720,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -717,6 +735,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -755,21 +774,25 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/log__source__id1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml index b6d7a860f1..4560af093b 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -674,6 +691,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -688,6 +706,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -724,21 +743,25 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/log__source__id1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml index 0f5b174c8e..19ead4eaa9 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1065,6 +1082,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1108,16 +1126,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1151,40 +1172,46 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/log__source__id1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml index 0f5b174c8e..19ead4eaa9 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_files_refresh_interval/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1065,6 +1082,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1108,16 +1126,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1151,40 +1172,46 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_pipeline1_log__source__id1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/log__source__id1_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/log__source__id1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml index 6619db4fec..3a7ae6f3e3 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -793,6 +810,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -807,6 +825,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -845,24 +864,28 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_kafka_kafka: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/kafka_0 - transform/kafka_1 - transform/kafka_2 - transform/kafka_3 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/kafka metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml index 527357b492..b4fb700270 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -764,6 +781,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -778,6 +796,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -814,24 +833,28 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_kafka_kafka: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/kafka_0 - transform/kafka_1 - transform/kafka_2 - transform/kafka_3 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/kafka metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml index b553a41f8a..571994eb0a 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1155,6 +1172,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1198,16 +1216,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1241,43 +1262,49 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_kafka_kafka: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/kafka_0 - transform/kafka_1 - transform/kafka_2 - transform/kafka_3 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/kafka metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml index b553a41f8a..571994eb0a 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_kafka/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1155,6 +1172,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1198,16 +1216,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1241,43 +1262,49 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_kafka_kafka: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/kafka_0 - transform/kafka_1 - transform/kafka_2 - transform/kafka_3 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/kafka metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/otel.yaml index baabb8c923..7308a4833b 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -1305,6 +1322,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/mysql__general: exclude: [] include: @@ -1318,6 +1336,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/mysql__slow: exclude: [] include: @@ -1331,6 +1350,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -1345,6 +1365,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1383,18 +1404,21 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_mysql_mysql__error: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/mysql__error_0 - transform/mysql__error_1 @@ -1402,11 +1426,12 @@ service: - transform/mysql__error_3 - transform/mysql__error_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__error logs/logs_mysql_mysql__general: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/mysql__general_0 - transform/mysql__general_1 @@ -1414,11 +1439,12 @@ service: - transform/mysql__general_3 - transform/mysql__general_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__general logs/logs_mysql_mysql__slow: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/mysql__slow_0 - transform/mysql__slow_1 @@ -1426,6 +1452,7 @@ service: - transform/mysql__slow_3 - transform/mysql__slow_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__slow metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/otel.yaml index 94e2e26971..94b4b24ac8 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -1276,6 +1293,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/mysql__general: exclude: [] include: @@ -1289,6 +1307,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/mysql__slow: exclude: [] include: @@ -1302,6 +1321,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -1316,6 +1336,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1352,18 +1373,21 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_mysql_mysql__error: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/mysql__error_0 - transform/mysql__error_1 @@ -1371,11 +1395,12 @@ service: - transform/mysql__error_3 - transform/mysql__error_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__error logs/logs_mysql_mysql__general: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/mysql__general_0 - transform/mysql__general_1 @@ -1383,11 +1408,12 @@ service: - transform/mysql__general_3 - transform/mysql__general_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__general logs/logs_mysql_mysql__slow: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/mysql__slow_0 - transform/mysql__slow_1 @@ -1395,6 +1421,7 @@ service: - transform/mysql__slow_3 - transform/mysql__slow_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__slow metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/otel.yaml index 7570ba6f32..bfcacd158f 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1667,6 +1684,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/mysql__general: exclude: [] include: @@ -1680,6 +1698,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/mysql__slow: exclude: [] include: @@ -1693,6 +1712,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1736,16 +1756,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1779,37 +1802,42 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_mysql_mysql__error: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/mysql__error_0 - transform/mysql__error_1 @@ -1817,11 +1845,12 @@ service: - transform/mysql__error_3 - transform/mysql__error_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__error logs/logs_mysql_mysql__general: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/mysql__general_0 - transform/mysql__general_1 @@ -1829,11 +1858,12 @@ service: - transform/mysql__general_3 - transform/mysql__general_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__general logs/logs_mysql_mysql__slow: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/mysql__slow_0 - transform/mysql__slow_1 @@ -1841,6 +1871,7 @@ service: - transform/mysql__slow_3 - transform/mysql__slow_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__slow metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/otel.yaml index 7570ba6f32..bfcacd158f 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_mysql/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1667,6 +1684,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/mysql__general: exclude: [] include: @@ -1680,6 +1698,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/mysql__slow: exclude: [] include: @@ -1693,6 +1712,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1736,16 +1756,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1779,37 +1802,42 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_mysql_mysql__error: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/mysql__error_0 - transform/mysql__error_1 @@ -1817,11 +1845,12 @@ service: - transform/mysql__error_3 - transform/mysql__error_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__error logs/logs_mysql_mysql__general: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/mysql__general_0 - transform/mysql__general_1 @@ -1829,11 +1858,12 @@ service: - transform/mysql__general_3 - transform/mysql__general_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__general logs/logs_mysql_mysql__slow: exporters: - - googlecloud/otel + - googlecloud/logs processors: - logstransform/mysql__slow_0 - transform/mysql__slow_1 @@ -1841,6 +1871,7 @@ service: - transform/mysql__slow_3 - transform/mysql__slow_4 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/mysql__slow metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml index dfbec29a17..d6b030e4d4 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -951,6 +968,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/nginx__error: exclude: [] include: @@ -964,6 +982,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -978,6 +997,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1016,33 +1036,38 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_nginx_nginx__access: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/nginx__access_0 - transform/nginx__access_1 - transform/nginx__access_2 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/nginx__access logs/logs_nginx_nginx__error: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/nginx__error_0 - transform/nginx__error_1 - transform/nginx__error_2 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/nginx__error metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml index 553b9ff114..c9344574fe 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -922,6 +939,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/nginx__error: exclude: [] include: @@ -935,6 +953,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/syslog: exclude: [] include: @@ -949,6 +968,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -985,33 +1005,38 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_nginx_nginx__access: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/nginx__access_0 - transform/nginx__access_1 - transform/nginx__access_2 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/nginx__access logs/logs_nginx_nginx__error: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/nginx__error_0 - transform/nginx__error_1 - transform/nginx__error_2 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/nginx__error metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml index bd359b30b8..674813745c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1313,6 +1330,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/nginx__error: exclude: [] include: @@ -1326,6 +1344,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1369,16 +1388,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1412,52 +1434,59 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_nginx_nginx__access: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/nginx__access_0 - transform/nginx__access_1 - transform/nginx__access_2 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/nginx__access logs/logs_nginx_nginx__error: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/nginx__error_0 - transform/nginx__error_1 - transform/nginx__error_2 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/nginx__error metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml index bd359b30b8..674813745c 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_nginx/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1313,6 +1330,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage filelog/nginx__error: exclude: [] include: @@ -1326,6 +1344,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -1369,16 +1388,19 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1412,52 +1434,59 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_nginx_nginx__access: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/nginx__access_0 - transform/nginx__access_1 - transform/nginx__access_2 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/nginx__access logs/logs_nginx_nginx__error: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/nginx__error_0 - transform/nginx__error_1 - transform/nginx__error_2 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/nginx__error metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml index a181caee42..d96bfd66dd 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -871,25 +888,29 @@ receivers: udp: listen_address: 2.2.2.2:2222 service: + extensions: + - file_storage pipelines: logs/logs_pipeline1_test__syslog__source__id__tcp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/test__syslog__source__id__tcp_0 - transform/test__syslog__source__id__tcp_1 - transform/logs_pipeline1_test__syslog__source__id__tcp_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - syslog/test__syslog__source__id__tcp logs/logs_pipeline2_test__syslog__source__id__udp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/test__syslog__source__id__udp_0 - transform/test__syslog__source__id__udp_1 - transform/logs_pipeline2_test__syslog__source__id__udp_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - syslog/test__syslog__source__id__udp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml index 98ce7a60bf..d40b91481d 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -840,25 +857,29 @@ receivers: udp: listen_address: 2.2.2.2:2222 service: + extensions: + - file_storage pipelines: logs/logs_pipeline1_test__syslog__source__id__tcp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/test__syslog__source__id__tcp_0 - transform/test__syslog__source__id__tcp_1 - transform/logs_pipeline1_test__syslog__source__id__tcp_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - syslog/test__syslog__source__id__tcp logs/logs_pipeline2_test__syslog__source__id__udp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/test__syslog__source__id__udp_0 - transform/test__syslog__source__id__udp_1 - transform/logs_pipeline2_test__syslog__source__id__udp_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - syslog/test__syslog__source__id__udp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml index 42606dd135..a482275061 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -953,25 +970,29 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_pipeline1_test__syslog__source__id__tcp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/test__syslog__source__id__tcp_0 - transform/test__syslog__source__id__tcp_1 - transform/logs_pipeline1_test__syslog__source__id__tcp_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - syslog/test__syslog__source__id__tcp logs/logs_pipeline2_test__syslog__source__id__udp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/test__syslog__source__id__udp_0 - transform/test__syslog__source__id__udp_1 - transform/logs_pipeline2_test__syslog__source__id__udp_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - syslog/test__syslog__source__id__udp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml index 42606dd135..a482275061 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_syslog_type_multiple_receivers/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -953,25 +970,29 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_pipeline1_test__syslog__source__id__tcp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/test__syslog__source__id__tcp_0 - transform/test__syslog__source__id__tcp_1 - transform/logs_pipeline1_test__syslog__source__id__tcp_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - syslog/test__syslog__source__id__tcp logs/logs_pipeline2_test__syslog__source__id__udp: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/test__syslog__source__id__udp_0 - transform/test__syslog__source__id__udp_1 - transform/logs_pipeline2_test__syslog__source__id__udp_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - syslog/test__syslog__source__id__udp metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml index db194d4ca9..d5c4a63968 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux-gpu/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -739,6 +756,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -757,6 +775,7 @@ receivers: journald/systemd__logs: priority: debug start_at: beginning + storage: file_storage nvml/hostmetrics_1: collection_interval: 60s otlpjsonfile/ops_agent: @@ -780,22 +799,26 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_systemd__pipeline_systemd__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/systemd__logs_0 - transform/systemd__logs_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - journald/systemd__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml index 7ae80a4e0d..ba04c38eb9 100644 --- a/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/logging-otel-receiver_systemd/golden/linux/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=linux;ShortName=linux_platform;ShortVersion=linux_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s cumulativetodelta/loggingmetrics_4: include: match_type: strict @@ -710,6 +727,7 @@ receivers: preserve_leading_whitespaces: true preserve_trailing_whitespaces: true start_at: beginning + storage: file_storage hostmetrics/hostmetrics: collection_interval: 60s scrapers: @@ -728,6 +746,7 @@ receivers: journald/systemd__logs: priority: debug start_at: beginning + storage: file_storage otlpjsonfile/ops_agent: include: - enabled_receivers_otlp.json @@ -749,22 +768,26 @@ receivers: - targets: - 0.0.0.0:20201 service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_syslog: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/syslog_0 - resourcedetection/_global_0 + - batch/_global_2 receivers: - filelog/syslog logs/logs_systemd__pipeline_systemd__logs: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/systemd__logs_0 - transform/systemd__logs_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - journald/systemd__logs metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml index 7e69ba25e6..d367cdd6c1 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml @@ -595,7 +595,7 @@ processors: resourcedetection/_global_0: detectors: - gcp - transform/_global_0: + transform/_global_3: metric_statements: - context: resource error_mode: silent @@ -768,7 +768,6 @@ service: - metricstransform/hostmetrics_2 - filter/default__pipeline_hostmetrics_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - hostmetrics/hostmetrics metrics/default__pipeline_hostmetrics_1: @@ -778,7 +777,6 @@ service: - metricstransform/hostmetrics_1_0 - filter/default__pipeline_hostmetrics_1_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - nvml/hostmetrics_1 metrics/fluentbit: @@ -789,7 +787,6 @@ service: - filter/fluentbit_0 - metricstransform/fluentbit_1 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/loggingmetrics: @@ -807,7 +804,6 @@ service: - interval/loggingmetrics_7 - metricstransform/loggingmetrics_8 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/opsagent: @@ -816,7 +812,6 @@ service: processors: - transform/ops_agent_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - otlpjsonfile/ops_agent metrics/otel: @@ -829,7 +824,6 @@ service: - filter/otel_2 - metricstransform/otel_3 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/prometheus__pipeline_prometheus: @@ -840,6 +834,7 @@ service: - resource/prometheus_1 - transform/prometheus_2 - metricstarttime/prometheus_3 + - transform/_global_3 receivers: - prometheus/prometheus telemetry: diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/otel.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/otel.yaml index 637aa315b0..518341b7f1 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux/otel.yaml @@ -566,7 +566,7 @@ processors: resourcedetection/_global_0: detectors: - gcp - transform/_global_0: + transform/_global_3: metric_statements: - context: resource error_mode: silent @@ -737,7 +737,6 @@ service: - metricstransform/hostmetrics_2 - filter/default__pipeline_hostmetrics_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - hostmetrics/hostmetrics metrics/fluentbit: @@ -748,7 +747,6 @@ service: - filter/fluentbit_0 - metricstransform/fluentbit_1 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/loggingmetrics: @@ -766,7 +764,6 @@ service: - interval/loggingmetrics_7 - metricstransform/loggingmetrics_8 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/opsagent: @@ -775,7 +772,6 @@ service: processors: - transform/ops_agent_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - otlpjsonfile/ops_agent metrics/otel: @@ -788,7 +784,6 @@ service: - filter/otel_2 - metricstransform/otel_3 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/prometheus__pipeline_prometheus: @@ -799,6 +794,7 @@ service: - resource/prometheus_1 - transform/prometheus_2 - metricstarttime/prometheus_3 + - transform/_global_3 receivers: - prometheus/prometheus telemetry: diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/otel.yaml index f674b1e655..4e6c896494 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows-2012/otel.yaml @@ -642,7 +642,7 @@ processors: resourcedetection/_global_0: detectors: - gcp - transform/_global_0: + transform/_global_3: metric_statements: - context: resource error_mode: silent @@ -860,7 +860,6 @@ service: - metricstransform/hostmetrics_2 - filter/default__pipeline_hostmetrics_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - hostmetrics/hostmetrics metrics/default__pipeline_iis: @@ -874,7 +873,7 @@ service: - resource/iis_4 - filter/default__pipeline_iis_0 - resourcedetection/_global_0 - - transform/_global_0 + - transform/_global_3 receivers: - windowsperfcounters/iis metrics/default__pipeline_mssql: @@ -886,7 +885,7 @@ service: - resource/mssql_2 - filter/default__pipeline_mssql_0 - resourcedetection/_global_0 - - transform/_global_0 + - transform/_global_3 receivers: - windowsperfcounters/mssql metrics/fluentbit: @@ -897,7 +896,6 @@ service: - filter/fluentbit_0 - metricstransform/fluentbit_1 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/loggingmetrics: @@ -915,7 +913,6 @@ service: - interval/loggingmetrics_7 - metricstransform/loggingmetrics_8 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/opsagent: @@ -924,7 +921,6 @@ service: processors: - transform/ops_agent_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - otlpjsonfile/ops_agent metrics/otel: @@ -937,7 +933,6 @@ service: - filter/otel_2 - metricstransform/otel_3 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/prometheus__pipeline_prometheus: @@ -948,6 +943,7 @@ service: - resource/prometheus_1 - transform/prometheus_2 - metricstarttime/prometheus_3 + - transform/_global_3 receivers: - prometheus/prometheus telemetry: diff --git a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/otel.yaml b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/otel.yaml index f674b1e655..4e6c896494 100644 --- a/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/windows/otel.yaml @@ -642,7 +642,7 @@ processors: resourcedetection/_global_0: detectors: - gcp - transform/_global_0: + transform/_global_3: metric_statements: - context: resource error_mode: silent @@ -860,7 +860,6 @@ service: - metricstransform/hostmetrics_2 - filter/default__pipeline_hostmetrics_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - hostmetrics/hostmetrics metrics/default__pipeline_iis: @@ -874,7 +873,7 @@ service: - resource/iis_4 - filter/default__pipeline_iis_0 - resourcedetection/_global_0 - - transform/_global_0 + - transform/_global_3 receivers: - windowsperfcounters/iis metrics/default__pipeline_mssql: @@ -886,7 +885,7 @@ service: - resource/mssql_2 - filter/default__pipeline_mssql_0 - resourcedetection/_global_0 - - transform/_global_0 + - transform/_global_3 receivers: - windowsperfcounters/mssql metrics/fluentbit: @@ -897,7 +896,6 @@ service: - filter/fluentbit_0 - metricstransform/fluentbit_1 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/loggingmetrics: @@ -915,7 +913,6 @@ service: - interval/loggingmetrics_7 - metricstransform/loggingmetrics_8 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/opsagent: @@ -924,7 +921,6 @@ service: processors: - transform/ops_agent_0 - resourcedetection/_global_0 - - transform/_global_0 receivers: - otlpjsonfile/ops_agent metrics/otel: @@ -937,7 +933,6 @@ service: - filter/otel_2 - metricstransform/otel_3 - resourcedetection/_global_0 - - transform/_global_0 receivers: - prometheus/agent_prometheus metrics/prometheus__pipeline_prometheus: @@ -948,6 +943,7 @@ service: - resource/prometheus_1 - transform/prometheus_2 - metricstarttime/prometheus_3 + - transform/_global_3 receivers: - prometheus/prometheus telemetry: diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml index 7782e9715f..866fe34bef 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1473,31 +1490,37 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2_1: channel: System ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2_2: channel: Microsoft-Windows-User Control Panel/Operational ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1531,59 +1554,67 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_winlog2_winlog2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_0 - transform/winlog2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2 logs/logs_winlog2_winlog2_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_1_0 - transform/winlog2_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2_1 logs/logs_winlog2_winlog2_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_2_0 - transform/winlog2_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2_2 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml index 7782e9715f..866fe34bef 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_new_channels/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1473,31 +1490,37 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2_1: channel: System ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2_2: channel: Microsoft-Windows-User Control Panel/Operational ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1531,59 +1554,67 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_winlog2_winlog2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_0 - transform/winlog2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2 logs/logs_winlog2_winlog2_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_1_0 - transform/winlog2_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2_1 logs/logs_winlog2_winlog2_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_2_0 - transform/winlog2_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2_2 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml index e0f1829722..9c14b45592 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows-2012/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1175,28 +1192,33 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2: channel: Application ignore_channel_errors: true include_log_record_original: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2_1: channel: System ignore_channel_errors: true include_log_record_original: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1230,50 +1252,57 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_winlog2_winlog2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_0 - transform/winlog2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2 logs/logs_winlog2_winlog2_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_1_0 - transform/winlog2_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2_1 metrics/default__pipeline_hostmetrics: diff --git a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml index e0f1829722..9c14b45592 100644 --- a/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml +++ b/confgenerator/testdata/goldens/windows-otel-logging-receiver_winlog2_xml/golden/windows/otel.yaml @@ -7,18 +7,35 @@ exporters: service_resource_labels: false skip_create_descriptor: true user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) - googlecloud/otel: + googlecloud/logs: + log: + grpc_pool_size: 20 metric: instrumentation_library_labels: true prefix: "" resource_filters: [] service_resource_labels: true skip_create_descriptor: true + sending_queue: + enabled: true + num_consumers: 40 + queue_size: 50000000 + sizer: bytes + storage: file_storage + timeout: 60s user_agent: Google-Cloud-Ops-Agent-Metrics/latest (BuildDistro=build_distro;Platform=windows;ShortName=win_platform;ShortVersion=win_platform_version) +extensions: + file_storage: + create_directory: true + directory: file_storage processors: agentmetrics/hostmetrics_0: blank_label_metrics: - system.cpu.utilization + batch/_global_2: + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 200s casttosum/iis_1: metrics: - agent.googleapis.com/iis/network/transferred_bytes_count @@ -1175,28 +1192,33 @@ receivers: ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_1: channel: Application ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/windows__event__log_2: channel: Security ignore_channel_errors: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2: channel: Application ignore_channel_errors: true include_log_record_original: true poll_interval: 1s start_at: beginning + storage: file_storage windowseventlog/winlog2_1: channel: System ignore_channel_errors: true include_log_record_original: true poll_interval: 1s start_at: beginning + storage: file_storage windowsperfcounters/iis: collection_interval: 60s perfcounters: @@ -1230,50 +1252,57 @@ receivers: - _Total object: SQLServer:Databases service: + extensions: + - file_storage pipelines: logs/logs_default__pipeline_windows__event__log: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_0 - transform/windows__event__log_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log logs/logs_default__pipeline_windows__event__log_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_1_0 - transform/windows__event__log_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_1 logs/logs_default__pipeline_windows__event__log_2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/windows__event__log_2_0 - transform/windows__event__log_2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/windows__event__log_2 logs/logs_winlog2_winlog2: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_0 - transform/winlog2_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2 logs/logs_winlog2_winlog2_1: exporters: - - googlecloud/otel + - googlecloud/logs processors: - transform/winlog2_1_0 - transform/winlog2_1_1 - resourcedetection/_global_0 + - batch/_global_2 receivers: - windowseventlog/winlog2_1 metrics/default__pipeline_hostmetrics: diff --git a/integration_test/soak_test/cmd/launcher/main.go b/integration_test/soak_test/cmd/launcher/main.go index 0404b49b60..d0b70f80ab 100644 --- a/integration_test/soak_test/cmd/launcher/main.go +++ b/integration_test/soak_test/cmd/launcher/main.go @@ -104,6 +104,52 @@ New-ItemProperty -Path 'HKLM:\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate\ `) } +// setExperimentalFeatures sets the EXPERIMENTAL_FEATURES environment variable. +func setExperimentalFeatures(ctx context.Context, logger *log.Logger, vm *gce.VM, feature string) error { + return gce.SetEnvironmentVariables(ctx, logger, vm, map[string]string{"EXPERIMENTAL_FEATURES": feature}) +} + +// defaultOtelLoggingConfig returns the default config that is required to use otel_logging. +func defaultOtelLoggingConfig() string { + return `logging: + service: + experimental_otel_logging: true +` +} + +// setExperimentalOtelLoggingInConfig in an Ops Agent config +func setExperimentalOtelLoggingInConfig(config string) string { + return strings.Replace( + config, + "service:\n", + "service:\n experimental_otel_logging: true\n", + 1, + ) +} + +// SetupOpsAgentWithFeatureFlag configures the VM and the config depending on the selected feature flag. +func SetupOpsAgentWithFeatureFlag(ctx context.Context, logger *log.Logger, vm *gce.VM, config string, feature string) error { + switch feature { + case OtelLoggingFeatureFlag: + // Set feature flag in config. + if config == "" { + config = defaultOtelLoggingConfig() + } else { + config = setExperimentalOtelLoggingInConfig(config) + } + // Set experimental feature environment variable. + if err := setExperimentalFeatures(ctx, logger, vm, feature); err != nil { + return err + } + } + return agents.SetupOpsAgent(ctx, logger, vm, config) +} + +const ( + OtelLoggingFeatureFlag = "otel_logging" + DefaultFeatureFlag = "default" +) + func mainErr() error { defer gce.CleanupKeysOrDie() @@ -159,18 +205,14 @@ func mainErr() error { type: files include_paths: - %s - exporters: - google: - type: google_cloud_logging service: pipelines: my_pipeline: receivers: - mylog_source - generator_debug_logs - exporters: [google] `, logPath, debugLogPath) - if err := agents.SetupOpsAgent(ctx, logger, vm, config); err != nil { + if err := SetupOpsAgentWithFeatureFlag(ctx, logger, vm, config, OtelLoggingFeatureFlag); err != nil { return err } diff --git a/submodules/opentelemetry-operations-collector b/submodules/opentelemetry-operations-collector index 6c8bfa06cc..ec757f2f48 160000 --- a/submodules/opentelemetry-operations-collector +++ b/submodules/opentelemetry-operations-collector @@ -1 +1 @@ -Subproject commit 6c8bfa06cc3c84a181ac68b61914e79c2c41124f +Subproject commit ec757f2f48c865c7aa1afaed27891d8727a28f2e diff --git a/systemd/google-cloud-ops-agent-opentelemetry-collector.service b/systemd/google-cloud-ops-agent-opentelemetry-collector.service index b51d1c1bbc..b4cd4a14eb 100644 --- a/systemd/google-cloud-ops-agent-opentelemetry-collector.service +++ b/systemd/google-cloud-ops-agent-opentelemetry-collector.service @@ -23,7 +23,7 @@ RuntimeDirectory=google-cloud-ops-agent-opentelemetry-collector StateDirectory=google-cloud-ops-agent/opentelemetry-collector LogsDirectory=google-cloud-ops-agent Type=simple -ExecStartPre=@PREFIX@/libexec/google_cloud_ops_agent_engine -service=otel -in @SYSCONFDIR@/google-cloud-ops-agent/config.yaml -logs ${LOGS_DIRECTORY} +ExecStartPre=@PREFIX@/libexec/google_cloud_ops_agent_engine -service=otel -in @SYSCONFDIR@/google-cloud-ops-agent/config.yaml -logs ${LOGS_DIRECTORY} -state ${STATE_DIRECTORY} ExecStart=@PREFIX@/subagents/opentelemetry-collector/otelopscol --config=${RUNTIME_DIRECTORY}/otel.yaml --feature-gates=receiver.prometheusreceiver.RemoveStartTimeAdjustment Restart=always # For debugging: diff --git a/transformation_test/testdata/logging_processor-cassandra-debug/output_otel.yaml b/transformation_test/testdata/logging_processor-cassandra-debug/output_otel.yaml index 8dfee3b0a9..d517e8cc20 100644 --- a/transformation_test/testdata/logging_processor-cassandra-debug/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-cassandra-debug/output_otel.yaml @@ -339,8 +339,6 @@ type: gce_instance severity: DEBUG timestamp: 2025-07-22T15:15:42.200Z - partialSuccess: true -- entries: - jsonPayload: javaClass: MerkleTree.java level: DEBUG diff --git a/transformation_test/testdata/logging_processor-cassandra-gc/output_otel.yaml b/transformation_test/testdata/logging_processor-cassandra-gc/output_otel.yaml index 201b595b45..93f650ad3d 100644 --- a/transformation_test/testdata/logging_processor-cassandra-gc/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-cassandra-gc/output_otel.yaml @@ -72,8 +72,6 @@ zone: test-zone type: gce_instance timestamp: 2025-07-22T13:45:30.678Z - partialSuccess: true -- entries: - jsonPayload: message: "2025-07-22T14:51:23.332+0000][4.595s][5195][5217][info ] Total time for which application threads were stopped: 0.0003091 seconds, Stopping threads took: 0.0000181 seconds" labels: diff --git a/transformation_test/testdata/logging_processor-cassandra-system/output_otel.yaml b/transformation_test/testdata/logging_processor-cassandra-system/output_otel.yaml index 79b76d3642..0245d9b352 100644 --- a/transformation_test/testdata/logging_processor-cassandra-system/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-cassandra-system/output_otel.yaml @@ -178,8 +178,6 @@ type: gce_instance severity: WARNING timestamp: 2025-07-22T18:40:00.409Z - partialSuccess: true -- entries: - jsonPayload: javaClass: HintsWriteExecutor.java level: ERROR diff --git a/transformation_test/testdata/logging_processor-couchdb/output_otel.yaml b/transformation_test/testdata/logging_processor-couchdb/output_otel.yaml index fbd197f3a3..20ba8af105 100644 --- a/transformation_test/testdata/logging_processor-couchdb/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-couchdb/output_otel.yaml @@ -72,8 +72,6 @@ type: gce_instance severity: WARNING timestamp: 2022-01-12T16:55:21.782456Z - partialSuccess: true -- entries: - jsonPayload: host: nohost level: info diff --git a/transformation_test/testdata/logging_processor-flink/output_otel.yaml b/transformation_test/testdata/logging_processor-flink/output_otel.yaml index 01cb854974..88c78d51cb 100644 --- a/transformation_test/testdata/logging_processor-flink/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-flink/output_otel.yaml @@ -194,8 +194,6 @@ type: gce_instance severity: DEBUG timestamp: 2025-05-21T04:34:14.094Z - partialSuccess: true -- entries: - jsonPayload: level: TRACE message: " [] - Successfully recovered 0 persisted job graphs." diff --git a/transformation_test/testdata/logging_processor-hadoop/output_otel.yaml b/transformation_test/testdata/logging_processor-hadoop/output_otel.yaml index a9ce672afe..b2cb094386 100644 --- a/transformation_test/testdata/logging_processor-hadoop/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-hadoop/output_otel.yaml @@ -81,8 +81,6 @@ type: gce_instance severity: ERROR timestamp: 2022-02-02T02:09:51.500Z - partialSuccess: true -- entries: - jsonPayload: message: Successfully started datanode on port 9866 severity: INFO diff --git a/transformation_test/testdata/logging_processor-hbase-system/output_otel.yaml b/transformation_test/testdata/logging_processor-hbase-system/output_otel.yaml index da7b04e8bf..a928422f9f 100644 --- a/transformation_test/testdata/logging_processor-hbase-system/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-hbase-system/output_otel.yaml @@ -367,8 +367,6 @@ type: gce_instance severity: INFO timestamp: 2025-07-23T05:19:58.105Z - partialSuccess: true -- entries: - jsonPayload: level: INFO message: "master.HMaster: HMaster shutdown completed" diff --git a/transformation_test/testdata/logging_processor-kafka/output_otel.yaml b/transformation_test/testdata/logging_processor-kafka/output_otel.yaml index 66df2effb3..4d51502d57 100644 --- a/transformation_test/testdata/logging_processor-kafka/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-kafka/output_otel.yaml @@ -192,8 +192,6 @@ type: gce_instance severity: INFO timestamp: 2025-07-22T16:02:11.567Z - partialSuccess: true -- entries: - jsonPayload: level: INFO logger: kafka.coordinator.group.GroupCoordinator diff --git a/transformation_test/testdata/logging_processor-mysql-general/output_otel.yaml b/transformation_test/testdata/logging_processor-mysql-general/output_otel.yaml index 62476d4e99..2a4ade6952 100644 --- a/transformation_test/testdata/logging_processor-mysql-general/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-mysql-general/output_otel.yaml @@ -111,8 +111,6 @@ zone: test-zone type: gce_instance timestamp: 2025-07-22T16:00:08.890123Z - partialSuccess: true -- entries: - jsonPayload: command: Query message: SELECT * FROM sessions WHERE active = 1; diff --git a/transformation_test/testdata/logging_processor-mysql-slow/output_otel.yaml b/transformation_test/testdata/logging_processor-mysql-slow/output_otel.yaml index b037ef7b24..7aa90e6c82 100644 --- a/transformation_test/testdata/logging_processor-mysql-slow/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-mysql-slow/output_otel.yaml @@ -137,8 +137,6 @@ zone: test-zone type: gce_instance timestamp: now - partialSuccess: true -- entries: - jsonPayload: host: 172.18.0.2 lockTime: 0.000311 diff --git a/transformation_test/testdata/logging_processor-oracledb_alert/output_otel.yaml b/transformation_test/testdata/logging_processor-oracledb_alert/output_otel.yaml index bbd5e189d1..d9a55b3b9d 100644 --- a/transformation_test/testdata/logging_processor-oracledb_alert/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-oracledb_alert/output_otel.yaml @@ -16,8 +16,6 @@ type: gce_instance severity: ALERT timestamp: now - partialSuccess: true -- entries: - jsonPayload: message: |- Checkpoint not complete, extending checkpoint until all current diff --git a/transformation_test/testdata/logging_processor-postgresql/output_otel.yaml b/transformation_test/testdata/logging_processor-postgresql/output_otel.yaml index c888798526..a62d732e45 100644 --- a/transformation_test/testdata/logging_processor-postgresql/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-postgresql/output_otel.yaml @@ -356,8 +356,6 @@ type: gce_instance severity: INFO timestamp: now - partialSuccess: true -- entries: - jsonPayload: level: LOG message: database system is ready to accept connections diff --git a/transformation_test/testdata/logging_processor-rabbitmq/output_otel.yaml b/transformation_test/testdata/logging_processor-rabbitmq/output_otel.yaml index 2567d3661f..c12b35f412 100644 --- a/transformation_test/testdata/logging_processor-rabbitmq/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-rabbitmq/output_otel.yaml @@ -201,8 +201,6 @@ zone: test-zone type: gce_instance timestamp: now - partialSuccess: true -- entries: - jsonPayload: message: "{shutdown,{connection_closed, \"Broken pipe\"}}" process_id: 123.456.789 diff --git a/transformation_test/testdata/logging_processor-solr/output_otel.yaml b/transformation_test/testdata/logging_processor-solr/output_otel.yaml index 2cc5c66143..d344e45368 100644 --- a/transformation_test/testdata/logging_processor-solr/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-solr/output_otel.yaml @@ -184,8 +184,6 @@ type: gce_instance severity: INFO timestamp: 2025-07-21T23:58:55.013Z - partialSuccess: true -- entries: - jsonPayload: level: INFO message: Startup complete. Ready to accept requests. diff --git a/transformation_test/testdata/logging_processor-tomcat-system/output_otel.yaml b/transformation_test/testdata/logging_processor-tomcat-system/output_otel.yaml index 1107d0823c..a7b332366d 100644 --- a/transformation_test/testdata/logging_processor-tomcat-system/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-tomcat-system/output_otel.yaml @@ -432,8 +432,6 @@ type: gce_instance severity: INFO timestamp: 2025-07-22T01:05:22.347Z - partialSuccess: true -- entries: - jsonPayload: level: INFO message: org.apache.catalina.core.StandardService Catalina service stopped. diff --git a/transformation_test/testdata/logging_processor-vault-audit/output_otel.yaml b/transformation_test/testdata/logging_processor-vault-audit/output_otel.yaml index 76ef562773..252040becd 100644 --- a/transformation_test/testdata/logging_processor-vault-audit/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-vault-audit/output_otel.yaml @@ -131,8 +131,6 @@ zone: test-zone type: gce_instance timestamp: 2025-08-25T15:02:00.111213141Z - partialSuccess: true -- entries: - jsonPayload: auth: token_type: default diff --git a/transformation_test/testdata/logging_processor-wildfly_system/output_otel.yaml b/transformation_test/testdata/logging_processor-wildfly_system/output_otel.yaml index ca505b5fce..836a03d35d 100644 --- a/transformation_test/testdata/logging_processor-wildfly_system/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-wildfly_system/output_otel.yaml @@ -104,8 +104,6 @@ type: gce_instance severity: DEBUG timestamp: 2022-02-03T23:38:01.506Z - partialSuccess: true -- entries: - jsonPayload: level: INFO message: "WFLYSRV0010: Deployed \"example.war\" (runtime-name : \"example.war\")" diff --git a/transformation_test/testdata/logging_processor-zookeeper_general/output_otel.yaml b/transformation_test/testdata/logging_processor-zookeeper_general/output_otel.yaml index 6cbdf00d52..cb229f2d9a 100644 --- a/transformation_test/testdata/logging_processor-zookeeper_general/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-zookeeper_general/output_otel.yaml @@ -82,8 +82,6 @@ type: gce_instance severity: WARNING timestamp: 2022-02-01T08:46:33.626Z - partialSuccess: true -- entries: - jsonPayload: level: ERROR line: 1520 diff --git a/transformation_test/testdata/ops_agent_test-TestLogEntrySpecialFields/output_otel.yaml b/transformation_test/testdata/ops_agent_test-TestLogEntrySpecialFields/output_otel.yaml index cf8ee02237..516422ca86 100644 --- a/transformation_test/testdata/ops_agent_test-TestLogEntrySpecialFields/output_otel.yaml +++ b/transformation_test/testdata/ops_agent_test-TestLogEntrySpecialFields/output_otel.yaml @@ -3,7 +3,7 @@ error: "could not process attribute gcp.source_location: json: cannot unmarshal string into Go struct field LogEntrySourceLocation.line of type int64" level: error msg: Exporting failed. Rejecting data. Try enabling sending_queue to survive temporary failures. - otelcol.component.id: googlecloud/otel + otelcol.component.id: googlecloud/logs otelcol.component.kind: exporter otelcol.signal: logs rejected_items: 1.0 @@ -22,65 +22,21 @@ go.opentelemetry.io/collector/service@v0.138.0/internal/refconsumer/logs.go:29 go.opentelemetry.io/collector/internal/fanoutconsumer.(*logsConsumer).ConsumeLogs go.opentelemetry.io/collector/internal/fanoutconsumer@v0.138.0/logs.go:62 - go.opentelemetry.io/collector/processor/processorhelper.NewLogs.func1 - go.opentelemetry.io/collector/processor/processorhelper@v0.138.0/logs.go:71 - go.opentelemetry.io/collector/consumer.ConsumeLogsFunc.ConsumeLogs - go.opentelemetry.io/collector/consumer@v1.44.0/logs.go:27 - go.opentelemetry.io/collector/service/internal/refconsumer.refLogs.ConsumeLogs - go.opentelemetry.io/collector/service@v0.138.0/internal/refconsumer/logs.go:29 - go.opentelemetry.io/collector/processor/processorhelper.NewLogs.func1 - go.opentelemetry.io/collector/processor/processorhelper@v0.138.0/logs.go:71 - go.opentelemetry.io/collector/consumer.ConsumeLogsFunc.ConsumeLogs - go.opentelemetry.io/collector/consumer@v1.44.0/logs.go:27 - go.opentelemetry.io/collector/service/internal/refconsumer.refLogs.ConsumeLogs - go.opentelemetry.io/collector/service@v0.138.0/internal/refconsumer/logs.go:29 - go.opentelemetry.io/collector/processor/processorhelper.NewLogs.func1 - go.opentelemetry.io/collector/processor/processorhelper@v0.138.0/logs.go:71 - go.opentelemetry.io/collector/consumer.ConsumeLogsFunc.ConsumeLogs - go.opentelemetry.io/collector/consumer@v1.44.0/logs.go:27 - go.opentelemetry.io/collector/service/internal/refconsumer.refLogs.ConsumeLogs - go.opentelemetry.io/collector/service@v0.138.0/internal/refconsumer/logs.go:29 - go.opentelemetry.io/collector/consumer.ConsumeLogsFunc.ConsumeLogs - go.opentelemetry.io/collector/consumer@v1.44.0/logs.go:27 - go.opentelemetry.io/collector/internal/fanoutconsumer.(*logsConsumer).ConsumeLogs - go.opentelemetry.io/collector/internal/fanoutconsumer@v0.138.0/logs.go:62 - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/consumerretry.(*logsConsumer).ConsumeLogs - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal@v0.138.0/consumerretry/logs.go:37 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/adapter.(*receiver).consumeEntries - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.138.0/adapter/receiver.go:59 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*BatchingLogEmitter).flusher - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.138.0/operator/helper/emitter.go:171 - - caller: consumerretry/logs.go:39 + go.opentelemetry.io/collector/processor/batchprocessor.(*batchLogs).export + go.opentelemetry.io/collector/processor/batchprocessor@v0.138.0/batch_processor.go:548 + go.opentelemetry.io/collector/processor/batchprocessor.(*shard[...]).sendItems + go.opentelemetry.io/collector/processor/batchprocessor@v0.138.0/batch_processor.go:273 + go.opentelemetry.io/collector/processor/batchprocessor.(*shard[...]).startLoop + go.opentelemetry.io/collector/processor/batchprocessor@v0.138.0/batch_processor.go:216 + - caller: batchprocessor@v0.138.0/batch_processor.go:275 error: "could not process attribute gcp.source_location: json: cannot unmarshal string into Go struct field LogEntrySourceLocation.line of type int64" - level: error - msg: ConsumeLogs() failed. Enable retry_on_failure to slow down reading logs and avoid dropping. - otelcol.component.id: filelog/my-log-name - otelcol.component.kind: receiver + level: warn + msg: Sender failed + otelcol.component.id: batch/_global_2 + otelcol.component.kind: processor + otelcol.pipeline.id: logs/logs_transformation__test_my-log-name otelcol.signal: logs resource: service.instance.id: test-service-instance-id service.name: otelopscol service.version: v0.138.0 - stacktrace: |- - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/consumerretry.(*logsConsumer).ConsumeLogs - github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal@v0.138.0/consumerretry/logs.go:39 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/adapter.(*receiver).consumeEntries - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.138.0/adapter/receiver.go:59 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*BatchingLogEmitter).flusher - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.138.0/operator/helper/emitter.go:171 - - caller: adapter/receiver.go:61 - error: "could not process attribute gcp.source_location: json: cannot unmarshal string into Go struct field LogEntrySourceLocation.line of type int64" - level: error - msg: ConsumeLogs() failed - otelcol.component.id: filelog/my-log-name - otelcol.component.kind: receiver - otelcol.signal: logs - resource: - service.instance.id: test-service-instance-id - service.name: otelopscol - service.version: v0.138.0 - stacktrace: |- - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/adapter.(*receiver).consumeEntries - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.138.0/adapter/receiver.go:61 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper.(*BatchingLogEmitter).flusher - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza@v0.138.0/operator/helper/emitter.go:171 diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go index 454ddb7926..945856360d 100644 --- a/transformation_test/transformation_test.go +++ b/transformation_test/transformation_test.go @@ -404,7 +404,7 @@ func (transformationConfig transformationTest) generateOTelConfig(ctx context.Co ReceiverPipelines: rps, Pipelines: pls, Exporters: map[otel.ExporterType]otel.Component{ - otel.OTel: { + otel.Logs: { Type: "googlecloud", Config: map[string]any{ "project": "my-project",