Using otlphttp exporter for self metrics

confgenerator/agentmetrics.go

@@ -15,6 +15,7 @@
 package confgenerator
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 	"time"
@@ -56,9 +57,9 @@ var grpcToHTTPStatus = map[string]string{
 	"DEADLINE_EXCEEDED":   "504",
 }
 
-func (r AgentSelfMetrics) AddSelfMetricsPipelines(receiverPipelines map[string]otel.ReceiverPipeline, pipelines map[string]otel.Pipeline) {
+func (r AgentSelfMetrics) AddSelfMetricsPipelines(receiverPipelines map[string]otel.ReceiverPipeline, pipelines map[string]otel.Pipeline, ctx context.Context) {
 	// Receiver pipelines names should have 1 underscore to avoid collision with user configurations.
-	receiverPipelines["agent_prometheus"] = r.PrometheusMetricsPipeline()
+	receiverPipelines["agent_prometheus"] = r.PrometheusMetricsPipeline(ctx)
 
 	// Pipeline names should have no underscores to avoid collision with user configurations.
 	pipelines["otel"] = otel.Pipeline{
@@ -79,15 +80,15 @@ func (r AgentSelfMetrics) AddSelfMetricsPipelines(receiverPipelines map[string]o
 		Processors:           r.LoggingMetricsPipelineProcessors(),
 	}
 
-	receiverPipelines["ops_agent"] = r.OpsAgentPipeline()
+	receiverPipelines["ops_agent"] = r.OpsAgentPipeline(ctx)
 	pipelines["opsagent"] = otel.Pipeline{
 		Type:                 "metrics",
 		ReceiverPipelineName: "ops_agent",
 	}
 }
 
-func (r AgentSelfMetrics) PrometheusMetricsPipeline() otel.ReceiverPipeline {
-	return otel.ReceiverPipeline{
+func (r AgentSelfMetrics) PrometheusMetricsPipeline(ctx context.Context) otel.ReceiverPipeline {
+	return ConvertGCMSystemExporterToOtlpExporter(otel.ReceiverPipeline{
 		Receiver: otel.Component{
 			Type: "prometheus",
 			Config: map[string]interface{}{
@@ -128,7 +129,7 @@ func (r AgentSelfMetrics) PrometheusMetricsPipeline() otel.ReceiverPipeline {
 				),
 			},
 		},
-	}
+	}, ctx)
 }
 
 func (r AgentSelfMetrics) OtelPipelineProcessors() []otel.Component {
@@ -303,18 +304,18 @@ func (r AgentSelfMetrics) LoggingMetricsPipelineProcessors() []otel.Component {
 	}
 }
 
-func (r AgentSelfMetrics) OpsAgentPipeline() otel.ReceiverPipeline {
-	receiver_config := map[string]any{
+func (r AgentSelfMetrics) OpsAgentPipeline(ctx context.Context) otel.ReceiverPipeline {
+	receiverConfig := map[string]any{
 		"include": []string{
 			filepath.Join(r.OtelRuntimeDir, "enabled_receivers_otlp.json"),
 			filepath.Join(r.OtelRuntimeDir, "feature_tracking_otlp.json")},
 		"replay_file":   true,
 		"poll_interval": time.Duration(60 * time.Second).String(),
 	}
-	return otel.ReceiverPipeline{
+	return ConvertGCMSystemExporterToOtlpExporter(otel.ReceiverPipeline{
 		Receiver: otel.Component{
 			Type:   "otlpjsonfile",
-			Config: receiver_config,
+			Config: receiverConfig,
 		},
 		ExporterTypes: map[string]otel.ExporterType{
 			"metrics": otel.System,
@@ -324,7 +325,7 @@ func (r AgentSelfMetrics) OpsAgentPipeline() otel.ReceiverPipeline {
 				otel.Transform("metric", "datapoint", []ottl.Statement{"set(time, Now())"}),
 			},
 		},
-	}
+	}, ctx)
 }
 
 // intentionally not registered as a component because this is not created by users

confgenerator/confgenerator.go

@@ -57,32 +57,43 @@ func googleCloudExporter(userAgent string, instrumentationLabels bool, serviceRe
 	}
 }
 
-func ConvertPrometheusExporterToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline {
-	return ConvertToOtlpExporter(receiver, ctx, true)
+func otelSetOtlpExporterComponents() []otel.Component {
+	return []otel.Component{otel.MetricStartTime()}
 }
 
-func ConvertGCMOtelExporterToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline {
-	return ConvertToOtlpExporter(receiver, ctx, false)
+func ConvertPrometheusExporterToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline {
+	return ConvertToOtlpExporter(pipeline, ctx, true, false)
 }
 
-func ConvertToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context, isPrometheus bool) otel.ReceiverPipeline {
+func ConvertGCMOtelExporterToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline {
+	return ConvertToOtlpExporter(pipeline, ctx, false, false)
+}
+
+func ConvertGCMSystemExporterToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline {
+	return ConvertToOtlpExporter(pipeline, ctx, false, true)
+}
+
+func ConvertToOtlpExporter(pipeline otel.ReceiverPipeline, ctx context.Context, isPrometheus bool, isSystem bool) otel.ReceiverPipeline {
 	expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"]
 	resource, _ := platform.FromContext(ctx).GetResource()
 	if !expOtlpExporter {
-		return receiver
+		return pipeline
 	}
-	_, err := receiver.ExporterTypes["metrics"]
+	_, err := pipeline.ExporterTypes["metrics"]
 	if !err {
-		return receiver
+		return pipeline
 	}
-	receiver.ExporterTypes["metrics"] = otel.OTLP
+	pipeline.ExporterTypes["metrics"] = otel.OTLP
 
-	receiver.Processors["metrics"] = append(receiver.Processors["metrics"], otel.GCPProjectID(resource.ProjectName()))
+	pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.GCPProjectID(resource.ProjectName()))
+	if isSystem {
+		pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricsRemoveInstrumentationLibraryLabelsAttributes())
+	}
 	if isPrometheus {
-		receiver.Processors["metrics"] = append(receiver.Processors["metrics"], otel.MetricUnknownCounter())
-		receiver.Processors["metrics"] = append(receiver.Processors["metrics"], otel.MetricStartTime())
+		pipeline.Processors["metrics"] = append(pipeline.Processors["metrics"], otel.MetricUnknownCounter())
+
 	}
-	return receiver
+	return pipeline
 }
 
 func otlpExporter(userAgent string) otel.Component {
@@ -127,6 +138,7 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir string)
 	userAgent, _ := p.UserAgent("Google-Cloud-Ops-Agent-Metrics")
 	metricVersionLabel, _ := p.VersionLabel("google-cloud-ops-agent-metrics")
 	loggingVersionLabel, _ := p.VersionLabel("google-cloud-ops-agent-logging")
+	expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"]
 
 	receiverPipelines, pipelines, err := uc.generateOtelPipelines(ctx)
 	if err != nil {
@@ -141,9 +153,8 @@ func (uc *UnifiedConfig) GenerateOtelConfig(ctx context.Context, outDir string)
 		OtelRuntimeDir:      outDir,
 		OtelLogging:         uc.Logging.Service.OTelLogging,
 	}
-	agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines)
+	agentSelfMetrics.AddSelfMetricsPipelines(receiverPipelines, pipelines, ctx)
 
-	expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"]
 	extensions := map[string]interface{}{}
 	if expOtlpExporter {
 		extensions["googleclientauth"] = map[string]interface{}{}

confgenerator/otel/modular.go

@@ -256,6 +256,12 @@ func (c ModularConfig) Generate(ctx context.Context, expOtlpExporter bool) (stri
 				copyProcessor := CopyHostIDToInstanceID()
 				processorNames = append(processorNames, copyProcessor.name("_global_0"))
 				processors[copyProcessor.name("_global_0")] = copyProcessor.Config
+				// Similar to the resource detector, for any pipeline that is using the
+				// otlphttp exporter, we add a MetricStartTime processor at the end of it.
+				// This mimics the current behavior on GCM exporter.
+				metricStartTime := MetricStartTime()
+				processorNames = append(processorNames, metricStartTime.name(fmt.Sprintf("%s_0", prefix)))
+				processors[metricStartTime.name(fmt.Sprintf("%s_0", prefix))] = metricStartTime.Config
 			}
 		}
 		exporterType := receiverPipeline.ExporterTypes[pipeline.Type]

confgenerator/otel/processors.go

@@ -291,6 +291,13 @@ func MetricsRemoveServiceAttributes() Component {
 	}
 }
 
+func MetricsRemoveInstrumentationLibraryLabelsAttributes() Component {
+	return TransformationMetrics(
+		SetScopeName(""),
+		SetScopeVersion(""),
+	)
+}
+
 func CopyHostIDToInstanceID() Component {
 	return Component{
 		Type: "transform",

confgenerator/testdata/goldens/metrics-exporter_prometheus_otlp/golden/linux-gpu/otel.yaml

@@ -86,7 +86,17 @@ processors:
         - googlecloudmonitoring/point_count
   interval/loggingmetrics_7:
     interval: 1m
-  metricstarttime/prometheus_3:
+  metricstarttime/default__pipeline_hostmetrics_0:
+    strategy: subtract_initial_point
+  metricstarttime/default__pipeline_hostmetrics_1_0:
+    strategy: subtract_initial_point
+  metricstarttime/fluentbit_0:
+    strategy: subtract_initial_point
+  metricstarttime/loggingmetrics_0:
+    strategy: subtract_initial_point
+  metricstarttime/opsagent_0:
+    strategy: subtract_initial_point
+  metricstarttime/otel_0:
     strategy: subtract_initial_point
   metricstransform/fluentbit_1:
     transforms:
@@ -587,6 +597,16 @@ processors:
       include: ^(.*)$$
       match_type: regexp
       new_name: agent.googleapis.com/$${1}
+  resource/agent_prometheus_1:
+    attributes:
+    - action: insert
+      key: gcp.project_id
+      value: test-project
+  resource/ops_agent_1:
+    attributes:
+    - action: insert
+      key: gcp.project_id
+      value: test-project
   resource/prometheus_1:
     attributes:
     - action: insert
@@ -610,6 +630,12 @@ processors:
       - delete_key(resource.attributes, "service.instance.id")
       - delete_key(resource.attributes, "server.port")
       - delete_key(resource.attributes, "url.scheme")
+  transform/agent_prometheus_2:
+    metric_statements:
+    - context: scope
+      statements:
+      - set(name, "")
+      - set(version, "")
   transform/loggingmetrics_0:
     error_mode: ignore
     metric_statements:
@@ -637,6 +663,12 @@ processors:
     - context: datapoint
       statements:
       - set(time, Now())
+  transform/ops_agent_2:
+    metric_statements:
+    - context: scope
+      statements:
+      - set(name, "")
+      - set(version, "")
   transform/otel_0:
     error_mode: ignore
     metric_statements:
@@ -769,6 +801,7 @@ service:
       - filter/default__pipeline_hostmetrics_0
       - resourcedetection/_global_0
       - transform/_global_0
+      - metricstarttime/default__pipeline_hostmetrics_0
       receivers:
       - hostmetrics/hostmetrics
     metrics/default__pipeline_hostmetrics_1:
@@ -779,24 +812,30 @@ service:
       - filter/default__pipeline_hostmetrics_1_0
       - resourcedetection/_global_0
       - transform/_global_0
+      - metricstarttime/default__pipeline_hostmetrics_1_0
       receivers:
       - nvml/hostmetrics_1
     metrics/fluentbit:
       exporters:
-      - googlecloud
+      - otlphttp/otlp
       processors:
       - transform/agent_prometheus_0
+      - resource/agent_prometheus_1
+      - transform/agent_prometheus_2
       - filter/fluentbit_0
       - metricstransform/fluentbit_1
       - resourcedetection/_global_0
       - transform/_global_0
+      - metricstarttime/fluentbit_0
       receivers:
       - prometheus/agent_prometheus
     metrics/loggingmetrics:
       exporters:
-      - googlecloud
+      - otlphttp/otlp
       processors:
       - transform/agent_prometheus_0
+      - resource/agent_prometheus_1
+      - transform/agent_prometheus_2
       - transform/loggingmetrics_0
       - filter/loggingmetrics_1
       - filter/loggingmetrics_2
@@ -808,28 +847,35 @@ service:
       - metricstransform/loggingmetrics_8
       - resourcedetection/_global_0
       - transform/_global_0
+      - metricstarttime/loggingmetrics_0
       receivers:
       - prometheus/agent_prometheus
     metrics/opsagent:
       exporters:
-      - googlecloud
+      - otlphttp/otlp
       processors:
       - transform/ops_agent_0
+      - resource/ops_agent_1
+      - transform/ops_agent_2
       - resourcedetection/_global_0
       - transform/_global_0
+      - metricstarttime/opsagent_0
       receivers:
       - otlpjsonfile/ops_agent
     metrics/otel:
       exporters:
-      - googlecloud
+      - otlphttp/otlp
       processors:
       - transform/agent_prometheus_0
+      - resource/agent_prometheus_1
+      - transform/agent_prometheus_2
       - transform/otel_0
       - filter/otel_1
       - filter/otel_2
       - metricstransform/otel_3
       - resourcedetection/_global_0
       - transform/_global_0
+      - metricstarttime/otel_0
       receivers:
       - prometheus/agent_prometheus
     metrics/prometheus__pipeline_prometheus:
@@ -839,7 +885,6 @@ service:
       - transform/prometheus_0
       - resource/prometheus_1
       - transform/prometheus_2
-      - metricstarttime/prometheus_3
       receivers:
       - prometheus/prometheus
   telemetry: