Adding otlphttp exporter to otlp receiver

apps/otlp.go

@@ -137,7 +137,11 @@ func (r ReceiverOTLP) Pipelines(ctx context.Context) ([]otel.ReceiverPipeline, e
 
 	receiverPipelineType, metricsRDM, metricsProcessors := r.metricsProcessors(ctx)
 
-	return []otel.ReceiverPipeline{{
+	converter := confgenerator.ConvertGCMOtelExporterToOtlpExporter
+	if r.MetricsMode != "googlecloudmonitoring" {
+		converter = confgenerator.ConvertOtlpPrometheusExporterToOtlpExporter
+	}
+	return []otel.ReceiverPipeline{converter(otel.ReceiverPipeline{
 		ExporterTypes: map[string]otel.ExporterType{
 			"metrics": receiverPipelineType,
 			"traces":  otel.OTel,
@@ -163,7 +167,7 @@ func (r ReceiverOTLP) Pipelines(ctx context.Context) ([]otel.ReceiverPipeline, e
 			"traces":  otel.SetIfMissing,
 			"logs":    otel.SetIfMissing,
 		},
-	}}, nil
+	}, ctx)}, nil
 }
 
 func init() {

confgenerator/confgenerator.go

@@ -57,15 +57,19 @@ func googleCloudExporter(userAgent string, instrumentationLabels bool, serviceRe
 	}
 }
 
+func ConvertOtlpPrometheusExporterToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline {
+	return ConvertToOtlpExporter(receiver, ctx, true, true)
+}
+
 func ConvertPrometheusExporterToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline {
-	return ConvertToOtlpExporter(receiver, ctx, true)
+	return ConvertToOtlpExporter(receiver, ctx, true, false)
 }
 
 func ConvertGCMOtelExporterToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context) otel.ReceiverPipeline {
-	return ConvertToOtlpExporter(receiver, ctx, false)
+	return ConvertToOtlpExporter(receiver, ctx, false, false)
 }
 
-func ConvertToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context, isPrometheus bool) otel.ReceiverPipeline {
+func ConvertToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context, isPrometheus, isOtlp bool) otel.ReceiverPipeline {
 	expOtlpExporter := experimentsFromContext(ctx)["otlp_exporter"]
 	resource, _ := platform.FromContext(ctx).GetResource()
 	if !expOtlpExporter {
@@ -81,6 +85,11 @@ func ConvertToOtlpExporter(receiver otel.ReceiverPipeline, ctx context.Context,
 	if isPrometheus {
 		receiver.Processors["metrics"] = append(receiver.Processors["metrics"], otel.MetricUnknownCounter())
 		receiver.Processors["metrics"] = append(receiver.Processors["metrics"], otel.MetricStartTime())
+		// if isOtlp {
+		// 	receiver.Processors["metrics"] = append(receiver.Processors["metrics"],
+		// 		otel.MetricsTransform(
+		// 			otel.AddPrefix("prometheus.googleapis.com")))
+		// }
 	}
 	return receiver
 }

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/EXPERIMENTAL_FEATURES

@@ -0,0 +1 @@
+otlp_exporter

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/0f15dbe303dc7122d43443c9a4c31632.lua

@@ -0,0 +1,28 @@
+
+function process(tag, timestamp, record)
+local v = "ops-agent";
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentKind"] = value
+end)(v)
+local v = "latest";
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["agent.googleapis.com/health/agentVersion"] = value
+end)(v)
+local v = "v1";
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["agent.googleapis.com/health/schemaVersion"] = value
+end)(v)
+return 2, timestamp, record
+end

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/4d6012ff003886818fb9b9285b4af962.lua

@@ -0,0 +1,19 @@
+
+function process(tag, timestamp, record)
+local __field_0 = (function()
+return record["severity"]
+end)();
+(function(value)
+record["severity"] = value
+end)(nil);
+local v = __field_0;
+if v == "debug" then v = "DEBUG"
+elseif v == "error" then v = "ERROR"
+elseif v == "info" then v = "INFO"
+elseif v == "warn" then v = "WARNING"
+end
+(function(value)
+record["logging.googleapis.com/severity"] = value
+end)(v)
+return 2, timestamp, record
+end

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/5fc5f42c16c9e1ab8292e3d42f74f3be.lua

@@ -0,0 +1,49 @@
+
+  function shallow_merge(record, parsedRecord)
+    -- If no exiting record exists
+    if (record == nil) then 
+        return parsedRecord
+    end
+
+    for k, v in pairs(parsedRecord) do
+        record[k] = v
+    end
+
+    return record
+end
+
+function merge(record, parsedRecord)
+    -- If no exiting record exists
+    if record == nil then 
+        return parsedRecord
+    end
+
+    -- Potentially overwrite or merge the original records.
+    for k, v in pairs(parsedRecord) do
+        -- If there is no conflict
+        if k == "logging.googleapis.com/logName" then 
+            -- Ignore the parsed payload since the logName is controlled
+            -- by the OpsAgent.
+        elseif k == "logging.googleapis.com/labels" then 
+            -- LogEntry.labels are basically a map[string]string and so only require a
+            -- shallow merge (one level deep merge).
+            record[k] = shallow_merge(record[k], v)
+        else
+            record[k] = v
+        end
+    end
+
+    return record
+end
+
+function parser_merge_record(tag, timestamp, record)
+    originalPayload = record["logging.googleapis.com/__tmp"]
+    if originalPayload == nil then
+        return 0, timestamp, record
+    end
+
+    -- Remove original payload
+    record["logging.googleapis.com/__tmp"] = nil
+    record = merge(originalPayload, record)
+    return 2, timestamp, record
+end

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/adea349dc2d92cd07daa1d7847f5e96a.lua

@@ -0,0 +1,74 @@
+
+function process(tag, timestamp, record)
+local __field_0 = (function()
+return record["agent.googleapis.com/log_file_path"]
+end)();
+local __field_1 = (function()
+if record["logging.googleapis.com/labels"] == nil
+then
+return nil
+end
+return record["logging.googleapis.com/labels"]["compute.googleapis.com/instance_group_manager/name"]
+end)();
+local __field_2 = (function()
+if record["logging.googleapis.com/labels"] == nil
+then
+return nil
+end
+return record["logging.googleapis.com/labels"]["compute.googleapis.com/instance_group_manager/zone"]
+end)();
+local __field_3 = (function()
+if record["logging.googleapis.com/labels"] == nil
+then
+return nil
+end
+return record["logging.googleapis.com/labels"]["compute.googleapis.com/resource_name"]
+end)();
+local __field_4 = (function()
+return record["logging.googleapis.com/logName"]
+end)();
+(function(value)
+record["agent.googleapis.com/log_file_path"] = value
+end)(nil);
+local v = __field_0;
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["agent.googleapis.com/log_file_path"] = value
+end)(v)
+local v = __field_1;
+if v == nil then v = "test-mig" end;
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["compute.googleapis.com/instance_group_manager/name"] = value
+end)(v)
+local v = __field_2;
+if v == nil then v = "test-zone" end;
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["compute.googleapis.com/instance_group_manager/zone"] = value
+end)(v)
+local v = __field_3;
+if v == nil then v = "" end;
+(function(value)
+if record["logging.googleapis.com/labels"] == nil
+then
+record["logging.googleapis.com/labels"] = {}
+end
+record["logging.googleapis.com/labels"]["compute.googleapis.com/resource_name"] = value
+end)(v)
+local v = __field_4;
+if v == nil then v = "syslog" end;
+(function(value)
+record["logging.googleapis.com/logName"] = value
+end)(v)
+return 2, timestamp, record
+end

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/b4a0dead382dce7b4fe011d3f59fdb6d.lua

@@ -0,0 +1,17 @@
+
+function parser_nest(tag, timestamp, record)
+  local nestedRecord = {}
+  local parseKey = "message"
+  for k, v in pairs(record) do
+      if k ~= parseKey then
+          nestedRecord[k] = v
+      end
+  end
+
+  local result = {}
+  result[parseKey] = record[parseKey]
+  result["logging.googleapis.com/__tmp"] = nestedRecord
+
+  return 2, timestamp, result
+end
+

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/enabled_receivers_otlp.json

@@ -0,0 +1 @@
+{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/ops_agent/enabled_receivers","gauge":{"dataPoints":[{"attributes":[{"key":"telemetry_type","value":{"stringValue":"metrics"}},{"key":"receiver_type","value":{"stringValue":"hostmetrics"}}],"asInt":"1"},{"attributes":[{"key":"telemetry_type","value":{"stringValue":"metrics"}},{"key":"receiver_type","value":{"stringValue":"otlp"}}],"asInt":"1"},{"attributes":[{"key":"telemetry_type","value":{"stringValue":"logs"}},{"key":"receiver_type","value":{"stringValue":"files"}}],"asInt":"1"}]}}]}]}]}

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/feature_tracking_otlp.json

@@ -0,0 +1 @@
+{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"combined"}},{"key":"feature","value":{"stringValue":"receivers:otlp"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"combined"}},{"key":"feature","value":{"stringValue":"receivers:otlp"}},{"key":"key","value":{"stringValue":"[0].metrics_mode"}},{"key":"value","value":{"stringValue":"googlemanagedprometheus"}}],"asInt":"1"}]}}]}]}]}

confgenerator/testdata/goldens/combined-receiver_otlp_otlphttp/golden/linux-gpu/features.yaml

@@ -0,0 +1,24 @@
+- module: logging
+  feature: service:pipelines
+  key: default_pipeline_overridden
+  value: "false"
+- module: metrics
+  feature: service:pipelines
+  key: default_pipeline_overridden
+  value: "false"
+- module: global
+  feature: default:self_log
+  key: default_self_log_file_collection
+  value: "true"
+- module: logging
+  feature: service:otel_logging
+  key: otel_logging_supported_config
+  value: "true"
+- module: combined
+  feature: receivers:otlp
+  key: "[0].enabled"
+  value: "true"
+- module: combined
+  feature: receivers:otlp
+  key: "[0].metrics_mode"
+  value: googlemanagedprometheus