Skip to content

Commit 0e06aa1

Browse files
codefromthecryptcodefromthecrypt
committed
access_log: add OTLP/HTTP exporter to OpenTelemetry access logger
Adds HTTP transport support to the OpenTelemetry access logger, complementing OTLP/HTTP tracing. This enables direct OTLP log export to backends that only accept HTTP (Dynatrace, Datadog, Logfire), without requiring an intermediate collector sidecar. Also refactors the config to match the tracer pattern: - Add top-level http_service and grpc_service fields - Add top-level log_name, buffer_flush_interval, buffer_size_bytes, filter_state_objects_to_log, and custom_tags fields - Add top-level resource_detectors field for dynamic resource attributes - Deprecate common_config field (still functional for backward compat) - Extract shared utilities (otlp_log_utils) from duplicated code Risk Level: Low (new optional feature, backward compatible) Testing: Unit tests, integration test, manual testing with OTel collector Fixes #26352 (access logs portion) Signed-off-by: Adrian Cole <[email protected]>
1 parent b34c60c commit 0e06aa1

File tree

21 files changed

+1854
-74
lines changed

21 files changed

+1854
-74
lines changed

‎api/envoy/extensions/access_loggers/open_telemetry/v3/BUILD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@ licenses(["notice"]) # Apache 2
66

77
api_proto_package(
88
deps = [
9+
"//envoy/annotations:pkg",
910
"//envoy/config/core/v3:pkg",
1011
"//envoy/extensions/access_loggers/grpc/v3:pkg",
12+
"//envoy/type/tracing/v3:pkg",
1113
"@com_github_cncf_xds//udpa/annotations:pkg",
1214
"@opentelemetry_proto//:common_proto",
1315
],

‎api/envoy/extensions/access_loggers/open_telemetry/v3/logs_service.proto‎

Lines changed: 67 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,30 +3,64 @@ syntax = "proto3";
33
package envoy.extensions.access_loggers.open_telemetry.v3;
44

55
import "envoy/config/core/v3/extension.proto";
6+
import "envoy/config/core/v3/grpc_service.proto";
7+
import "envoy/config/core/v3/http_service.proto";
68
import "envoy/extensions/access_loggers/grpc/v3/als.proto";
9+
import "envoy/type/tracing/v3/custom_tag.proto";
10+
11+
import "google/protobuf/duration.proto";
12+
import "google/protobuf/wrappers.proto";
713

814
import "opentelemetry/proto/common/v1/common.proto";
915

16+
import "envoy/annotations/deprecation.proto";
17+
import "udpa/annotations/migrate.proto";
1018
import "udpa/annotations/status.proto";
11-
import "validate/validate.proto";
1219

1320
option java_package = "io.envoyproxy.envoy.extensions.access_loggers.open_telemetry.v3";
1421
option java_outer_classname = "LogsServiceProto";
1522
option java_multiple_files = true;
1623
option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/open_telemetry/v3;open_telemetryv3";
1724
option (udpa.annotations.file_status).package_version_status = ACTIVE;
1825

19-
// [#protodoc-title: OpenTelemetry (gRPC) Access Log]
26+
// [#protodoc-title: OpenTelemetry Access Log]
2027

2128
// Configuration for the built-in ``envoy.access_loggers.open_telemetry``
2229
// :ref:`AccessLog <envoy_v3_api_msg_config.accesslog.v3.AccessLog>`. This configuration will
2330
// populate `opentelemetry.proto.collector.v1.logs.ExportLogsServiceRequest.resource_logs <https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/collector/logs/v1/logs_service.proto>`_.
2431
// In addition, the request start time is set in the dedicated field.
2532
// [#extension: envoy.access_loggers.open_telemetry]
26-
// [#next-free-field: 8]
33+
// [#next-free-field: 16]
2734
message OpenTelemetryAccessLogConfig {
2835
// [#comment:TODO(itamarkam): add 'filter_state_objects_to_log' to logs.]
29-
grpc.v3.CommonGrpcAccessLogConfig common_config = 1 [(validate.rules).message = {required: true}];
36+
// This field is deprecated. Use ``grpc_service`` and other top-level fields instead.
37+
// This field is kept for backward compatibility with existing gRPC configurations.
38+
// Note: Only one of ``common_config``, ``grpc_service``, or ``http_service`` may be used.
39+
grpc.v3.CommonGrpcAccessLogConfig common_config = 1 [
40+
deprecated = true,
41+
(envoy.annotations.deprecated_at_minor_version) = "3.0",
42+
(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"
43+
];
44+
45+
// The upstream HTTP cluster that will receive OTLP logs via
46+
// `OTLP/HTTP <https://opentelemetry.io/docs/specs/otlp/#otlphttp>`_.
47+
// Note: Only one of ``common_config``, ``grpc_service``, or ``http_service`` may be used.
48+
//
49+
// .. note::
50+
//
51+
// The ``request_headers_to_add`` property in the OTLP HTTP exporter service
52+
// does not support the :ref:`format specifier <config_access_log_format>` as used for
53+
// :ref:`HTTP access logging <config_access_log>`.
54+
// The values configured are added as HTTP headers on the OTLP export request
55+
// without any formatting applied.
56+
config.core.v3.HttpService http_service = 8
57+
[(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"];
58+
59+
// The upstream gRPC cluster that will receive OTLP logs.
60+
// Note: Only one of ``common_config``, ``grpc_service``, or ``http_service`` may be used.
61+
// This field is preferred over ``common_config.grpc_service``.
62+
config.core.v3.GrpcService grpc_service = 9
63+
[(udpa.annotations.field_migrate).oneof_promotion = "otlp_exporter"];
3064

3165
// If specified, Envoy will not generate built-in resource labels
3266
// like ``log_name``, ``zone_name``, ``cluster_name``, ``node_name``.
@@ -57,4 +91,33 @@ message OpenTelemetryAccessLogConfig {
5791
// See the formatters extensions documentation for details.
5892
// [#extension-category: envoy.formatter]
5993
repeated config.core.v3.TypedExtensionConfig formatters = 7;
94+
95+
// Friendly identifier for the access log, used as a resource attribute.
96+
// This field is preferred over ``common_config.log_name``.
97+
string log_name = 10;
98+
99+
// The interval for flushing access logs to the transport. Default: 1 second.
100+
// This field is preferred over ``common_config.buffer_flush_interval``.
101+
google.protobuf.Duration buffer_flush_interval = 11;
102+
103+
// Soft size limit in bytes for the access log buffer. When the buffer exceeds
104+
// this limit, logs will be flushed. Default: 16KB.
105+
// This field is preferred over ``common_config.buffer_size_bytes``.
106+
google.protobuf.UInt32Value buffer_size_bytes = 12;
107+
108+
// Additional filter state objects to log.
109+
// This field is preferred over ``common_config.filter_state_objects_to_log``.
110+
repeated string filter_state_objects_to_log = 13;
111+
112+
// Optional custom tags to include in the log record.
113+
// This field is preferred over ``common_config.custom_tags``.
114+
repeated type.tracing.v3.CustomTag custom_tags = 14;
115+
116+
// An ordered list of resource detectors for populating resource attributes.
117+
// This enables using the ``OTEL_RESOURCE_ATTRIBUTES`` environment variable for
118+
// ``service.name`` and other resource attributes, matching the tracer and
119+
// metrics sink patterns. When configured, the detected resources are merged
120+
// with any static ``resource_attributes``, with detector values taking precedence.
121+
// [#extension-category: envoy.tracers.opentelemetry.resource_detectors]
122+
repeated config.core.v3.TypedExtensionConfig resource_detectors = 15;
60123
}

‎changelogs/current.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,5 +458,27 @@ new_features:
458458
- area: attributes
459459
change: |
460460
added :ref:`attributes <arch_overview_attributes>` for looking up request or response headers bytes.
461+
- area: access_log
462+
change: |
463+
Added support for exporting OpenTelemetry access logs via HTTP and refactored the configuration
464+
to match the tracer config pattern. New top-level fields ``http_service``, ``grpc_service``,
465+
``log_name``, ``buffer_flush_interval``, ``buffer_size_bytes``, ``filter_state_objects_to_log``,
466+
and ``custom_tags`` provide a cleaner, transport-agnostic configuration.
467+
The ``common_config`` field is deprecated but remains functional for backward compatibility.
468+
See :ref:`http_service <envoy_v3_api_field_extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig.http_service>`
469+
and :ref:`grpc_service <envoy_v3_api_field_extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig.grpc_service>`.
470+
- area: access_log
471+
change: |
472+
Added ``resource_detectors`` to the OpenTelemetry access logger configuration. This enables
473+
consistent resource attribute configuration across all signals (tracing, metrics, access logs)
474+
using the ``OTEL_RESOURCE_ATTRIBUTES`` environment variable via the environment resource detector.
475+
See :ref:`resource_detectors
476+
<envoy_v3_api_field_extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig.resource_detectors>`.
461477
462478
deprecated:
479+
- area: access_log
480+
change: |
481+
The ``common_config`` field in
482+
:ref:`OpenTelemetryAccessLogConfig <envoy_v3_api_msg_extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig>`
483+
is deprecated. Use the new top-level fields ``grpc_service``, ``log_name``, ``buffer_flush_interval``,
484+
``buffer_size_bytes``, ``filter_state_objects_to_log``, and ``custom_tags`` instead.

‎configs/envoy-otel-http.yaml‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# Example configuration for exporting OpenTelemetry logs and traces via HTTP.
2+
# This demonstrates OTLP/HTTP transport for both access logs and tracing.
3+
#
4+
# Usage:
5+
# 1. Start an OTLP HTTP collector on localhost:4318 (e.g., otel-tui)
6+
# 2. Run: OTEL_RESOURCE_ATTRIBUTES=service.name=envoy-example bazel-bin/source/exe/envoy-static -c configs/envoy-otel-http.yaml
7+
# 3. Test: curl localhost:10080/get
8+
9+
admin:
10+
address:
11+
socket_address:
12+
protocol: TCP
13+
address: 127.0.0.1
14+
port_value: 9901
15+
16+
static_resources:
17+
listeners:
18+
- name: listener_0
19+
address:
20+
socket_address:
21+
protocol: TCP
22+
address: 0.0.0.0
23+
port_value: 10080
24+
filter_chains:
25+
- filters:
26+
- name: envoy.filters.network.http_connection_manager
27+
typed_config:
28+
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
29+
stat_prefix: ingress_http
30+
generate_request_id: true
31+
tracing:
32+
random_sampling:
33+
value: 100.0
34+
provider:
35+
name: envoy.tracers.opentelemetry
36+
typed_config:
37+
"@type": type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig
38+
http_service:
39+
http_uri:
40+
uri: "http://localhost:4318/v1/traces"
41+
cluster: otel_collector
42+
timeout: 1s
43+
request_headers_to_add:
44+
- header:
45+
key: "Authorization"
46+
value: "Bearer fake"
47+
sampler:
48+
name: envoy.tracers.opentelemetry.samplers.always_on
49+
typed_config:
50+
"@type": type.googleapis.com/envoy.extensions.tracers.opentelemetry.samplers.v3.AlwaysOnSamplerConfig
51+
access_log:
52+
- name: envoy.access_loggers.open_telemetry
53+
typed_config:
54+
"@type": type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig
55+
log_name: envoy_access_log
56+
http_service:
57+
http_uri:
58+
uri: "http://localhost:4318/v1/logs"
59+
cluster: otel_collector
60+
timeout: 1s
61+
request_headers_to_add:
62+
- header:
63+
key: "Authorization"
64+
value: "Bearer fake"
65+
resource_detectors:
66+
- name: envoy.tracers.opentelemetry.resource_detectors.environment
67+
typed_config:
68+
# yamllint disable-line rule:line-length
69+
"@type": type.googleapis.com/envoy.extensions.tracers.opentelemetry.resource_detectors.v3.EnvironmentResourceDetectorConfig
70+
body:
71+
string_value: "%REQ(:METHOD)% %REQ(:PATH)% %PROTOCOL% %RESPONSE_CODE%"
72+
attributes:
73+
values:
74+
- key: "response_code_details"
75+
value:
76+
string_value: "%RESPONSE_CODE_DETAILS%"
77+
route_config:
78+
name: local_route
79+
virtual_hosts:
80+
- name: backend
81+
domains: ["*"]
82+
routes:
83+
- match:
84+
prefix: "/"
85+
route:
86+
cluster: httpbin
87+
http_filters:
88+
- name: envoy.filters.http.router
89+
typed_config:
90+
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
91+
clusters:
92+
- name: otel_collector
93+
type: STRICT_DNS
94+
lb_policy: ROUND_ROBIN
95+
load_assignment:
96+
cluster_name: otel_collector
97+
endpoints:
98+
- lb_endpoints:
99+
- endpoint:
100+
address:
101+
socket_address:
102+
address: localhost
103+
port_value: 4318
104+
- name: httpbin
105+
type: LOGICAL_DNS
106+
dns_lookup_family: V4_ONLY
107+
lb_policy: ROUND_ROBIN
108+
load_assignment:
109+
cluster_name: httpbin
110+
endpoints:
111+
- lb_endpoints:
112+
- endpoint:
113+
address:
114+
socket_address:
115+
address: httpbin.org
116+
port_value: 80

‎source/extensions/access_loggers/open_telemetry/BUILD‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,28 @@ licenses(["notice"]) # Apache 2
99

1010
envoy_extension_package()
1111

12+
envoy_cc_library(
13+
name = "otlp_log_utils_lib",
14+
srcs = ["otlp_log_utils.cc"],
15+
hdrs = ["otlp_log_utils.h"],
16+
deps = [
17+
"//source/common/common:assert_lib",
18+
"//source/common/common:hex_lib",
19+
"//source/common/common:macros",
20+
"//source/common/version:version_lib",
21+
"@com_google_absl//absl/strings",
22+
"@envoy_api//envoy/extensions/access_loggers/open_telemetry/v3:pkg_cc_proto",
23+
"@opentelemetry_proto//:common_proto_cc",
24+
"@opentelemetry_proto//:logs_proto_cc",
25+
],
26+
)
27+
1228
envoy_cc_library(
1329
name = "grpc_access_log_lib",
1430
srcs = ["grpc_access_log_impl.cc"],
1531
hdrs = ["grpc_access_log_impl.h"],
1632
deps = [
33+
":otlp_log_utils_lib",
1734
"//envoy/event:dispatcher_interface",
1835
"//envoy/grpc:async_client_manager_interface",
1936
"//envoy/local_info:local_info_interface",
@@ -23,6 +40,7 @@ envoy_cc_library(
2340
"//source/common/protobuf",
2441
"//source/extensions/access_loggers/common:grpc_access_logger",
2542
"//source/extensions/access_loggers/common:grpc_access_logger_clients_lib",
43+
"//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib",
2644
"@envoy_api//envoy/extensions/access_loggers/grpc/v3:pkg_cc_proto",
2745
"@envoy_api//envoy/extensions/access_loggers/open_telemetry/v3:pkg_cc_proto",
2846
"@opentelemetry_proto//:logs_proto_cc",
@@ -36,11 +54,13 @@ envoy_cc_library(
3654
hdrs = ["access_log_impl.h"],
3755
deps = [
3856
":grpc_access_log_lib",
57+
":otlp_log_utils_lib",
3958
":substitution_formatter_lib",
4059
"//envoy/access_log:access_log_interface",
4160
"//envoy/protobuf:message_validator_interface",
4261
"//source/common/protobuf:utility_lib",
4362
"//source/extensions/access_loggers/common:access_log_base",
63+
"//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib",
4464
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
4565
"@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto",
4666
"@envoy_api//envoy/extensions/access_loggers/grpc/v3:pkg_cc_proto",
@@ -50,6 +70,37 @@ envoy_cc_library(
5070
],
5171
)
5272

73+
envoy_cc_library(
74+
name = "http_access_log_lib",
75+
srcs = ["http_access_log_impl.cc"],
76+
hdrs = ["http_access_log_impl.h"],
77+
deps = [
78+
":otlp_log_utils_lib",
79+
":substitution_formatter_lib",
80+
"//envoy/access_log:access_log_interface",
81+
"//envoy/event:dispatcher_interface",
82+
"//envoy/local_info:local_info_interface",
83+
"//envoy/singleton:instance_interface",
84+
"//envoy/thread_local:thread_local_interface",
85+
"//envoy/upstream:cluster_manager_interface",
86+
"//source/common/config:utility_lib",
87+
"//source/common/http:async_client_lib",
88+
"//source/common/http:async_client_utility_lib",
89+
"//source/common/http:header_map_lib",
90+
"//source/common/http:message_lib",
91+
"//source/common/http:utility_lib",
92+
"//source/common/protobuf",
93+
"//source/common/protobuf:utility_lib",
94+
"//source/extensions/access_loggers/common:access_log_base",
95+
"//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib",
96+
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
97+
"@envoy_api//envoy/data/accesslog/v3:pkg_cc_proto",
98+
"@envoy_api//envoy/extensions/access_loggers/open_telemetry/v3:pkg_cc_proto",
99+
"@opentelemetry_proto//:logs_proto_cc",
100+
"@opentelemetry_proto//:logs_service_proto_cc",
101+
],
102+
)
103+
53104
envoy_cc_library(
54105
name = "access_log_proto_descriptors_lib",
55106
srcs = ["access_log_proto_descriptors.cc"],
@@ -72,6 +123,8 @@ envoy_cc_extension(
72123
"//source/extensions/access_loggers/open_telemetry:access_log_lib",
73124
"//source/extensions/access_loggers/open_telemetry:access_log_proto_descriptors_lib",
74125
"//source/extensions/access_loggers/open_telemetry:grpc_access_log_lib",
126+
"//source/extensions/access_loggers/open_telemetry:http_access_log_lib",
127+
"//source/extensions/tracers/opentelemetry/resource_detectors:resource_detector_lib",
75128
"@envoy_api//envoy/extensions/access_loggers/open_telemetry/v3:pkg_cc_proto",
76129
],
77130
)

0 commit comments

Comments
 (0)