diff --git a/.chloggen/remove-jaeger-dbmodel-dependency.yaml b/.chloggen/remove-jaeger-dbmodel-dependency.yaml new file mode 100644 index 000000000000..28738af5dbe5 --- /dev/null +++ b/.chloggen/remove-jaeger-dbmodel-dependency.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: logzioexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Remove jaeger dbmodel dependency. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [36972] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/logzioexporter/from_domain.go b/exporter/logzioexporter/from_domain.go new file mode 100644 index 000000000000..5eee5652677d --- /dev/null +++ b/exporter/logzioexporter/from_domain.go @@ -0,0 +1,129 @@ +// Copyright The OpenTelemetry Authors +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2018 Uber Technologies, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package logzioexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter" + +import ( + "strings" + + "github.com/jaegertracing/jaeger/model" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" +) + +// newFromDomain creates fromDomain used to convert model span to db span +func newFromDomain(allTagsAsObject bool, tagKeysAsFields []string, tagDotReplacement string) fromDomain { + tags := map[string]bool{} + for _, k := range tagKeysAsFields { + tags[k] = true + } + return fromDomain{allTagsAsFields: allTagsAsObject, tagKeysAsFields: tags, tagDotReplacement: tagDotReplacement} +} + +// fromDomain is used to convert model span to db span +type fromDomain struct { + allTagsAsFields bool + tagKeysAsFields map[string]bool + tagDotReplacement string +} + +// fromDomainEmbedProcess converts model.span into json.span format. +// This format includes a ParentSpanID and an embedded process. +func (fd fromDomain) fromDomainEmbedProcess(span *model.Span) *logzioSpan { + return fd.convertSpanEmbedProcess(span) +} + +func (fd fromDomain) convertSpanInternal(span *model.Span) logzioSpan { + tags, tagsMap := fd.convertKeyValuesString(span.Tags) + return logzioSpan{ + TraceID: dbmodel.TraceID(span.TraceID.String()), + SpanID: dbmodel.SpanID(span.SpanID.String()), + Flags: uint32(span.Flags), + OperationName: span.OperationName, + StartTime: model.TimeAsEpochMicroseconds(span.StartTime), + StartTimeMillis: model.TimeAsEpochMicroseconds(span.StartTime) / 1000, + Duration: model.DurationAsMicroseconds(span.Duration), + Tags: tags, + Tag: tagsMap, + Logs: fd.convertLogs(span.Logs), + } +} + +func (fd fromDomain) convertSpanEmbedProcess(span *model.Span) *logzioSpan { + s := fd.convertSpanInternal(span) + s.Process = fd.convertProcess(span.Process) + s.References = fd.convertReferences(span) + return &s +} + +func (fd fromDomain) convertReferences(span *model.Span) []dbmodel.Reference { + out := make([]dbmodel.Reference, 0, len(span.References)) + for _, ref := range span.References { + out = append(out, dbmodel.Reference{ + RefType: fd.convertRefType(ref.RefType), + TraceID: dbmodel.TraceID(ref.TraceID.String()), + SpanID: dbmodel.SpanID(ref.SpanID.String()), + }) + } + return out +} + +func (fromDomain) convertRefType(refType model.SpanRefType) dbmodel.ReferenceType { + if refType == model.FollowsFrom { + return dbmodel.FollowsFrom + } + return dbmodel.ChildOf +} + +func (fd fromDomain) convertKeyValuesString(keyValues model.KeyValues) ([]dbmodel.KeyValue, map[string]any) { + var tagsMap map[string]any + var kvs []dbmodel.KeyValue + for _, kv := range keyValues { + if kv.GetVType() != model.BinaryType && (fd.allTagsAsFields || fd.tagKeysAsFields[kv.Key]) { + if tagsMap == nil { + tagsMap = map[string]any{} + } + tagsMap[strings.ReplaceAll(kv.Key, ".", fd.tagDotReplacement)] = kv.Value() + } else { + kvs = append(kvs, convertKeyValue(kv)) + } + } + if kvs == nil { + kvs = make([]dbmodel.KeyValue, 0) + } + return kvs, tagsMap +} + +func (fromDomain) convertLogs(logs []model.Log) []dbmodel.Log { + out := make([]dbmodel.Log, len(logs)) + for i, log := range logs { + var kvs []dbmodel.KeyValue + for _, kv := range log.Fields { + kvs = append(kvs, convertKeyValue(kv)) + } + out[i] = dbmodel.Log{ + Timestamp: model.TimeAsEpochMicroseconds(log.Timestamp), + Fields: kvs, + } + } + return out +} + +func (fd fromDomain) convertProcess(process *model.Process) dbmodel.Process { + tags, tagsMap := fd.convertKeyValuesString(process.Tags) + return dbmodel.Process{ + ServiceName: process.ServiceName, + Tags: tags, + Tag: tagsMap, + } +} + +func convertKeyValue(kv model.KeyValue) dbmodel.KeyValue { + return dbmodel.KeyValue{ + Key: kv.Key, + Type: dbmodel.ValueType(strings.ToLower(kv.VType.String())), + Value: kv.AsString(), + } +} diff --git a/exporter/logzioexporter/from_domain_test.go b/exporter/logzioexporter/from_domain_test.go new file mode 100644 index 000000000000..b345d3d961ee --- /dev/null +++ b/exporter/logzioexporter/from_domain_test.go @@ -0,0 +1,136 @@ +// Copyright The OpenTelemetry Authors +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2018 Uber Technologies, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package logzioexporter + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/gogo/protobuf/jsonpb" + "github.com/jaegertracing/jaeger/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" +) + +func TestFromDomainEmbedProcess(t *testing.T) { + domainStr, jsonStr := loadModel(t) + + var span model.Span + require.NoError(t, jsonpb.Unmarshal(bytes.NewReader(domainStr), &span)) + converter := newFromDomain(false, nil, ":") + embeddedSpan := converter.fromDomainEmbedProcess(&span) + + var expectedSpan logzioSpan + require.NoError(t, json.Unmarshal(jsonStr, &expectedSpan)) + + testJSONEncoding(t, jsonStr, embeddedSpan) +} + +// Loads and returns domain model and JSON model. +func loadModel(t *testing.T) ([]byte, []byte) { + inStr, err := os.ReadFile("./testdata/span.json") + require.NoError(t, err) + outStr, err := os.ReadFile("./testdata/logziospan.json") + require.NoError(t, err) + return inStr, outStr +} + +func testJSONEncoding(t *testing.T, expectedStr []byte, object any) { + buf := &bytes.Buffer{} + enc := json.NewEncoder(buf) + enc.SetIndent("", " ") + require.NoError(t, enc.Encode(object)) + if !assert.Equal(t, string(expectedStr), buf.String()) { + err := os.WriteFile("model-actual.json", buf.Bytes(), 0o600) + require.NoError(t, err) + } +} + +func TestEmptyTags(t *testing.T) { + tags := make([]model.KeyValue, 0) + span := model.Span{Tags: tags, Process: &model.Process{Tags: tags}} + converter := newFromDomain(false, nil, ":") + dbSpan := converter.fromDomainEmbedProcess(&span) + assert.Empty(t, dbSpan.Tags) + assert.Empty(t, dbSpan.Tag) +} + +func TestTagMap(t *testing.T) { + tags := []model.KeyValue{ + model.String("foo", "foo"), + model.Bool("a", true), + model.Int64("b.b", 1), + } + span := model.Span{Tags: tags, Process: &model.Process{Tags: tags}} + converter := newFromDomain(false, []string{"a", "b.b", "b*"}, ":") + dbSpan := converter.fromDomainEmbedProcess(&span) + + assert.Len(t, dbSpan.Tags, 1) + assert.Equal(t, "foo", dbSpan.Tags[0].Key) + assert.Len(t, dbSpan.Process.Tags, 1) + assert.Equal(t, "foo", dbSpan.Process.Tags[0].Key) + + tagsMap := map[string]any{} + tagsMap["a"] = true + tagsMap["b:b"] = int64(1) + assert.Equal(t, tagsMap, dbSpan.Tag) + assert.Equal(t, tagsMap, dbSpan.Process.Tag) +} + +func TestConvertKeyValueValue(t *testing.T) { + longString := `Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues + Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues + Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues + Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues + Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues Bender Bending Rodrigues ` + key := "key" + tests := []struct { + kv model.KeyValue + expected dbmodel.KeyValue + }{ + { + kv: model.Bool(key, true), + expected: dbmodel.KeyValue{Key: key, Value: "true", Type: "bool"}, + }, + { + kv: model.Bool(key, false), + expected: dbmodel.KeyValue{Key: key, Value: "false", Type: "bool"}, + }, + { + kv: model.Int64(key, int64(1499)), + expected: dbmodel.KeyValue{Key: key, Value: "1499", Type: "int64"}, + }, + { + kv: model.Float64(key, float64(15.66)), + expected: dbmodel.KeyValue{Key: key, Value: "15.66", Type: "float64"}, + }, + { + kv: model.String(key, longString), + expected: dbmodel.KeyValue{Key: key, Value: longString, Type: "string"}, + }, + { + kv: model.Binary(key, []byte(longString)), + expected: dbmodel.KeyValue{Key: key, Value: hex.EncodeToString([]byte(longString)), Type: "binary"}, + }, + { + kv: model.KeyValue{VType: 1500, Key: key}, + expected: dbmodel.KeyValue{Key: key, Value: "unknown type 1500", Type: "1500"}, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("%s:%s", test.expected.Type, test.expected.Key), func(t *testing.T) { + actual := convertKeyValue(test.kv) + assert.Equal(t, test.expected, actual) + }) + } +} diff --git a/exporter/logzioexporter/go.mod b/exporter/logzioexporter/go.mod index e5830c48ce41..7476427aa019 100644 --- a/exporter/logzioexporter/go.mod +++ b/exporter/logzioexporter/go.mod @@ -5,6 +5,7 @@ go 1.22.7 toolchain go1.22.8 require ( + github.com/gogo/protobuf v1.3.2 github.com/hashicorp/go-hclog v1.6.3 github.com/jaegertracing/jaeger v1.64.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.116.0 @@ -38,7 +39,6 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-version v1.7.0 // indirect diff --git a/exporter/logzioexporter/internal/dbmodel/model.go b/exporter/logzioexporter/internal/dbmodel/model.go new file mode 100644 index 000000000000..af42f10eecb9 --- /dev/null +++ b/exporter/logzioexporter/internal/dbmodel/model.go @@ -0,0 +1,70 @@ +// Copyright The OpenTelemetry Authors +// Copyright (c) 2019 The Jaeger Authors. +// Copyright (c) 2018 Uber Technologies, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package dbmodel // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" + +// ReferenceType is the reference type of one span to another +type ReferenceType string + +// TraceID is the shared trace ID of all spans in the trace. +type TraceID string + +// SpanID is the id of a span +type SpanID string + +// ValueType is the type of a value stored in KeyValue struct. +type ValueType string + +const ( + // ChildOf means a span is the child of another span + ChildOf ReferenceType = "CHILD_OF" + // FollowsFrom means a span follows from another span + FollowsFrom ReferenceType = "FOLLOWS_FROM" + + // StringType indicates a string value stored in KeyValue + StringType ValueType = "string" + // BoolType indicates a Boolean value stored in KeyValue + BoolType ValueType = "bool" + // Int64Type indicates a 64bit signed integer value stored in KeyValue + Int64Type ValueType = "int64" + // Float64Type indicates a 64bit float value stored in KeyValue + Float64Type ValueType = "float64" + // BinaryType indicates an arbitrary byte array stored in KeyValue + BinaryType ValueType = "binary" +) + +// Reference is a reference from one span to another +type Reference struct { + RefType ReferenceType `json:"refType"` + TraceID TraceID `json:"traceID"` + SpanID SpanID `json:"spanID"` +} + +// Process is the process emitting a set of spans +type Process struct { + ServiceName string `json:"serviceName"` + Tags []KeyValue `json:"tags"` + // Alternative representation of tags for better kibana support + Tag map[string]any `json:"tag,omitempty"` +} + +// Log is a log emitted in a span +type Log struct { + Timestamp uint64 `json:"timestamp"` + Fields []KeyValue `json:"fields"` +} + +// KeyValue is a key-value pair with typed value. +type KeyValue struct { + Key string `json:"key"` + Type ValueType `json:"type,omitempty"` + Value any `json:"value"` +} + +// Service is the JSON struct for service:operation documents in ElasticSearch +type Service struct { + ServiceName string `json:"serviceName"` + OperationName string `json:"operationName"` +} diff --git a/exporter/logzioexporter/logziospan.go b/exporter/logzioexporter/logziospan.go index 7bc485fd1409..edeef69b0ea9 100644 --- a/exporter/logzioexporter/logziospan.go +++ b/exporter/logzioexporter/logziospan.go @@ -7,7 +7,8 @@ import ( "encoding/json" "github.com/jaegertracing/jaeger/model" - "github.com/jaegertracing/jaeger/plugin/storage/es/spanstore/dbmodel" + + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/logzioexporter/internal/dbmodel" ) const ( @@ -45,8 +46,8 @@ func getTagsValues(tags []model.KeyValue) []string { // transformToLogzioSpanBytes receives a Jaeger span, converts it to logzio span and returns it as a byte array. // The main differences between Jaeger span and logzio span are arrays which are represented as maps func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { - spanConverter := dbmodel.NewFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) - jsonSpan := spanConverter.FromDomainEmbedProcess(span) + spanConverter := newFromDomain(true, getTagsValues(span.Tags), tagDotReplacementCharacter) + jsonSpan := spanConverter.fromDomainEmbedProcess(span) newSpan := logzioSpan{ TraceID: jsonSpan.TraceID, OperationName: jsonSpan.OperationName, @@ -65,21 +66,3 @@ func transformToLogzioSpanBytes(span *model.Span) ([]byte, error) { } return json.Marshal(newSpan) } - -// transformToDbModelSpan coverts logz.io span to ElasticSearch span -func (span *logzioSpan) transformToDbModelSpan() *dbmodel.Span { - return &dbmodel.Span{ - OperationName: span.OperationName, - Process: span.Process, - Tags: span.Tags, - Tag: span.Tag, - References: span.References, - Logs: span.Logs, - Duration: span.Duration, - StartTimeMillis: span.StartTimeMillis, - StartTime: span.StartTime, - Flags: span.Flags, - SpanID: span.SpanID, - TraceID: span.TraceID, - } -} diff --git a/exporter/logzioexporter/logziospan_test.go b/exporter/logzioexporter/logziospan_test.go index faea671f4b40..af4b768e1b2a 100644 --- a/exporter/logzioexporter/logziospan_test.go +++ b/exporter/logzioexporter/logziospan_test.go @@ -31,20 +31,3 @@ func TestTransformToLogzioSpanBytes(tester *testing.T) { tester.Error("error converting span to logzioSpan, JaegerTag is not found") } } - -func TestTransformToDbModelSpan(tester *testing.T) { - inStr, err := os.ReadFile("./testdata/span.json") - require.NoError(tester, err, "error opening sample span file") - var span model.Span - err = json.Unmarshal(inStr, &span) - if err != nil { - fmt.Println("json.Unmarshal") - } - newSpan, err := transformToLogzioSpanBytes(&span) - require.NoError(tester, err) - var testLogzioSpan logzioSpan - err = json.Unmarshal(newSpan, &testLogzioSpan) - require.NoError(tester, err) - dbModelSpan := testLogzioSpan.transformToDbModelSpan() - require.Len(tester, dbModelSpan.References, 3, "Error converting logzio span to dbmodel span") -} diff --git a/exporter/logzioexporter/testdata/logziospan.json b/exporter/logzioexporter/testdata/logziospan.json new file mode 100644 index 000000000000..a13250d2ce9b --- /dev/null +++ b/exporter/logzioexporter/testdata/logziospan.json @@ -0,0 +1,92 @@ +{ + "traceID": "0000000000000001", + "operationName": "test-general-conversion", + "spanID": "0000000000000002", + "references": [ + { + "refType": "CHILD_OF", + "traceID": "0000000000000001", + "spanID": "0000000000000003" + }, + { + "refType": "FOLLOWS_FROM", + "traceID": "0000000000000001", + "spanID": "0000000000000004" + }, + { + "refType": "CHILD_OF", + "traceID": "00000000000000ff", + "spanID": "00000000000000ff" + } + ], + "flags": 1, + "startTime": 1485467191639875, + "startTimeMillis": 1485467191639, + "@timestamp": 0, + "duration": 5, + "JaegerTags": [ + { + "key": "peer.service", + "type": "string", + "value": "service-y" + }, + { + "key": "peer.ipv4", + "type": "int64", + "value": "23456" + }, + { + "key": "error", + "type": "bool", + "value": "true" + }, + { + "key": "temperature", + "type": "float64", + "value": "72.5" + }, + { + "key": "blob", + "type": "binary", + "value": "00003039" + } + ], + "logs": [ + { + "timestamp": 1485467191639875, + "fields": [ + { + "key": "event", + "type": "int64", + "value": "123415" + } + ] + }, + { + "timestamp": 1485467191639875, + "fields": [ + { + "key": "x", + "type": "string", + "value": "y" + } + ] + } + ], + "process": { + "serviceName": "service-x", + "tags": [ + { + "key": "peer.ipv4", + "type": "int64", + "value": "23456" + }, + { + "key": "error", + "type": "bool", + "value": "true" + } + ] + }, + "type": "" +}