diff --git a/internal/storage/v2/elasticsearch/tracestore/fixtures/.gitignore b/internal/storage/v2/elasticsearch/tracestore/fixtures/.gitignore new file mode 100644 index 00000000000..d89b8bfb9eb --- /dev/null +++ b/internal/storage/v2/elasticsearch/tracestore/fixtures/.gitignore @@ -0,0 +1 @@ +actual_* \ No newline at end of file diff --git a/internal/storage/v2/elasticsearch/tracestore/fixtures/es_01.json b/internal/storage/v2/elasticsearch/tracestore/fixtures/es_01.json new file mode 100644 index 00000000000..c599adbb20e --- /dev/null +++ b/internal/storage/v2/elasticsearch/tracestore/fixtures/es_01.json @@ -0,0 +1,100 @@ +{ + "traceID": "00000000000000010000000000000000", + "spanID": "0000000000000002", + "flags": 1, + "operationName": "test-general-conversion", + "references": [ + { + "refType": "CHILD_OF", + "traceID": "00000000000000010000000000000000", + "spanID": "0000000000000003" + }, + { + "refType": "FOLLOWS_FROM", + "traceID": "00000000000000010000000000000000", + "spanID": "0000000000000004" + }, + { + "refType": "CHILD_OF", + "traceID": "00000000000000ff0000000000000000", + "spanID": "00000000000000ff" + } + ], + "startTime": 1485467191639875, + "startTimeMillis": 1485467191639, + "duration": 5, + "tags": [ + { + "key": "otel.scope.name", + "type": "string", + "value": "testing-library" + }, + { + "key": "otel.scope.version", + "type": "string", + "value": "1.1.1" + }, + { + "key": "peer.service", + "type": "string", + "value": "service-y" + }, + { + "key": "peer.ipv4", + "type": "int64", + "value": "23456" + }, + { + "key": "blob", + "type": "binary", + "value": "00003039" + }, + { + "key": "temperature", + "type": "float64", + "value": "72.5" + }, + { + "key": "error", + "type": "bool", + "value": "true" + } + ], + "logs": [ + { + "timestamp": 1485467191639875, + "fields": [ + { + "key": "event", + "type": "string", + "value": "testing-event" + }, + { + "key": "event-x", + "type": "string", + "value": "event-y" + } + ] + }, + { + "timestamp": 1485467191639875, + "fields": [ + { + "key": "x", + "type": "string", + "value": "y" + } + ] + } + ], + "process": { + "serviceName": "service-x", + "tags": [ + { + "key": "sdk.version", + "type": "string", + "value": "1.2.1" + } + ] + } +} diff --git a/internal/storage/v2/elasticsearch/tracestore/fixtures/otel_traces_01.json b/internal/storage/v2/elasticsearch/tracestore/fixtures/otel_traces_01.json new file mode 100644 index 00000000000..eebe7e80c3b --- /dev/null +++ b/internal/storage/v2/elasticsearch/tracestore/fixtures/otel_traces_01.json @@ -0,0 +1,121 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "service-x" + } + }, + { + "key": "sdk.version", + "value": { + "stringValue": "1.2.1" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "testing-library", + "version": "1.1.1" + }, + "spans": [ + { + "traceId": "00000000000000010000000000000000", + "spanId": "0000000000000002", + "parentSpanId": "0000000000000003", + "flags": 1, + "name": "test-general-conversion", + "startTimeUnixNano": "1485467191639875000", + "endTimeUnixNano": "1485467191639880000", + "attributes": [ + { + "key": "peer.service", + "value": { + "stringValue": "service-y" + } + }, + { + "key": "peer.ipv4", + "value": { + "intValue": "23456" + } + }, + { + "key": "blob", + "value": { + "bytesValue": "AAAwOQ==" + } + }, + { + "key": "temperature", + "value": { + "doubleValue": 72.5 + } + } + ], + "events": [ + { + "timeUnixNano": "1485467191639875000", + "name": "testing-event", + "attributes": [ + { + "key": "event-x", + "value": { + "stringValue": "event-y" + } + } + ] + }, + { + "timeUnixNano": "1485467191639875000", + "attributes": [ + { + "key": "x", + "value": { + "stringValue": "y" + } + } + ] + } + ], + "links": [ + { + "traceId": "00000000000000010000000000000000", + "spanId": "0000000000000004", + "attributes": [ + { + "key": "opentracing.ref_type", + "value": { + "stringValue": "follows_from" + } + } + ] + }, + { + "traceId": "00000000000000ff0000000000000000", + "spanId": "00000000000000ff", + "attributes": [ + { + "key": "opentracing.ref_type", + "value": { + "stringValue": "child_of" + } + } + ] + } + ], + "status": { + "code": 2 + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/internal/storage/v2/elasticsearch/tracestore/from_dbmodel.go b/internal/storage/v2/elasticsearch/tracestore/from_dbmodel.go index 0b093af58f1..2119b61d2aa 100644 --- a/internal/storage/v2/elasticsearch/tracestore/from_dbmodel.go +++ b/internal/storage/v2/elasticsearch/tracestore/from_dbmodel.go @@ -92,6 +92,7 @@ func dbSpanToSpan(dbSpan *dbmodel.Span, span ptrace.Span) error { span.SetTraceID(traceId) span.SetSpanID(spanId) span.SetName(dbSpan.OperationName) + span.SetFlags(dbSpan.Flags) startTime := model.EpochMicrosecondsAsTime(dbSpan.StartTime) span.SetStartTimestamp(pcommon.NewTimestampFromTime(startTime)) @@ -189,6 +190,19 @@ func setSpanStatus(attrs pcommon.Map, span ptrace.Span) { statusMessage := "" statusExists := false + if errorVal, ok := attrs.Get(tagError); ok && errorVal.Type() == pcommon.ValueTypeBool { + if errorVal.Bool() { + statusCode = ptrace.StatusCodeError + attrs.Remove(tagError) + statusExists = true + if desc, ok := extractStatusDescFromAttr(attrs); ok { + statusMessage = desc + } else if descAttr, ok := attrs.Get(tagHTTPStatusMsg); ok { + statusMessage = descAttr.Str() + } + } + } + if codeAttr, ok := attrs.Get(conventions.OtelStatusCode); ok { if !statusExists { // The error tag is the ultimate truth for a Jaeger spans' error @@ -335,7 +349,7 @@ func dbSpanLogsToSpanEvents(logs []dbmodel.Log, events ptrace.SpanEventSlice) { attrs := event.Attributes() attrs.EnsureCapacity(len(log.Fields)) dbTagsToAttributes(log.Fields, attrs) - if name, ok := attrs.Get(eventNameAttr); ok { + if name, ok := attrs.Get(eventNameAttr); ok && name.Type() == pcommon.ValueTypeStr { event.SetName(name.Str()) attrs.Remove(eventNameAttr) } diff --git a/internal/storage/v2/elasticsearch/tracestore/from_dbmodel_test.go b/internal/storage/v2/elasticsearch/tracestore/from_dbmodel_test.go index eb3022b7fc9..64a28099723 100644 --- a/internal/storage/v2/elasticsearch/tracestore/from_dbmodel_test.go +++ b/internal/storage/v2/elasticsearch/tracestore/from_dbmodel_test.go @@ -8,11 +8,12 @@ package tracestore import ( "encoding/hex" + "encoding/json" + "os" "strconv" "testing" "time" - idutils "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/core/xidutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" @@ -23,15 +24,7 @@ import ( "github.com/jaegertracing/jaeger/internal/storage/elasticsearch/dbmodel" ) -// Use timespamp with microsecond granularity to work well with jaeger thrift translation -var ( - testSpanStartTime = time.Date(2020, 2, 11, 20, 26, 12, 321000, time.UTC) - testSpanStartTimestamp = pcommon.NewTimestampFromTime(testSpanStartTime) - testSpanEventTime = time.Date(2020, 2, 11, 20, 26, 13, 123000, time.UTC) - testSpanEventTimestamp = pcommon.NewTimestampFromTime(testSpanEventTime) - testSpanEndTime = time.Date(2020, 2, 11, 20, 26, 13, 789000, time.UTC) - testSpanEndTimestamp = pcommon.NewTimestampFromTime(testSpanEndTime) -) +var testSpanEventTime = time.Date(2020, 2, 11, 20, 26, 13, 123000, time.UTC) func TestCodeFromAttr(t *testing.T) { tests := []struct { @@ -333,88 +326,6 @@ func TestSetAttributesFromDbTags(t *testing.T) { } } -func TestFromDBModel(t *testing.T) { - tests := []struct { - name string - jb []dbmodel.Span - td ptrace.Traces - }{ - { - name: "empty", - jb: []dbmodel.Span{}, - td: ptrace.NewTraces(), - }, - { - name: "two-spans-child-parent", - jb: []dbmodel.Span{ - generateProtoSpan(), - generateProtoChildSpan(), - }, - td: generateTracesWithDifferentResourceTwoSpansChildParent(), - }, - { - name: "two-spans-with-follower", - jb: []dbmodel.Span{ - generateProtoSpan(), - generateProtoFollowerSpan(), - }, - td: generateTracesWithDifferentResourceTwoSpansWithFollower(), - }, - { - name: "a-spans-with-two-parent", - jb: []dbmodel.Span{ - generateProtoSpan(), - generateProtoFollowerSpan(), - generateProtoTwoParentsSpan(), - }, - td: generateTracesWithDifferentResourceSpanWithTwoParents(), - }, - { - name: "no-error-from-server-span-with-4xx-http-code", - jb: []dbmodel.Span{ - { - StartTime: model.TimeAsEpochMicroseconds(testSpanStartTime), - Duration: model.DurationAsMicroseconds(testSpanEndTime.Sub(testSpanStartTime)), - Tags: []dbmodel.KeyValue{ - { - Key: model.SpanKindKey, - Type: dbmodel.StringType, - Value: string(model.SpanKindServer), - }, - { - Key: conventions.AttributeHTTPStatusCode, - Type: dbmodel.StringType, - Value: "404", - }, - }, - Process: dbmodel.Process{ - ServiceName: noServiceName, - }, - }, - }, - td: func() ptrace.Traces { - traces := ptrace.NewTraces() - span := traces.ResourceSpans().AppendEmpty().ScopeSpans().AppendEmpty().Spans().AppendEmpty() - span.SetStartTimestamp(testSpanStartTimestamp) - span.SetEndTimestamp(testSpanEndTimestamp) - span.SetKind(ptrace.SpanKindClient) - span.SetKind(ptrace.SpanKindServer) - span.Status().SetCode(ptrace.StatusCodeUnset) - span.Attributes().PutStr(conventions.AttributeHTTPStatusCode, "404") - return traces - }(), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - td, err := FromDBModel(test.jb) - require.NoError(t, err) - assert.Equal(t, test.td, td) - }) - } -} - func TestFromDBModelErrors(t *testing.T) { tests := []struct { name string @@ -470,63 +381,6 @@ func TestParentIdWhenRefTraceIdIsDifferent(t *testing.T) { assert.True(t, trace.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).ParentSpanID().IsEmpty()) } -func TestFromDBModelForTracesWithTwoLibraries(t *testing.T) { - jb := []dbmodel.Span{ - { - StartTime: model.TimeAsEpochMicroseconds(testSpanStartTime), - Duration: model.DurationAsMicroseconds(testSpanEndTime.Sub(testSpanStartTime)), - OperationName: "operation2", - Tags: []dbmodel.KeyValue{ - { - Key: conventions.AttributeOtelScopeName, - Type: dbmodel.StringType, - Value: "library2", - }, { - Key: conventions.AttributeOtelScopeVersion, - Type: dbmodel.StringType, - Value: "0.42.0", - }, - }, - }, - { - TraceID: dbmodel.TraceID("0000000000000000"), - StartTime: model.TimeAsEpochMicroseconds(testSpanStartTime), - Duration: model.DurationAsMicroseconds(testSpanEndTime.Sub(testSpanStartTime)), - OperationName: "operation1", - Tags: []dbmodel.KeyValue{ - { - Key: conventions.AttributeOtelScopeName, - Type: dbmodel.StringType, - Value: "library1", - }, { - Key: conventions.AttributeOtelScopeVersion, - Type: dbmodel.StringType, - Value: "0.42.0", - }, - }, - }, - } - expected := generateTracesTwoSpansFromTwoLibraries() - library1Span := expected.ResourceSpans().At(0).ScopeSpans().At(0) - library2Span := expected.ResourceSpans().At(1).ScopeSpans().At(0) - - actual, err := FromDBModel(jb) - require.NoError(t, err) - - assert.Equal(t, 2, actual.ResourceSpans().Len()) - assert.Equal(t, 1, actual.ResourceSpans().At(0).ScopeSpans().Len()) - - ils0 := actual.ResourceSpans().At(0).ScopeSpans().At(0) - ils1 := actual.ResourceSpans().At(1).ScopeSpans().At(0) - if ils0.Scope().Name() == "library1" { - assert.Equal(t, library1Span, ils0) - assert.Equal(t, library2Span, ils1) - } else { - assert.Equal(t, library1Span, ils1) - assert.Equal(t, library2Span, ils0) - } -} - func TestSetInternalSpanStatus(t *testing.T) { okStatus := ptrace.NewStatus() okStatus.SetCode(ptrace.StatusCodeOk) @@ -612,6 +466,24 @@ func TestSetInternalSpanStatus(t *testing.T) { status: errorStatusWith404Message, attrsModifiedLen: 2, }, + { + name: "error tag set and message present", + attrs: map[string]any{ + tagError: true, + conventions.OtelStatusDescription: "Error: Invalid argument", + }, + status: errorStatusWithMessage, + attrsModifiedLen: 0, + }, + { + name: "error tag set and http tag message present", + attrs: map[string]any{ + tagError: true, + tagHTTPStatusMsg: "HTTP 404: Not Found", + }, + status: errorStatusWith404Message, + attrsModifiedLen: 1, + }, } for _, test := range tests { @@ -628,49 +500,6 @@ func TestSetInternalSpanStatus(t *testing.T) { } } -func TestFromDBModelToInternalTraces(t *testing.T) { - batches := []dbmodel.Span{ - generateProtoSpan(), - generateProtoSpan(), - generateProtoChildSpan(), - } - - expected := generateTracesOneSpanNoResource() - resource := generateTracesResourceOnly().ResourceSpans().At(0).Resource() - resource.CopyTo(expected.ResourceSpans().At(0).Resource()) - span := expected.ResourceSpans().AppendEmpty().ScopeSpans().AppendEmpty().Spans().AppendEmpty() - resource.CopyTo(expected.ResourceSpans().At(1).Resource()) - expected.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).CopyTo(span) - tgt := expected.ResourceSpans().AppendEmpty() - twoSpans := generateTracesWithDifferentResourceTwoSpansChildParent().ResourceSpans().At(0) - twoSpans.CopyTo(tgt) - - got, err := FromDBModel(batches) - - require.NoError(t, err) - - assert.Equal(t, expected.ResourceSpans().Len(), got.ResourceSpans().Len()) - assert.Equal(t, expected.SpanCount(), got.SpanCount()) - - lenbatches := expected.ResourceSpans().Len() - found := 0 - - for i := 0; i < lenbatches; i++ { - rsExpected := expected.ResourceSpans().At(i) - for j := 0; j < lenbatches; j++ { - got.ResourceSpans().RemoveIf(func(_ ptrace.ResourceSpans) bool { - nameExpected := rsExpected.ScopeSpans().At(0).Spans().At(0).Name() - nameGot := got.ResourceSpans().At(j).ScopeSpans().At(0).Scope().Name() - if nameExpected == nameGot { - assert.Equal(t, nameGot, found) - assert.Equal(t, got.SpanCount(), found) - } - return nameExpected == nameGot - }) - } - } -} - func TestDBSpanKindToOTELSpanKind(t *testing.T) { tests := []struct { jSpanKind string @@ -709,163 +538,17 @@ func TestDBSpanKindToOTELSpanKind(t *testing.T) { } } -func generateTracesResourceOnly() ptrace.Traces { - td := generateTracesOneEmptyResourceSpans() - rs := td.ResourceSpans().At(0).Resource() - rs.Attributes().PutStr(conventions.AttributeServiceName, "service-1") - rs.Attributes().PutInt("int-attr-1", 123) - return td -} - -func generateTracesOneEmptyResourceSpans() ptrace.Traces { - td := ptrace.NewTraces() - td.ResourceSpans().AppendEmpty() - return td -} - -func generateTracesResourceOnlyWithNoAttrs() ptrace.Traces { - return generateTracesOneEmptyResourceSpans() -} - -func GenerateTracesOneSpanNoResource() ptrace.Traces { - td := generateTracesOneEmptyResourceSpans() - rs0 := td.ResourceSpans().At(0) - fillSpanOne(rs0.ScopeSpans().AppendEmpty().Spans().AppendEmpty()) - return td -} - -func fillSpanOne(span ptrace.Span) { - span.SetName("operationA") - span.SetStartTimestamp(testSpanStartTimestamp) - span.SetEndTimestamp(testSpanEndTimestamp) - span.SetDroppedAttributesCount(1) - evs := span.Events() - ev0 := evs.AppendEmpty() - ev0.SetTimestamp(testSpanEventTimestamp) - ev0.SetName("event-with-attr") - ev0.Attributes().PutStr("span-event-attr", "span-event-attr-val") - ev0.SetDroppedAttributesCount(2) - ev1 := evs.AppendEmpty() - ev1.SetTimestamp(testSpanEventTimestamp) - ev1.SetName("event") - ev1.SetDroppedAttributesCount(2) - span.SetDroppedEventsCount(1) - status := span.Status() - status.SetCode(ptrace.StatusCodeError) - status.SetMessage("status-cancelled") -} - -func generateTracesOneSpanNoResource() ptrace.Traces { - td := GenerateTracesOneSpanNoResource() - span := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0) - span.SetSpanID([8]byte{0xAF, 0xAE, 0xAD, 0xAC, 0xAB, 0xAA, 0xA9, 0xA8}) - span.SetTraceID([16]byte{0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x80}) - span.SetDroppedAttributesCount(0) - span.SetDroppedEventsCount(0) - span.SetStartTimestamp(testSpanStartTimestamp) - span.SetEndTimestamp(testSpanEndTimestamp) - span.SetKind(ptrace.SpanKindClient) - span.Status().SetCode(ptrace.StatusCodeError) - span.Events().At(0).SetTimestamp(testSpanEventTimestamp) - span.Events().At(0).SetDroppedAttributesCount(0) - span.Events().At(0).SetName("event-with-attr") - span.Events().At(1).SetTimestamp(testSpanEventTimestamp) - span.Events().At(1).SetDroppedAttributesCount(0) - span.Events().At(1).SetName("") - span.Events().At(1).Attributes().PutInt("attr-int", 123) - return td -} - -func generateTracesWithLibraryInfo() ptrace.Traces { - td := generateTracesOneSpanNoResource() - rs0 := td.ResourceSpans().At(0) - rs0ils0 := rs0.ScopeSpans().At(0) - rs0ils0.Scope().SetName("io.opentelemetry.test") - rs0ils0.Scope().SetVersion("0.42.0") - return td -} - -func generateTracesOneSpanNoResourceWithTraceState() ptrace.Traces { - td := generateTracesOneSpanNoResource() - span := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0) - span.TraceState().FromRaw("lasterror=f39cd56cc44274fd5abd07ef1164246d10ce2955") - return td -} - -func generateProtoSpan() dbmodel.Span { - spanId := [8]byte{0xAF, 0xAE, 0xAD, 0xAC, 0xAB, 0xAA, 0xA9, 0xA8} - traceId := [16]byte{0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x80} - return dbmodel.Span{ - TraceID: getDbTraceIdFromByteArray(traceId), - SpanID: getDbSpanIdFromByteArray(spanId), - OperationName: "operationA", - StartTime: model.TimeAsEpochMicroseconds(testSpanStartTime), - Duration: model.DurationAsMicroseconds(testSpanEndTime.Sub(testSpanStartTime)), - Logs: []dbmodel.Log{ - { - Timestamp: model.TimeAsEpochMicroseconds(testSpanEventTime), - Fields: []dbmodel.KeyValue{ - { - Key: eventNameAttr, - Type: dbmodel.StringType, - Value: "event-with-attr", - }, - { - Key: "span-event-attr", - Type: dbmodel.StringType, - Value: "span-event-attr-val", - }, - }, - }, - { - Timestamp: model.TimeAsEpochMicroseconds(testSpanEventTime), - Fields: []dbmodel.KeyValue{ - { - Key: "attr-int", - Type: dbmodel.Int64Type, - Value: "123", - }, - }, - }, - }, - Tags: []dbmodel.KeyValue{ - { - Key: model.SpanKindKey, - Type: dbmodel.StringType, - Value: string(model.SpanKindClient), - }, - { - Key: conventions.OtelStatusCode, - Type: dbmodel.StringType, - Value: statusError, - }, - { - Key: conventions.OtelStatusDescription, - Type: dbmodel.StringType, - Value: "status-cancelled", - }, - }, - Process: dbmodel.Process{ - ServiceName: noServiceName, - }, - } -} - -func generateProtoSpanWithLibraryInfo(libraryName string) dbmodel.Span { - span := generateProtoSpan() - span.Tags = append([]dbmodel.KeyValue{ - { - Key: conventions.AttributeOtelScopeName, - Type: dbmodel.StringType, - Value: libraryName, - }, { - Key: conventions.AttributeOtelScopeVersion, - Type: dbmodel.StringType, - Value: "0.42.0", - }, - }, span.Tags...) - - return span +func TestFromDbModel_Fixtures(t *testing.T) { + tracesData, spansData := loadFixtures(t, 1) + unmarshaller := ptrace.JSONUnmarshaler{} + expectedTd, err := unmarshaller.UnmarshalTraces(tracesData) + require.NoError(t, err) + spans := ToDBModel(expectedTd) + assert.Len(t, spans, 1) + testSpans(t, spansData, spans[0]) + actualTd, err := FromDBModel(spans) + require.NoError(t, err) + testTraces(t, tracesData, actualTd) } func getDbTraceIdFromByteArray(arr [16]byte) dbmodel.TraceID { @@ -876,295 +559,13 @@ func getDbSpanIdFromByteArray(arr [8]byte) dbmodel.SpanID { return dbmodel.SpanID(hex.EncodeToString(arr[:])) } -func generateProtoSpanWithTraceState() dbmodel.Span { - spanId := [8]byte{0xAF, 0xAE, 0xAD, 0xAC, 0xAB, 0xAA, 0xA9, 0xA8} - traceId := [16]byte{0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x80} - return dbmodel.Span{ - TraceID: getDbTraceIdFromByteArray(traceId), - SpanID: getDbSpanIdFromByteArray(spanId), - OperationName: "operationA", - StartTime: model.TimeAsEpochMicroseconds(testSpanStartTime), - Duration: model.DurationAsMicroseconds(testSpanEndTime.Sub(testSpanStartTime)), - Logs: []dbmodel.Log{ - { - Timestamp: model.TimeAsEpochMicroseconds(testSpanEventTime), - Fields: []dbmodel.KeyValue{ - { - Key: eventNameAttr, - Type: dbmodel.StringType, - Value: "event-with-attr", - }, - { - Key: "span-event-attr", - Type: dbmodel.StringType, - Value: "span-event-attr-val", - }, - }, - }, - { - Timestamp: model.TimeAsEpochMicroseconds(testSpanEventTime), - Fields: []dbmodel.KeyValue{ - { - Key: "attr-int", - Type: dbmodel.Int64Type, - Value: "123", - }, - }, - }, - }, - Tags: []dbmodel.KeyValue{ - { - Key: model.SpanKindKey, - Type: dbmodel.StringType, - Value: string(model.SpanKindClient), - }, - { - Key: conventions.OtelStatusCode, - Type: dbmodel.StringType, - Value: statusError, - }, - { - Key: conventions.OtelStatusDescription, - Type: dbmodel.StringType, - Value: "status-cancelled", - }, - { - Key: tagW3CTraceState, - Type: dbmodel.StringType, - Value: "lasterror=f39cd56cc44274fd5abd07ef1164246d10ce2955", - }, - }, - Process: dbmodel.Process{ - ServiceName: noServiceName, - }, - } -} - -func generateTracesTwoSpansChildParent() ptrace.Traces { - td := generateTracesOneSpanNoResource() - spans := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans() - - span := spans.AppendEmpty() - setChildSpan(span, spans.At(0)) - return td -} - -func generateTracesWithDifferentResourceTwoSpansChildParent() ptrace.Traces { - td := generateTracesOneSpanNoResource() - parentSpan := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0) - spans := td.ResourceSpans().AppendEmpty().ScopeSpans().AppendEmpty().Spans() - span := spans.AppendEmpty() - setChildSpan(span, parentSpan) - return td -} - -func setChildSpan(span, parentSpan ptrace.Span) { - span.SetName("operationB") - span.SetSpanID([8]byte{0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x18}) - span.SetParentSpanID(parentSpan.SpanID()) - span.SetKind(ptrace.SpanKindServer) - span.SetTraceID(parentSpan.TraceID()) - span.SetStartTimestamp(parentSpan.StartTimestamp()) - span.SetEndTimestamp(parentSpan.EndTimestamp()) - span.Status().SetCode(ptrace.StatusCodeUnset) - span.Attributes().PutInt(conventions.AttributeHTTPStatusCode, 404) -} - -func generateProtoChildSpan() dbmodel.Span { - traceID := getDbTraceIdFromByteArray([16]byte{0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x80}) - return dbmodel.Span{ - TraceID: traceID, - SpanID: getDbSpanIdFromByteArray([8]byte{0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x18}), - OperationName: "operationB", - StartTime: model.TimeAsEpochMicroseconds(testSpanStartTime), - Duration: model.DurationAsMicroseconds(testSpanEndTime.Sub(testSpanStartTime)), - Tags: []dbmodel.KeyValue{ - { - Key: conventions.AttributeHTTPStatusCode, - Type: dbmodel.Int64Type, - Value: "404", - }, - { - Key: model.SpanKindKey, - Type: dbmodel.StringType, - Value: string(model.SpanKindServer), - }, - }, - References: []dbmodel.Reference{ - { - TraceID: traceID, - SpanID: getDbSpanIdFromByteArray([8]byte{0xAF, 0xAE, 0xAD, 0xAC, 0xAB, 0xAA, 0xA9, 0xA8}), - RefType: dbmodel.ChildOf, - }, - }, - Process: dbmodel.Process{ - ServiceName: noServiceName, - }, - } -} - -func generateTracesTwoSpansWithFollower() ptrace.Traces { - td := generateTracesOneSpanNoResource() - spans := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans() - - span := spans.AppendEmpty() - setFollowFromSpan(span, spans.At(0)) - return td -} - -func generateTracesWithDifferentResourceTwoSpansWithFollower() ptrace.Traces { - td := generateTracesOneSpanNoResource() - followFromSpan := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0) - spans := td.ResourceSpans().AppendEmpty().ScopeSpans().AppendEmpty().Spans() - span := spans.AppendEmpty() - setFollowFromSpan(span, followFromSpan) - return td -} - -func setFollowFromSpan(span, followFromSpan ptrace.Span) { - span.SetName("operationC") - span.SetSpanID([8]byte{0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x18}) - span.SetTraceID(followFromSpan.TraceID()) - span.SetParentSpanID(followFromSpan.SpanID()) - span.SetStartTimestamp(followFromSpan.EndTimestamp()) - span.SetEndTimestamp(followFromSpan.EndTimestamp() + 1000000) - span.SetKind(ptrace.SpanKindConsumer) - span.Status().SetCode(ptrace.StatusCodeOk) - span.Status().SetMessage("status-ok") - link := span.Links().AppendEmpty() - link.SetTraceID(span.TraceID()) - link.SetSpanID(followFromSpan.SpanID()) - link.Attributes().PutStr( - conventions.AttributeOpentracingRefType, - conventions.AttributeOpentracingRefTypeFollowsFrom, - ) -} - -func generateProtoFollowerSpan() dbmodel.Span { - traceID := getDbTraceIdFromByteArray([16]byte{0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x80}) - return dbmodel.Span{ - TraceID: traceID, - SpanID: getDbSpanIdFromByteArray([8]byte{0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x18}), - OperationName: "operationC", - StartTime: model.TimeAsEpochMicroseconds(testSpanEndTime), - Duration: model.DurationAsMicroseconds(time.Millisecond), - Tags: []dbmodel.KeyValue{ - { - Key: model.SpanKindKey, - Type: dbmodel.StringType, - Value: string(model.SpanKindConsumer), - }, - { - Key: conventions.OtelStatusCode, - Type: dbmodel.StringType, - Value: statusOk, - }, - { - Key: conventions.OtelStatusDescription, - Type: dbmodel.StringType, - Value: "status-ok", - }, - }, - References: []dbmodel.Reference{ - { - TraceID: traceID, - SpanID: getDbSpanIdFromByteArray([8]byte{0xAF, 0xAE, 0xAD, 0xAC, 0xAB, 0xAA, 0xA9, 0xA8}), - RefType: dbmodel.FollowsFrom, - }, - }, - Process: dbmodel.Process{ - ServiceName: noServiceName, - }, - } -} - -func generateTracesSpanWithTwoParents() ptrace.Traces { - td := generateTracesTwoSpansWithFollower() - spans := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans() - parent := spans.At(0) - parent2 := spans.At(1) - span := spans.AppendEmpty() - setSpanWithTwoParents(span, parent, parent2) - return td -} - -func generateTracesWithDifferentResourceSpanWithTwoParents() ptrace.Traces { - td := generateTracesWithDifferentResourceTwoSpansWithFollower() - parent1 := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0) - parent2 := td.ResourceSpans().At(1).ScopeSpans().At(0).Spans().At(0) - span := td.ResourceSpans().AppendEmpty().ScopeSpans().AppendEmpty().Spans().AppendEmpty() - setSpanWithTwoParents(span, parent1, parent2) - return td -} - -func setSpanWithTwoParents(span, parent, parent2 ptrace.Span) { - span.SetName("operationD") - span.SetSpanID([8]byte{0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x20}) - span.SetTraceID(parent.TraceID()) - span.SetStartTimestamp(parent.StartTimestamp()) - span.SetEndTimestamp(parent.EndTimestamp()) - span.SetParentSpanID(parent.SpanID()) - span.SetKind(ptrace.SpanKindConsumer) - span.Status().SetCode(ptrace.StatusCodeOk) - span.Status().SetMessage("status-ok") - - link := span.Links().AppendEmpty() - link.SetTraceID(parent2.TraceID()) - link.SetSpanID(parent2.SpanID()) - link.Attributes().PutStr( - conventions.AttributeOpentracingRefType, - conventions.AttributeOpentracingRefTypeChildOf, - ) -} - -func generateProtoTwoParentsSpan() dbmodel.Span { - traceID := getDbTraceIdFromByteArray([16]byte{0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x80}) - return dbmodel.Span{ - TraceID: traceID, - SpanID: getDbSpanIdFromByteArray([8]byte{0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x20}), - OperationName: "operationD", - StartTime: model.TimeAsEpochMicroseconds(testSpanStartTime), - Duration: model.DurationAsMicroseconds(testSpanEndTime.Sub(testSpanStartTime)), - Tags: []dbmodel.KeyValue{ - { - Key: model.SpanKindKey, - Type: dbmodel.StringType, - Value: string(model.SpanKindConsumer), - }, - { - Key: conventions.OtelStatusCode, - Type: dbmodel.StringType, - Value: statusOk, - }, - { - Key: conventions.OtelStatusDescription, - Type: dbmodel.StringType, - Value: "status-ok", - }, - }, - References: []dbmodel.Reference{ - { - TraceID: traceID, - SpanID: getDbSpanIdFromByteArray([8]byte{0xAF, 0xAE, 0xAD, 0xAC, 0xAB, 0xAA, 0xA9, 0xA8}), - RefType: dbmodel.ChildOf, - }, - { - TraceID: traceID, - SpanID: getDbSpanIdFromByteArray([8]byte{0x1F, 0x1E, 0x1D, 0x1C, 0x1B, 0x1A, 0x19, 0x18}), - RefType: dbmodel.ChildOf, - }, - }, - Process: dbmodel.Process{ - ServiceName: noServiceName, - }, - } -} - func BenchmarkProtoBatchToInternalTraces(b *testing.B) { - jb := []dbmodel.Span{ - generateProtoSpan(), - generateProtoChildSpan(), - } + data, err := os.ReadFile("fixtures.es_01.json") + require.NoError(b, err) + var dbSpan dbmodel.Span + err = json.Unmarshal(data, &dbSpan) + require.NoError(b, err) + jb := []dbmodel.Span{dbSpan} b.ResetTimer() for n := 0; n < b.N; n++ { @@ -1172,32 +573,3 @@ func BenchmarkProtoBatchToInternalTraces(b *testing.B) { assert.NoError(b, err) } } - -func generateTracesTwoSpansFromTwoLibraries() ptrace.Traces { - td := generateTracesOneEmptyResourceSpans() - - rs0 := td.ResourceSpans().At(0) - rs0.ScopeSpans().EnsureCapacity(1) - - rs0ils0 := rs0.ScopeSpans().AppendEmpty() - rs0ils0.Scope().SetName("library1") - rs0ils0.Scope().SetVersion("0.42.0") - span1 := rs0ils0.Spans().AppendEmpty() - span1.SetTraceID(idutils.UInt64ToTraceID(0, 0)) - span1.SetSpanID(idutils.UInt64ToSpanID(0)) - span1.SetName("operation1") - span1.SetStartTimestamp(testSpanStartTimestamp) - span1.SetEndTimestamp(testSpanEndTimestamp) - - rs0ils1 := td.ResourceSpans().AppendEmpty().ScopeSpans().AppendEmpty() - rs0ils1.Scope().SetName("library2") - rs0ils1.Scope().SetVersion("0.42.0") - span2 := rs0ils1.Spans().AppendEmpty() - span2.SetTraceID(span1.TraceID()) - span2.SetSpanID(span1.SpanID()) - span2.SetName("operation2") - span2.SetStartTimestamp(testSpanStartTimestamp) - span2.SetEndTimestamp(testSpanEndTimestamp) - - return td -} diff --git a/internal/storage/v2/elasticsearch/tracestore/to_dbmodel.go b/internal/storage/v2/elasticsearch/tracestore/to_dbmodel.go index 54b14553e83..74e42a71381 100644 --- a/internal/storage/v2/elasticsearch/tracestore/to_dbmodel.go +++ b/internal/storage/v2/elasticsearch/tracestore/to_dbmodel.go @@ -7,6 +7,8 @@ package tracestore import ( + "encoding/hex" + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/ptrace" conventions "go.opentelemetry.io/collector/semconv/v1.16.0" @@ -22,6 +24,7 @@ const ( statusOk = "OK" tagW3CTraceState = "w3c.tracestate" tagHTTPStatusMsg = "http.status_message" + tagError = "error" ) // ToDBModel translates internal trace data into the DB Spans. @@ -96,8 +99,13 @@ func appendTagsFromAttributes(dest []dbmodel.KeyValue, attrs pcommon.Map) []dbmo } func attributeToDbTag(key string, attr pcommon.Value) dbmodel.KeyValue { - // TODO why are all values being converted to strings? - tag := dbmodel.KeyValue{Key: key, Value: attr.AsString()} + var tag dbmodel.KeyValue + if attr.Type() == pcommon.ValueTypeBytes { + tag = dbmodel.KeyValue{Key: key, Value: hex.EncodeToString(attr.Bytes().AsRaw())} + } else { + // TODO why are all values being converted to strings? + tag = dbmodel.KeyValue{Key: key, Value: attr.AsString()} + } switch attr.Type() { case pcommon.ValueTypeStr: tag.Type = dbmodel.StringType @@ -120,15 +128,17 @@ func spanToDbSpan(span ptrace.Span, libraryTags pcommon.InstrumentationScope, pr parentSpanID := dbmodel.SpanID(span.ParentSpanID().String()) startTime := span.StartTimestamp().AsTime() return dbmodel.Span{ - TraceID: traceID, - SpanID: dbmodel.SpanID(span.SpanID().String()), - OperationName: span.Name(), - References: linksToDbSpanRefs(span.Links(), parentSpanID, traceID), - StartTime: model.TimeAsEpochMicroseconds(startTime), - Duration: model.DurationAsMicroseconds(span.EndTimestamp().AsTime().Sub(startTime)), - Tags: getDbSpanTags(span, libraryTags), - Logs: spanEventsToDbSpanLogs(span.Events()), - Process: process, + TraceID: traceID, + SpanID: dbmodel.SpanID(span.SpanID().String()), + OperationName: span.Name(), + References: linksToDbSpanRefs(span.Links(), parentSpanID, traceID), + StartTime: model.TimeAsEpochMicroseconds(startTime), + StartTimeMillis: model.TimeAsEpochMicroseconds(startTime) / 1000, + Duration: model.DurationAsMicroseconds(span.EndTimestamp().AsTime().Sub(startTime)), + Tags: getDbSpanTags(span, libraryTags), + Logs: spanEventsToDbSpanLogs(span.Events()), + Process: process, + Flags: span.Flags(), } } @@ -280,14 +290,13 @@ func getTagFromSpanKind(spanKind ptrace.SpanKind) (dbmodel.KeyValue, bool) { } func getTagFromStatusCode(statusCode ptrace.StatusCode) (dbmodel.KeyValue, bool) { - switch statusCode { - case ptrace.StatusCodeError: + if statusCode == ptrace.StatusCodeError { return dbmodel.KeyValue{ - Key: conventions.OtelStatusCode, - Type: dbmodel.StringType, - Value: statusError, + Key: tagError, + Type: dbmodel.BoolType, + Value: "true", }, true - case ptrace.StatusCodeOk: + } else if statusCode == ptrace.StatusCodeOk { return dbmodel.KeyValue{ Key: conventions.OtelStatusCode, Type: dbmodel.StringType, diff --git a/internal/storage/v2/elasticsearch/tracestore/to_dbmodel_test.go b/internal/storage/v2/elasticsearch/tracestore/to_dbmodel_test.go index 8b2308568e3..6ec33e5e126 100644 --- a/internal/storage/v2/elasticsearch/tracestore/to_dbmodel_test.go +++ b/internal/storage/v2/elasticsearch/tracestore/to_dbmodel_test.go @@ -7,6 +7,10 @@ package tracestore import ( + "bytes" + "encoding/json" + "fmt" + "os" "testing" "github.com/stretchr/testify/assert" @@ -39,9 +43,9 @@ func TestGetTagFromStatusCode(t *testing.T) { name: "error", code: ptrace.StatusCodeError, tag: dbmodel.KeyValue{ - Key: conventions.OtelStatusCode, - Type: dbmodel.StringType, - Value: statusError, + Key: tagError, + Type: dbmodel.BoolType, + Value: "true", }, }, } @@ -236,7 +240,7 @@ func TestAttributesToDbSpanTags(t *testing.T) { { Key: "bytes-val", Type: dbmodel.BinaryType, - Value: "AQIDBA==", + Value: "01020304", }, { Key: conventions.AttributeServiceName, @@ -263,108 +267,69 @@ func TestAttributesToDbSpanTags_MapType(t *testing.T) { require.Equal(t, expected, got) } -func TestToDBModel(t *testing.T) { - tests := []struct { - name string - td ptrace.Traces - db []dbmodel.Span - }{ - { - name: "empty", - td: ptrace.NewTraces(), - }, - - { - name: "no-resource-attrs", - td: generateTracesResourceOnlyWithNoAttrs(), - }, - - { - name: "one-span-no-resources", - td: generateTracesOneSpanNoResourceWithTraceState(), - db: []dbmodel.Span{generateProtoSpanWithTraceState()}, - }, - { - name: "library-info", - td: generateTracesWithLibraryInfo(), - db: []dbmodel.Span{generateProtoSpanWithLibraryInfo("io.opentelemetry.test")}, - }, - { - name: "two-spans-child-parent", - td: generateTracesTwoSpansChildParent(), - db: []dbmodel.Span{ - generateProtoSpan(), - generateProtoChildSpan(), - }, - }, - - { - name: "two-spans-with-follower", - td: generateTracesTwoSpansWithFollower(), - db: []dbmodel.Span{ - generateProtoSpan(), - generateProtoFollowerSpan(), - }, - }, +func TestToDbModel_Fixtures(t *testing.T) { + tracesStr, spansStr := loadFixtures(t, 1) + var span dbmodel.Span + err := json.Unmarshal(spansStr, &span) + require.NoError(t, err) + td, err := FromDBModel([]dbmodel.Span{span}) + require.NoError(t, err) + testTraces(t, tracesStr, td) + spans := ToDBModel(td) + assert.Len(t, spans, 1) + testSpans(t, spansStr, spans[0]) +} - { - name: "span-with-span-event-attribute", - td: generateTracesOneSpanNoResourceWithEventAttribute(), - db: []dbmodel.Span{generateJProtoSpanWithEventAttribute()}, - }, - { - name: "a-spans-with-two-parent", - td: generateTracesSpanWithTwoParents(), - db: []dbmodel.Span{ - generateProtoSpan(), - generateProtoFollowerSpan(), - generateProtoTwoParentsSpan(), - }, - }, - } +func writeActualData(t *testing.T, name string, data []byte) { + var prettyJson bytes.Buffer + err := json.Indent(&prettyJson, data, "", " ") + require.NoError(t, err) + path := "fixtures/actual_" + name + ".json" + err = os.WriteFile(path, prettyJson.Bytes(), 0o644) + require.NoError(t, err) + t.Log("Saved the actual " + name + " to " + path) +} - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - jbs := ToDBModel(test.td) - if test.db == nil { - assert.Empty(t, jbs) - } else { - require.Len(t, jbs, len(test.db)) - assert.Equal(t, test.db, jbs) - } - }) - } +// Loads and returns domain model and JSON model fixtures with given number i. +func loadFixtures(t *testing.T, i int) (tracesData []byte, spansData []byte) { + var err error + inTraces := fmt.Sprintf("fixtures/otel_traces_%02d.json", i) + tracesData, err = os.ReadFile(inTraces) + require.NoError(t, err) + inSpans := fmt.Sprintf("fixtures/es_%02d.json", i) + spansData, err = os.ReadFile(inSpans) + require.NoError(t, err) + return tracesData, spansData } -func generateTracesOneSpanNoResourceWithEventAttribute() ptrace.Traces { - td := generateTracesOneSpanNoResource() - event := td.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Events().At(0) - event.SetName("must-be-ignorred") - event.Attributes().PutStr("event", "must-be-used-instead-of-event-name") - return td +func testTraces(t *testing.T, expectedTraces []byte, actualTraces ptrace.Traces) { + unmarshaller := ptrace.JSONUnmarshaler{} + expectedTd, err := unmarshaller.UnmarshalTraces(expectedTraces) + require.NoError(t, err) + if !assert.Equal(t, expectedTd, actualTraces) { + marshaller := ptrace.JSONMarshaler{} + actualTd, err := marshaller.MarshalTraces(actualTraces) + require.NoError(t, err) + writeActualData(t, "traces", actualTd) + } } -func generateJProtoSpanWithEventAttribute() dbmodel.Span { - span := generateProtoSpan() - span.Logs[0].Fields = []dbmodel.KeyValue{ - { - Key: "span-event-attr", - Type: dbmodel.StringType, - Value: "span-event-attr-val", - }, - { - Key: eventNameAttr, - Type: dbmodel.StringType, - Value: "must-be-used-instead-of-event-name", - }, +func testSpans(t *testing.T, expectedSpan []byte, actualSpan dbmodel.Span) { + buf := &bytes.Buffer{} + enc := json.NewEncoder(buf) + enc.SetIndent("", " ") + require.NoError(t, enc.Encode(actualSpan)) + if !assert.Equal(t, string(expectedSpan), buf.String()) { + writeActualData(t, "spans", buf.Bytes()) } - return span } func BenchmarkInternalTracesToDbSpans(b *testing.B) { - td := generateTracesTwoSpansChildParent() - resource := generateTracesResourceOnly().ResourceSpans().At(0).Resource() - resource.CopyTo(td.ResourceSpans().At(0).Resource()) + unmarshaller := ptrace.JSONUnmarshaler{} + data, err := os.ReadFile("fixtures/otel_traces_01.json") + require.NoError(b, err) + td, err := unmarshaller.UnmarshalTraces(data) + require.NoError(b, err) b.ResetTimer() for n := 0; n < b.N; n++ {