Skip to content

Commit 2328a7a

Browse files
felixbarnyfelixbarny
committed
Optimize timestamp serialization
1 parent 4277f76 commit 2328a7a

File tree

4 files changed

+62
-29
lines changed

4 files changed

+62
-29
lines changed

‎exporter/elasticsearchexporter/exporter_test.go

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ func TestExporterLogs(t *testing.T) {
427427
body: func() pcommon.Value {
428428
return pcommon.NewValueStr("foo")
429429
}(),
430-
wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"text":"foo"}}`),
430+
wantDocument: []byte(`{"@timestamp":"0.0","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"0.0","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"text":"foo"}}`),
431431
},
432432
{
433433
body: func() pcommon.Value {
@@ -438,7 +438,7 @@ func TestExporterLogs(t *testing.T) {
438438
m.PutEmptyMap("inner").PutStr("foo", "bar")
439439
return vm
440440
}(),
441-
wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"flattened":{"true":true,"false":false,"inner":{"foo":"bar"}}}}`),
441+
wantDocument: []byte(`{"@timestamp":"0.0","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"0.0","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"flattened":{"true":true,"false":false,"inner":{"foo":"bar"}}}}`),
442442
},
443443
{
444444
body: func() pcommon.Value {
@@ -450,7 +450,7 @@ func TestExporterLogs(t *testing.T) {
450450
return vm
451451
}(),
452452
isEvent: true,
453-
wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value","event.name":"foo"},"event_name":"foo","data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"structured":{"true":true,"false":false,"inner":{"foo":"bar"}}}}`),
453+
wantDocument: []byte(`{"@timestamp":"0.0","attributes":{"attr.foo":"attr.foo.value","event.name":"foo"},"event_name":"foo","data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"0.0","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"structured":{"true":true,"false":false,"inner":{"foo":"bar"}}}}`),
454454
},
455455
{
456456
body: func() pcommon.Value {
@@ -461,7 +461,7 @@ func TestExporterLogs(t *testing.T) {
461461
s.AppendEmpty().SetEmptyMap().PutStr("foo", "bar")
462462
return vs
463463
}(),
464-
wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"flattened":{"value":["foo",false,{"foo":"bar"}]}}}`),
464+
wantDocument: []byte(`{"@timestamp":"0.0","attributes":{"attr.foo":"attr.foo.value"},"data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"0.0","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"flattened":{"value":["foo",false,{"foo":"bar"}]}}}`),
465465
},
466466
{
467467
body: func() pcommon.Value {
@@ -473,7 +473,7 @@ func TestExporterLogs(t *testing.T) {
473473
return vs
474474
}(),
475475
isEvent: true,
476-
wantDocument: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","attributes":{"attr.foo":"attr.foo.value","event.name":"foo"},"event_name":"foo","data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"1970-01-01T00:00:00.000000000Z","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"structured":{"value":["foo",false,{"foo":"bar"}]}}}`),
476+
wantDocument: []byte(`{"@timestamp":"0.0","attributes":{"attr.foo":"attr.foo.value","event.name":"foo"},"event_name":"foo","data_stream":{"dataset":"attr.dataset.otel","namespace":"resource.attribute.namespace","type":"logs"},"observed_timestamp":"0.0","resource":{"attributes":{"resource.attr.foo":"resource.attr.foo.value"}},"scope":{},"body":{"structured":{"value":["foo",false,{"foo":"bar"}]}}}`),
477477
},
478478
} {
479479
rec := newBulkRecorder()
@@ -1196,19 +1196,19 @@ func TestExporterMetrics(t *testing.T) {
11961196
expected := []itemRequest{
11971197
{
11981198
Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.foo":"histogram"}}}`),
1199-
Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"metric.foo":{"counts":[1,2,3,4],"values":[0.5,1.5,2.5,3.0]}},"resource":{},"scope":{}}`),
1199+
Document: []byte(`{"@timestamp":"0.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"metric.foo":{"counts":[1,2,3,4],"values":[0.5,1.5,2.5,3.0]}},"resource":{},"scope":{}}`),
12001200
},
12011201
{
12021202
Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.foo":"histogram"}}}`),
1203-
Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"metric.foo":{"counts":[4,5,6,7],"values":[2.0,4.5,5.5,6.0]}},"resource":{},"scope":{}}`),
1203+
Document: []byte(`{"@timestamp":"3600000.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"metric.foo":{"counts":[4,5,6,7],"values":[2.0,4.5,5.5,6.0]}},"resource":{},"scope":{}}`),
12041204
},
12051205
{
12061206
Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.sum":"gauge_double"}}}`),
1207-
Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"metric.sum":1.5},"resource":{},"scope":{},"start_timestamp":"1970-01-01T02:00:00.000000000Z"}`),
1207+
Document: []byte(`{"@timestamp":"3600000.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"metric.sum":1.5},"resource":{},"scope":{},"start_timestamp":"7200000.0"}`),
12081208
},
12091209
{
12101210
Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.metric.summary":"summary"}}}`),
1211-
Document: []byte(`{"@timestamp":"1970-01-01T03:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"metric.summary":{"sum":1.5,"value_count":1}},"resource":{},"scope":{},"start_timestamp":"1970-01-01T03:00:00.000000000Z"}`),
1211+
Document: []byte(`{"@timestamp":"10800000.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"metric.summary":{"sum":1.5,"value_count":1}},"resource":{},"scope":{},"start_timestamp":"10800000.0"}`),
12121212
},
12131213
}
12141214

@@ -1277,7 +1277,7 @@ func TestExporterMetrics(t *testing.T) {
12771277
expected := []itemRequest{
12781278
{
12791279
Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.sum":"gauge_long","metrics.summary":"summary"}}}`),
1280-
Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"sum":0,"summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{}}`),
1280+
Document: []byte(`{"@timestamp":"0.0","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"sum":0,"summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{}}`),
12811281
},
12821282
}
12831283

@@ -1327,11 +1327,11 @@ func TestExporterMetrics(t *testing.T) {
13271327
expected := []itemRequest{
13281328
{
13291329
Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.histogram.summary":"summary"}}}`),
1330-
Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"histogram.summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{}}`),
1330+
Document: []byte(`{"@timestamp":"0.0","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"histogram.summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{}}`),
13311331
},
13321332
{
13331333
Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.exphistogram.summary":"summary"}}}`),
1334-
Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"exphistogram.summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{}}`),
1334+
Document: []byte(`{"@timestamp":"3600000.0","_doc_count":10,"data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"exphistogram.summary":{"sum":1.0,"value_count":10}},"resource":{},"scope":{}}`),
13351335
},
13361336
}
13371337

@@ -1370,7 +1370,7 @@ func TestExporterMetrics(t *testing.T) {
13701370
expected := []itemRequest{
13711371
{
13721372
Action: []byte(`{"create":{"_index":"metrics-generic.otel-default","dynamic_templates":{"metrics.foo.bar":"gauge_long","metrics.foo":"gauge_long","metrics.foo.bar.baz":"gauge_long"}}}`),
1373-
Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"foo":0,"foo.bar":0,"foo.bar.baz":0},"resource":{},"scope":{}}`),
1373+
Document: []byte(`{"@timestamp":"0.0","data_stream":{"dataset":"generic.otel","namespace":"default","type":"metrics"},"attributes":{},"metrics":{"foo":0,"foo.bar":0,"foo.bar.baz":0},"resource":{},"scope":{}}`),
13741374
},
13751375
}
13761376

@@ -1655,11 +1655,11 @@ func TestExporterTraces(t *testing.T) {
16551655
expected := []itemRequest{
16561656
{
16571657
Action: []byte(`{"create":{"_index":"traces-generic.otel-default"}}`),
1658-
Document: []byte(`{"@timestamp":"1970-01-01T01:00:00.000000000Z","attributes":{"attr.foo":"attr.bar"},"data_stream":{"dataset":"generic.otel","namespace":"default","type":"traces"},"dropped_attributes_count":2,"dropped_events_count":3,"dropped_links_count":4,"duration":3600000000000,"kind":"Unspecified","links":[{"attributes":{"link.attr.foo":"link.attr.bar"},"dropped_attributes_count":11,"span_id":"0100000000000000","trace_id":"01000000000000000000000000000000","trace_state":"bar"}],"name":"name","resource":{"attributes":{"resource.foo":"resource.bar"}},"scope":{},"status":{"code":"Unset"},"trace_state":"foo"}`),
1658+
Document: []byte(`{"@timestamp":"3600000.0","attributes":{"attr.foo":"attr.bar"},"data_stream":{"dataset":"generic.otel","namespace":"default","type":"traces"},"dropped_attributes_count":2,"dropped_events_count":3,"dropped_links_count":4,"duration":3600000000000,"kind":"Unspecified","links":[{"attributes":{"link.attr.foo":"link.attr.bar"},"dropped_attributes_count":11,"span_id":"0100000000000000","trace_id":"01000000000000000000000000000000","trace_state":"bar"}],"name":"name","resource":{"attributes":{"resource.foo":"resource.bar"}},"scope":{},"status":{"code":"Unset"},"trace_state":"foo"}`),
16591659
},
16601660
{
16611661
Action: []byte(`{"create":{"_index":"logs-generic.otel-default"}}`),
1662-
Document: []byte(`{"@timestamp":"1970-01-01T00:00:00.000000000Z","event_name":"exception","attributes":{"event.attr.foo":"event.attr.bar","event.name":"exception"},"event_name":"exception","data_stream":{"dataset":"generic.otel","namespace":"default","type":"logs"},"dropped_attributes_count":1,"resource":{"attributes":{"resource.foo":"resource.bar"}},"scope":{}}`),
1662+
Document: []byte(`{"@timestamp":"0.0","event_name":"exception","attributes":{"event.attr.foo":"event.attr.bar","event.name":"exception"},"event_name":"exception","data_stream":{"dataset":"generic.otel","namespace":"default","type":"logs"},"dropped_attributes_count":1,"resource":{"attributes":{"resource.foo":"resource.bar"}},"scope":{}}`),
16631663
},
16641664
}
16651665

‎exporter/elasticsearchexporter/model_test.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -914,8 +914,6 @@ func TestMapLogAttributesToECS(t *testing.T) {
914914
type OTelRecord struct {
915915
TraceID OTelTraceID `json:"trace_id"`
916916
SpanID OTelSpanID `json:"span_id"`
917-
Timestamp time.Time `json:"@timestamp"`
918-
ObservedTimestamp time.Time `json:"observed_timestamp"`
919917
SeverityNumber int32 `json:"severity_number"`
920918
SeverityText string `json:"severity_text"`
921919
EventName string `json:"event_name"`
@@ -1145,8 +1143,6 @@ func TestEncodeLogOtelMode(t *testing.T) {
11451143
// helper function that creates the OTel LogRecord from the test structure
11461144
func createTestOTelLogRecord(t *testing.T, rec OTelRecord) (plog.LogRecord, pcommon.InstrumentationScope, pcommon.Resource) {
11471145
record := plog.NewLogRecord()
1148-
record.SetTimestamp(pcommon.Timestamp(uint64(rec.Timestamp.UnixNano()))) //nolint:gosec // this input is controlled by tests
1149-
record.SetObservedTimestamp(pcommon.Timestamp(uint64(rec.ObservedTimestamp.UnixNano()))) //nolint:gosec // this input is controlled by tests
11501146

11511147
record.SetTraceID(pcommon.TraceID(rec.TraceID))
11521148
record.SetSpanID(pcommon.SpanID(rec.SpanID))

‎exporter/elasticsearchexporter/pdata_serializer.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"bytes"
88
"encoding/hex"
99
"fmt"
10+
"strconv"
1011
"strings"
1112

1213
"github.com/elastic/go-structform"
@@ -431,7 +432,10 @@ func writeValue(v *json.Visitor, val pcommon.Value, stringifyMaps bool) {
431432

432433
func writeTimestampField(v *json.Visitor, key string, timestamp pcommon.Timestamp) {
433434
_ = v.OnKey(key)
434-
_ = v.OnString(timestamp.AsTime().UTC().Format(tsLayout))
435+
nsec := uint64(timestamp)
436+
msec := nsec / 1e6
437+
nsec -= msec * 1e6
438+
_ = v.OnString(strconv.FormatUint(msec, 10) + "." + strconv.FormatUint(nsec, 10))
435439
}
436440

437441
func writeUIntField(v *json.Visitor, key string, i uint64) {

‎exporter/elasticsearchexporter/pdata_serializer_test.go

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ func TestSerializeLog(t *testing.T) {
4040
resource.Attributes().PutEmptyMap("resource_map").PutStr("foo", "bar")
4141
scope.Attributes().PutEmptyMap("scope_map").PutStr("foo", "bar")
4242
}, wantErr: false, expected: map[string]any{
43-
"@timestamp": "1970-01-01T00:00:00.000000000Z",
44-
"observed_timestamp": "1970-01-01T00:00:00.000000000Z",
43+
"@timestamp": "0.0",
44+
"observed_timestamp": "0.0",
4545
"data_stream": map[string]any{
4646
"type": "logs",
4747
},
@@ -84,8 +84,8 @@ func TestSerializeLog(t *testing.T) {
8484
},
8585
wantErr: false,
8686
expected: map[string]any{
87-
"@timestamp": "1970-01-01T00:00:00.000000000Z",
88-
"observed_timestamp": "1970-01-01T00:00:00.000000000Z",
87+
"@timestamp": "0.0",
88+
"observed_timestamp": "0.0",
8989
"data_stream": map[string]any{},
9090
"resource": map[string]any{},
9191
"scope": map[string]any{},
@@ -101,8 +101,8 @@ func TestSerializeLog(t *testing.T) {
101101
},
102102
wantErr: false,
103103
expected: map[string]any{
104-
"@timestamp": "1970-01-01T00:00:00.000000000Z",
105-
"observed_timestamp": "1970-01-01T00:00:00.000000000Z",
104+
"@timestamp": "0.0",
105+
"observed_timestamp": "0.0",
106106
"data_stream": map[string]any{},
107107
"resource": map[string]any{},
108108
"scope": map[string]any{},
@@ -130,8 +130,8 @@ func TestSerializeLog(t *testing.T) {
130130
},
131131
wantErr: false,
132132
expected: map[string]any{
133-
"@timestamp": "1970-01-01T00:00:00.000000000Z",
134-
"observed_timestamp": "1970-01-01T00:00:00.000000000Z",
133+
"@timestamp": "0.0",
134+
"observed_timestamp": "0.0",
135135
"data_stream": map[string]any{},
136136
"resource": map[string]any{},
137137
"scope": map[string]any{},
@@ -148,6 +148,39 @@ func TestSerializeLog(t *testing.T) {
148148
},
149149
},
150150
},
151+
{
152+
name: "event_name takes precedent over attributes.event.name",
153+
logCustomizer: func(_ pcommon.Resource, _ pcommon.InstrumentationScope, record plog.LogRecord) {
154+
record.Attributes().PutStr("event.name", "foo")
155+
record.SetEventName("bar")
156+
},
157+
wantErr: false,
158+
expected: map[string]any{
159+
"@timestamp": "0.0",
160+
"observed_timestamp": "0.0",
161+
"event_name": "bar",
162+
"data_stream": map[string]any{},
163+
"resource": map[string]any{},
164+
"scope": map[string]any{},
165+
"attributes": map[string]any{
166+
"event.name": "foo",
167+
},
168+
},
169+
},
170+
{
171+
name: "timestamp",
172+
logCustomizer: func(_ pcommon.Resource, _ pcommon.InstrumentationScope, record plog.LogRecord) {
173+
record.SetTimestamp(1721314113467654123)
174+
},
175+
wantErr: false,
176+
expected: map[string]any{
177+
"@timestamp": "1721314113467.654123",
178+
"observed_timestamp": "0.0",
179+
"data_stream": map[string]any{},
180+
"resource": map[string]any{},
181+
"scope": map[string]any{},
182+
},
183+
},
151184
}
152185
for _, tt := range tests {
153186
t.Run(tt.name, func(t *testing.T) {
@@ -207,7 +240,7 @@ func TestSerializeMetricsConflict(t *testing.T) {
207240
assert.Equal(t, fmt.Errorf("metric with name 'foo' has already been serialized in document with timestamp 1970-01-01T00:00:00.000000000Z"), validationErrors[0])
208241

209242
assert.Equal(t, map[string]any{
210-
"@timestamp": "1970-01-01T00:00:00.000000000Z",
243+
"@timestamp": "0.0",
211244
"data_stream": map[string]any{},
212245
"resource": map[string]any{},
213246
"scope": map[string]any{},

0 commit comments

Comments
 (0)