Skip to content

Commit 15ddc29

Browse files
authored
[receiver/awsfirehosereceiver] Fix timestamp in cwlogs (#36122)
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue. Ex. Adding a feature - Explain what this achieves.--> #### Description When testing ingesting json format logs from CloudWatch log groups using Firehose, I see the timestamp field has been populated wrong. For example the timestamp for the log message I ingested should be `2024-10-23T21:28:49.707` but I'm getting a timestamp from `1970-01-01T00:28:49.707Z`. ``` { "cloud": { "account": { "id": "123456789012" } }, "agent": { "name": "otlp", "version": "unknown" }, "@timestamp": "1970-01-01T00:28:49.707Z", "log": {}, "data_stream": { "namespace": "default", "type": "logs", "dataset": "apm.app.unknown" }, "service": { "environment": "unset", "name": "unknown", "language": { "name": "unknown" } }, "event": {}, "message": "test-1", "labels": { "aws_cloudwatch_log_stream_name": "test-log-stream", "aws_cloudwatch_log_group_name": "test-cloudwatch-logs-ks" } } } ``` This issue is caused by `pcommon.Timestamp` is a time specified as UNIX Epoch time in nanoseconds but timestamp in cloudwatch logs are in milliseconds. So converting the timestamp from milliseconds to nanoseconds is needed. #### Testing Added unit test.
1 parent 022c83f commit 15ddc29

File tree

3 files changed

+52
-1
lines changed

3 files changed

+52
-1
lines changed

.chloggen/fix_timestamp.yaml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: bug_fix
3+
4+
# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
5+
component: awsfirehosereceiver
6+
7+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
8+
note: fix timestamp when ingesting logs from CloudWatch through firehose
9+
10+
# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
11+
issues: [36122]
12+
13+
# (Optional) One or more lines of additional information to render under the primary note.
14+
# These lines will be padded with 2 spaces and then inserted directly into the document.
15+
# Use pipe (|) for multiline entries.
16+
subtext:
17+
18+
# If your change doesn't affect end users or the exported elements of any package,
19+
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
20+
# Optional: The change log or logs in which this entry should be included.
21+
# e.g. '[user]' or '[user, api]'
22+
# Include 'user' if the change is relevant to end users.
23+
# Include 'api' if there is a change to a library API.
24+
# Default: '[user]'
25+
change_logs: [user]

receiver/awsfirehosereceiver/internal/unmarshaler/cwlog/logsbuilder.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
package cwlog // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver/internal/unmarshaler/cwlog"
55

66
import (
7+
"time"
8+
79
"go.opentelemetry.io/collector/pdata/pcommon"
810
"go.opentelemetry.io/collector/pdata/plog"
911
conventions "go.opentelemetry.io/collector/semconv/v1.6.1"
@@ -44,7 +46,9 @@ func newResourceLogsBuilder(logs plog.Logs, attrs resourceAttributes) *resourceL
4446
func (rlb *resourceLogsBuilder) AddLog(log cWLog) {
4547
for _, event := range log.LogEvents {
4648
logLine := rlb.rls.AppendEmpty()
47-
logLine.SetTimestamp(pcommon.Timestamp(event.Timestamp))
49+
// pcommon.Timestamp is a time specified as UNIX Epoch time in nanoseconds
50+
// but timestamp in cloudwatch logs are in milliseconds.
51+
logLine.SetTimestamp(pcommon.Timestamp(event.Timestamp * int64(time.Millisecond)))
4852
logLine.Body().SetStr(event.Message)
4953
}
5054
}

receiver/awsfirehosereceiver/internal/unmarshaler/cwlog/unmarshaler_test.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,3 +81,25 @@ func TestUnmarshal(t *testing.T) {
8181
})
8282
}
8383
}
84+
85+
func TestLogTimestamp(t *testing.T) {
86+
unmarshaler := NewUnmarshaler(zap.NewNop())
87+
record, err := os.ReadFile(filepath.Join(".", "testdata", "single_record"))
88+
require.NoError(t, err)
89+
90+
compressedRecord, err := compression.Zip(record)
91+
require.NoError(t, err)
92+
records := [][]byte{compressedRecord}
93+
94+
got, err := unmarshaler.Unmarshal(records)
95+
require.NoError(t, err)
96+
require.NotNil(t, got)
97+
require.Equal(t, 1, got.ResourceLogs().Len())
98+
99+
rm := got.ResourceLogs().At(0)
100+
require.Equal(t, 1, rm.ScopeLogs().Len())
101+
ilm := rm.ScopeLogs().At(0)
102+
ilm.LogRecords().At(0).Timestamp()
103+
expectedTimestamp := "2024-09-05 13:47:15.523 +0000 UTC"
104+
require.Equal(t, expectedTimestamp, ilm.LogRecords().At(0).Timestamp().String())
105+
}

0 commit comments

Comments
 (0)