[pkg/translator/azure] Allow timestamp to be used instead of time

.chloggen/azuretranslator-parse-timestamp-resource-logs.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: azuretranslatorpkg
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: When receiving data from Azure some data does not meet the Common Specifications when sending the timestamp. Allow the attribute timeStamp to be used as an alternative to the standard time.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [28806]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

pkg/translator/azure/resourcelogs_to_logs.go

@@ -6,6 +6,7 @@ package azure // import "github.com/open-telemetry/opentelemetry-collector-contr
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"strconv"
 
 	jsoniter "github.com/json-iterator/go"
@@ -36,6 +37,10 @@ const (
 	azureTenantID          = "azure.tenant.id"
 )
 
+var (
+	errMissingTimestamp = errors.New("missing timestamp")
+)
+
 // azureRecords represents an array of Azure log records
 // as exported via an Azure Event Hub
 type azureRecords struct {
@@ -47,6 +52,7 @@ type azureRecords struct {
 // https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/resource-logs-schema
 type azureLogRecord struct {
 	Time              string       `json:"time"`
+	Timestamp         string       `json:"timeStamp"`
 	ResourceID        string       `json:"resourceId"`
 	TenantID          *string      `json:"tenantId"`
 	OperationName     string       `json:"operationName"`
@@ -101,7 +107,7 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) {
 
 		for i := 0; i < len(logs); i++ {
 			log := logs[i]
-			nanos, err := asTimestamp(log.Time)
+			nanos, err := getTimestamp(log)
 			if err != nil {
 				r.Logger.Warn("Unable to convert timestamp from log", zap.String("timestamp", log.Time))
 				continue
@@ -125,6 +131,16 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) {
 	return l, nil
 }
 
+func getTimestamp(record azureLogRecord) (pcommon.Timestamp, error) {
+	if record.Time != "" {
+		return asTimestamp(record.Time)
+	} else if record.Timestamp != "" {
+		return asTimestamp(record.Timestamp)
+	}
+
+	return 0, errMissingTimestamp
+}
+
 // asTimestamp will parse an ISO8601 string into an OpenTelemetry
 // nanosecond timestamp. If the string cannot be parsed, it will
 // return zero and the error.
@@ -133,6 +149,7 @@ func asTimestamp(s string) (pcommon.Timestamp, error) {
 	if err != nil {
 		return 0, err
 	}
+
 	return pcommon.Timestamp(t.UnixNano()), nil
 }
 

pkg/translator/azure/resourcelogs_to_logs_test.go

@@ -167,6 +167,49 @@ var badLevelLogRecord = func() plog.LogRecord {
 	return lr
 }()
 
+var badTimeLogRecord = func() plog.LogRecord {
+	lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty()
+
+	ts, _ := asTimestamp("2021-10-14T22:17:11+00:00")
+	lr.SetTimestamp(ts)
+
+	lr.Attributes().PutStr(azureOperationName, "ApplicationGatewayAccess")
+	lr.Attributes().PutStr(azureCategory, "ApplicationGatewayAccessLog")
+	lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure)
+
+	m := lr.Attributes().PutEmptyMap(azureProperties)
+	m.PutStr("instanceId", "appgw_2")
+	m.PutStr("clientIP", "185.42.129.24")
+	m.PutDouble("clientPort", 45057)
+	m.PutStr("httpMethod", "GET")
+	m.PutStr("originalRequestUriWithArgs", "/")
+	m.PutStr("requestUri", "/")
+	m.PutStr("requestQuery", "")
+	m.PutStr("userAgent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36")
+	m.PutDouble("httpStatus", 200)
+	m.PutStr("httpVersion", "HTTP/1.1")
+	m.PutDouble("receivedBytes", 184)
+	m.PutDouble("sentBytes", 466)
+	m.PutDouble("clientResponseTime", 0)
+	m.PutDouble("timeTaken", 0.034)
+	m.PutStr("WAFEvaluationTime", "0.000")
+	m.PutStr("WAFMode", "Detection")
+	m.PutStr("transactionId", "592d1649f75a8d480a3c4dc6a975309d")
+	m.PutStr("sslEnabled", "on")
+	m.PutStr("sslCipher", "ECDHE-RSA-AES256-GCM-SHA384")
+	m.PutStr("sslProtocol", "TLSv1.2")
+	m.PutStr("sslClientVerify", "NONE")
+	m.PutStr("sslClientCertificateFingerprint", "")
+	m.PutStr("sslClientCertificateIssuerName", "")
+	m.PutStr("serverRouted", "52.239.221.65:443")
+	m.PutStr("serverStatus", "200")
+	m.PutStr("serverResponseLatency", "0.028")
+	m.PutStr("upstreamSourcePort", "21564")
+	m.PutStr("originalHost", "20.110.30.194")
+	m.PutStr("host", "20.110.30.194")
+	return lr
+}()
+
 func TestAsTimestamp(t *testing.T) {
 	timestamp := "2022-11-11T04:48:27.6767145Z"
 	nanos, err := asTimestamp(timestamp)
@@ -371,6 +414,15 @@ func TestUnmarshalLogs(t *testing.T) {
 	lr = scopeLogs.LogRecords().AppendEmpty()
 	badLevelLogRecord.CopyTo(lr)
 
+	expectedBadTime := plog.NewLogs()
+	resourceLogs = expectedBadTime.ResourceLogs().AppendEmpty()
+	resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID")
+	scopeLogs = resourceLogs.ScopeLogs().AppendEmpty()
+	scopeLogs.Scope().SetName("otelcol/azureresourcelogs")
+	scopeLogs.Scope().SetVersion(testBuildInfo.Version)
+	lr = scopeLogs.LogRecords().AppendEmpty()
+	badTimeLogRecord.CopyTo(lr)
+
 	tests := []struct {
 		file     string
 		expected plog.Logs
@@ -391,6 +443,10 @@ func TestUnmarshalLogs(t *testing.T) {
 			file:     "log-bad-level.json",
 			expected: expectedBadLevel,
 		},
+		{
+			file:     "log-bad-time.json",
+			expected: expectedBadTime,
+		},
 	}
 
 	sut := &ResourceLogsUnmarshaler{

pkg/translator/azure/testdata/log-bad-time.json

@@ -0,0 +1,45 @@
+{
+    "records": [
+        {
+            "timeStamp": "2021-10-14T22:17:11+00:00",
+            "resourceId": "/RESOURCE_ID",
+            "listenerName": "HTTP-Listener",
+            "ruleName": "Storage-Static-Rule",
+            "backendPoolName": "StaticStorageAccount",
+            "backendSettingName": "StorageStatic-HTTPS-Setting",
+            "operationName": "ApplicationGatewayAccess",
+            "category": "ApplicationGatewayAccessLog",
+            "properties": {
+                "instanceId": "appgw_2",
+                "clientIP": "185.42.129.24",
+                "clientPort": 45057,
+                "httpMethod": "GET",
+                "originalRequestUriWithArgs": "\/",
+                "requestUri": "\/",
+                "requestQuery": "",
+                "userAgent": "Mozilla\/5.0 (Windows NT 6.1; WOW64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/52.0.2743.116 Safari\/537.36",
+                "httpStatus": 200,
+                "httpVersion": "HTTP\/1.1",
+                "receivedBytes": 184,
+                "sentBytes": 466,
+                "clientResponseTime": 0,
+                "timeTaken": 0.034,
+                "WAFEvaluationTime": "0.000",
+                "WAFMode": "Detection",
+                "transactionId": "592d1649f75a8d480a3c4dc6a975309d",
+                "sslEnabled": "on",
+                "sslCipher": "ECDHE-RSA-AES256-GCM-SHA384",
+                "sslProtocol": "TLSv1.2",
+                "sslClientVerify": "NONE",
+                "sslClientCertificateFingerprint": "",
+                "sslClientCertificateIssuerName": "",
+                "serverRouted": "52.239.221.65:443",
+                "serverStatus": "200",
+                "serverResponseLatency": "0.028",
+                "upstreamSourcePort": "21564",
+                "originalHost": "20.110.30.194",
+                "host": "20.110.30.194"
+            }
+        }
+    ]
+}

receiver/azureeventhubreceiver/README.md

@@ -95,10 +95,11 @@ and the OpenTelemetry attributes.
 | resultSignature (optional)       | azure.result.signature (attribute)     | 
 | resultType (optional)            | azure.result.type (attribute)          | 
 | tenantId (required, tenant logs) | azure.tenant.id (attribute)            | 
-| time (required)                  | time_unix_nano (field)                 | 
+| time or timeStamp (required)     | time_unix_nano (time takes precedence) |
 | identity (optional)              | azure.identity (attribute, nested)     |
 
-Note: JSON does not distinguish between fixed and floating point numbers. All
+Notes:
+* JSON does not distinguish between fixed and floating point numbers. All
 JSON numbers are encoded as doubles.
 
 For Metrics the Azure Metric Records are an array