Skip to content

Commit 011dd09

Browse files
LucianoGiannottiatoulmedehaansaandrzej-stencelmar4uk
authored andcommitted
[exporter/awscloudwatchlogsexporter] added placeholder names for loggroup and logstream (open-telemetry#37297)
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue. Ex. Adding a feature - Explain what this achieves.--> #### Description Add dynamic log_group_name and log_group_stream naming, based on [awsemfexporter](https://github.com/LucianoGiannotti/opentelemetry-collector-contrib/tree/main/exporter/awsemfexporter) <!-- Issue number (e.g. open-telemetry#1234) or full URL to issue, if applicable. --> #### Link to tracking issue Fixes [open-telemetry#31382](open-telemetry#31382) <!--Describe what testing was performed and which tests were added.--> #### Testing Similar testing as on awsemfexporter, testing that names are replaced properly before pushing logs to CW. <!--Please delete paragraphs that you did not use before submitting.--> --------- Signed-off-by: Paschalis Tsilias <[email protected]> Signed-off-by: Matthieu MOREL <[email protected]> Signed-off-by: Jared Tan <[email protected]> Signed-off-by: Michael Shen <[email protected]> Signed-off-by: Dani Louca <[email protected]> Signed-off-by: Célian Garcia <[email protected]> Signed-off-by: ChrsMark <[email protected]> Signed-off-by: Marc Lopez Rubio <[email protected]> Signed-off-by: Murphy Chen <[email protected]> Co-authored-by: Antoine Toulme <[email protected]> Co-authored-by: Sam DeHaan <[email protected]> Co-authored-by: Andrzej Stencel <[email protected]> Co-authored-by: Irina <[email protected]> Co-authored-by: Curtis Robert <[email protected]> Co-authored-by: Constança Manteigas <[email protected]> Co-authored-by: Paschalis Tsilias <[email protected]> Co-authored-by: Evan Bradley <[email protected]> Co-authored-by: Evan Bradley <[email protected]> Co-authored-by: Matthieu MOREL <[email protected]> Co-authored-by: Adriel Perkins <[email protected]> Co-authored-by: Sean Marciniak <[email protected]> Co-authored-by: Nikos Angelopoulos <[email protected]> Co-authored-by: Antoine Toulme <[email protected]> Co-authored-by: Alex Boten <[email protected]> Co-authored-by: mcube8 <[email protected]> Co-authored-by: Will Li <[email protected]> Co-authored-by: Luke Gao <[email protected]> Co-authored-by: Andrew Wilkins <[email protected]> Co-authored-by: Jared Tan <[email protected]> Co-authored-by: Alex Van Boxel <[email protected]> Co-authored-by: Michael Shen <[email protected]> Co-authored-by: Roger Coll <[email protected]> Co-authored-by: Braydon Kains <[email protected]> Co-authored-by: Dani Louca <[email protected]> Co-authored-by: wojtekzyla <[email protected]> Co-authored-by: Célian GARCIA <[email protected]> Co-authored-by: Murphy Chen <[email protected]> Co-authored-by: Anthony Yeh <[email protected]> Co-authored-by: Christos Markou <[email protected]> Co-authored-by: Yang Song <[email protected]> Co-authored-by: Vihas Makwana <[email protected]> Co-authored-by: Daniel Jaglowski <[email protected]> Co-authored-by: Khurshidali Shiekh <[email protected]> Co-authored-by: Cyril Cressent <[email protected]> Co-authored-by: Marc Lopez Rubio <[email protected]> Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: opentelemetrybot <[email protected]> Co-authored-by: Stuart Buckingham <[email protected]> Co-authored-by: Anthony Mirabella <[email protected]> Co-authored-by: Jorge Creixell <[email protected]> Co-authored-by: Robert Lankford <[email protected]> Co-authored-by: Paulo Janotti <[email protected]> Co-authored-by: Ludovic Fernandez <[email protected]> Co-authored-by: Vishal Raj <[email protected]> Co-authored-by: Gregor Jasny <[email protected]> Co-authored-by: Pablo Baeyens <[email protected]>
1 parent c22ad59 commit 011dd09

File tree

6 files changed

+392
-4
lines changed

6 files changed

+392
-4
lines changed
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Use this changelog template to create an entry for release notes.
2+
3+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
4+
change_type: enhancement
5+
6+
# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
7+
component: awscloudwatchlogsexporter
8+
9+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
10+
note: Add dynamic log_group_name and log_group_stream naming, based on awsemfexporter
11+
12+
# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
13+
issues: [31382]
14+
15+
# (Optional) One or more lines of additional information to render under the primary note.
16+
# These lines will be padded with 2 spaces and then inserted directly into the document.
17+
# Use pipe (|) for multiline entries.
18+
subtext:
19+
20+
# If your change doesn't affect end users or the exported elements of any package,
21+
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
22+
# Optional: The change log or logs in which this entry should be included.
23+
# e.g. '[user]' or '[user, api]'
24+
# Include 'user' if the change is relevant to end users.
25+
# Include 'api' if there is a change to a library API.
26+
# Default: '[user]'
27+
change_logs: [user]

exporter/awscloudwatchlogsexporter/README.md

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,21 @@ NOTE: OpenTelemetry Logging support is experimental, hence this exporter is subj
2222

2323
The following settings are required:
2424

25-
- `log_group_name`: The group name of the CloudWatch Logs. If it does not exist it will be created automatically.
26-
- `log_stream_name`: The stream name of the CloudWatch Logs. If it does not exist it will be created automatically.
25+
26+
- `log_group_name`: The group name of the CloudWatch Logs. If it does not exist it will be created automatically. It supports several placeholder names. One valid example is `/aws/metrics/{ClusterName}`. It will search for ClusterName (or aws.ecs.cluster.name) resource attribute in the metrics data and replace with the actual cluster name. If none of them are found in the resource attribute map, {ClusterName} will be replaced by undefined. Similar way, for the {TaskId}, it searches for TaskId (or aws.ecs.task.id) key in the resource attribute map. For {NodeName}, it searches for NodeName (or k8s.node.name)
27+
- List of valid placeholders:
28+
- `{ClusterName}`: `aws.ecs.cluster.name`
29+
- `{TaskId}`: `aws.ecs.task.id`
30+
- `{NodeName}`: `k8s.node.name`
31+
- `{PodName}`: `pod`
32+
- `{ServiceName}`: `service.name`
33+
- `{ContainerInstanceId}`: `aws.ecs.container.instance.id`
34+
- `{TaskDefinitionFamily}`: `aws.ecs.task.family`
35+
- `{InstanceId}`: `service.instance.id`
36+
- `{FaasName}`: `faas.name`
37+
- `{FaasVersion}`: `faas.version`
38+
- `log_stream_name`: The stream name of the CloudWatch Logs. If it does not exist it will be created automatically. It supports the same placeholders as `log_group_name`
39+
2740

2841
The following settings can be optionally configured:
2942

exporter/awscloudwatchlogsexporter/config.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,13 @@ func (config *Config) Validate() error {
6565
return errors.New("'log_stream_name' must be set")
6666
}
6767

68+
if isPatternValid, invalidPattern := (isPatternValid(config.LogGroupName)); !isPatternValid {
69+
return errors.New("'log_group_name' has an invalid pattern between curly brackets: " + invalidPattern)
70+
}
71+
if isPatternValid, invalidPattern := (isPatternValid(config.LogStreamName)); !isPatternValid {
72+
return errors.New("'log_stream_name' has an invalid pattern between curly brackets: " + invalidPattern)
73+
}
74+
6875
if err := config.QueueSettings.Validate(); err != nil {
6976
return err
7077
}

exporter/awscloudwatchlogsexporter/exporter.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,8 @@ type cwLogBody struct {
179179
func logToCWLog(resourceAttrs map[string]any, scope pcommon.InstrumentationScope, log plog.LogRecord, config *Config) (*cwlogs.Event, error) {
180180
// TODO(jbd): Benchmark and improve the allocations.
181181
// Evaluate go.elastic.co/fastjson as a replacement for encoding/json.
182-
logGroupName := config.LogGroupName
183-
logStreamName := config.LogStreamName
182+
// Replace loggroup and logstream with resource attribute
183+
logGroupName, logStreamName, _ := getLogInfo(resourceAttrs, config)
184184

185185
var bodyJSON []byte
186186
var err error
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// Copyright The OpenTelemetry Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package awscloudwatchlogsexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter"
5+
6+
import (
7+
"fmt"
8+
"regexp"
9+
"strconv"
10+
"strings"
11+
12+
"go.uber.org/zap"
13+
)
14+
15+
var patternKeyToAttributeMap = map[string]string{
16+
"ClusterName": "aws.ecs.cluster.name",
17+
"TaskId": "aws.ecs.task.id",
18+
"NodeName": "k8s.node.name",
19+
"PodName": "pod",
20+
"ServiceName": "service.name",
21+
"ContainerInstanceId": "aws.ecs.container.instance.id",
22+
"TaskDefinitionFamily": "aws.ecs.task.family",
23+
"InstanceId": "service.instance.id",
24+
"FaasName": "faas.name",
25+
"FaasVersion": "faas.version",
26+
}
27+
28+
func isPatternValid(s string) (bool, string) {
29+
if !strings.Contains(s, "{") && !strings.Contains(s, "}") {
30+
return true, ""
31+
}
32+
33+
re := regexp.MustCompile(`\{([^{}]*)\}`)
34+
matches := re.FindAllStringSubmatch(s, -1)
35+
36+
for _, match := range matches {
37+
if len(match) > 1 {
38+
key := match[1]
39+
if _, exists := patternKeyToAttributeMap[key]; !exists {
40+
return false, key
41+
}
42+
}
43+
}
44+
return true, ""
45+
}
46+
47+
func replacePatterns(s string, attrMap map[string]string, logger *zap.Logger) (string, bool) {
48+
success := true
49+
var foundAndReplaced bool
50+
for key := range patternKeyToAttributeMap {
51+
s, foundAndReplaced = replacePatternWithAttrValue(s, key, attrMap, logger)
52+
success = success && foundAndReplaced
53+
}
54+
return s, success
55+
}
56+
57+
func replacePatternWithAttrValue(s, patternKey string, attrMap map[string]string, logger *zap.Logger) (string, bool) {
58+
pattern := "{" + patternKey + "}"
59+
if strings.Contains(s, pattern) {
60+
if value, ok := attrMap[patternKey]; ok {
61+
return replace(s, pattern, value, logger)
62+
} else if value, ok := attrMap[patternKeyToAttributeMap[patternKey]]; ok {
63+
return replace(s, pattern, value, logger)
64+
}
65+
logger.Debug("No resource attribute found for pattern " + pattern)
66+
return strings.ReplaceAll(s, pattern, "undefined"), false
67+
}
68+
return s, true
69+
}
70+
71+
func replace(s, pattern string, value string, logger *zap.Logger) (string, bool) {
72+
if value == "" {
73+
logger.Debug("Empty resource attribute value found for pattern " + pattern)
74+
return strings.ReplaceAll(s, pattern, "undefined"), false
75+
}
76+
return strings.ReplaceAll(s, pattern, value), true
77+
}
78+
79+
// getLogInfo retrieves the log group and log stream names from a given set of metrics.
80+
func getLogInfo(resourceAttrs map[string]any, config *Config) (string, string, bool) {
81+
var logGroup, logStream string
82+
groupReplaced := true
83+
streamReplaced := true
84+
85+
// Convert to map[string]string
86+
strAttributeMap := anyMapToStringMap(resourceAttrs)
87+
88+
// Override log group/stream if specified in config. However, in this case, customer won't have correlation experience
89+
if len(config.LogGroupName) > 0 {
90+
logGroup, groupReplaced = replacePatterns(config.LogGroupName, strAttributeMap, config.logger)
91+
}
92+
if len(config.LogStreamName) > 0 {
93+
logStream, streamReplaced = replacePatterns(config.LogStreamName, strAttributeMap, config.logger)
94+
}
95+
96+
return logGroup, logStream, (groupReplaced && streamReplaced)
97+
}
98+
99+
func anyMapToStringMap(resourceAttrs map[string]any) map[string]string {
100+
strMap := make(map[string]string)
101+
for key, value := range resourceAttrs {
102+
switch v := value.(type) {
103+
case string:
104+
strMap[key] = v
105+
case int:
106+
strMap[key] = strconv.Itoa(v)
107+
case bool:
108+
strMap[key] = strconv.FormatBool(v)
109+
default:
110+
strMap[key] = fmt.Sprintf("%v", v)
111+
}
112+
}
113+
return strMap
114+
}

0 commit comments

Comments
 (0)