Skip to content

Commit 701fe2b

Browse files
karakayasemikarakayasemi
committed
[processor/redaction] apply redaction to log.body (open-telemetry#37239)
1 parent 8522e31 commit 701fe2b

File tree

3 files changed

+423
-116
lines changed

3 files changed

+423
-116
lines changed

‎.chloggen/redact-log-body.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Use this changelog template to create an entry for release notes.
2+
3+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
4+
change_type: enhancement
5+
6+
# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
7+
component: processor/redaction
8+
9+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
10+
note: "Apply redaction to log.body"
11+
12+
# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
13+
issues: [37239]
14+
15+
# (Optional) One or more lines of additional information to render under the primary note.
16+
# These lines will be padded with 2 spaces and then inserted directly into the document.
17+
# Use pipe (|) for multiline entries.
18+
subtext:
19+
20+
# If your change doesn't affect end users or the exported elements of any package,
21+
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
22+
# Optional: The change log or logs in which this entry should be included.
23+
# e.g. '[user]' or '[user, api]'
24+
# Include 'user' if the change is relevant to end users.
25+
# Include 'api' if there is a change to a library API.
26+
# Default: '[user]'
27+
change_logs: []

‎processor/redactionprocessor/processor.go

Lines changed: 183 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,102 @@ func (s *redaction) processResourceLog(ctx context.Context, rl plog.ResourceLogs
144144
for k := 0; k < ils.LogRecords().Len(); k++ {
145145
log := ils.LogRecords().At(k)
146146
s.processAttrs(ctx, log.Attributes())
147+
s.processLogBody(ctx, log.Body(), log.Attributes())
148+
}
149+
}
150+
}
151+
152+
func (s *redaction) processLogBody(ctx context.Context, body pcommon.Value, attributes pcommon.Map) {
153+
var redactedKeys, maskedKeys, allowedKeys, ignoredKeys []string
154+
155+
switch body.Type() {
156+
case pcommon.ValueTypeMap:
157+
var redactedBodyKeys []string
158+
body.Map().Range(func(k string, v pcommon.Value) bool {
159+
if s.shouldIgnoreKey(k) {
160+
ignoredKeys = append(ignoredKeys, k)
161+
return true
162+
}
163+
if s.shouldRedactKey(k) {
164+
redactedBodyKeys = append(redactedBodyKeys, k)
165+
return true
166+
}
167+
if s.shouldMaskKey(k) {
168+
maskedKeys = append(maskedKeys, k)
169+
v.SetStr(s.maskValue(v.Str(), regexp.MustCompile(".*")))
170+
return true
171+
}
172+
s.redactLogBodyRecursive(ctx, k, v, &redactedKeys, &maskedKeys, &allowedKeys, &ignoredKeys)
173+
return true
174+
})
175+
for _, k := range redactedBodyKeys {
176+
body.Map().Remove(k)
177+
redactedKeys = append(redactedKeys, k)
178+
}
179+
case pcommon.ValueTypeSlice:
180+
for i := 0; i < body.Slice().Len(); i++ {
181+
s.redactLogBodyRecursive(ctx, fmt.Sprintf("[%d]", i), body.Slice().At(i), &redactedKeys, &maskedKeys, &allowedKeys, &ignoredKeys)
182+
}
183+
default:
184+
strVal := body.Str()
185+
if s.shouldAllowValue(strVal) {
186+
allowedKeys = append(allowedKeys, "body")
187+
return
188+
}
189+
processedValue := s.processStringValue(strVal)
190+
if strVal != processedValue {
191+
maskedKeys = append(maskedKeys, "body")
192+
body.SetStr(processedValue)
193+
}
194+
}
195+
196+
s.addMetaAttrs(redactedKeys, attributes, redactionBodyRedactedKeys, redactionBodyRedactedCount)
197+
s.addMetaAttrs(maskedKeys, attributes, redactionBodyMaskedKeys, redactionBodyMaskedCount)
198+
s.addMetaAttrs(allowedKeys, attributes, redactionBodyAllowedKeys, redactionBodyAllowedCount)
199+
s.addMetaAttrs(ignoredKeys, attributes, "", redactionBodyIgnoredCount)
200+
}
201+
202+
func (s *redaction) redactLogBodyRecursive(ctx context.Context, key string, value pcommon.Value, redactedKeys, maskedKeys, allowedKeys, ignoredKeys *[]string) {
203+
switch value.Type() {
204+
case pcommon.ValueTypeMap:
205+
var redactedCurrentValueKeys []string
206+
value.Map().Range(func(k string, v pcommon.Value) bool {
207+
keyWithPath := fmt.Sprintf("%s.%s", key, k)
208+
if s.shouldIgnoreKey(k) {
209+
*ignoredKeys = append(*ignoredKeys, keyWithPath)
210+
return true
211+
}
212+
if s.shouldRedactKey(k) {
213+
redactedCurrentValueKeys = append(redactedCurrentValueKeys, k)
214+
return true
215+
}
216+
if s.shouldMaskKey(k) {
217+
*maskedKeys = append(*maskedKeys, keyWithPath)
218+
value.SetStr(s.maskValue(value.Str(), regexp.MustCompile(".*")))
219+
return true
220+
}
221+
s.redactLogBodyRecursive(ctx, keyWithPath, v, redactedKeys, maskedKeys, allowedKeys, ignoredKeys)
222+
return true
223+
})
224+
for _, k := range redactedCurrentValueKeys {
225+
value.Map().Remove(k)
226+
keyWithPath := fmt.Sprintf("%s.%s", key, k)
227+
*redactedKeys = append(*redactedKeys, keyWithPath)
228+
}
229+
case pcommon.ValueTypeSlice:
230+
for i := 0; i < value.Slice().Len(); i++ {
231+
s.redactLogBodyRecursive(ctx, fmt.Sprintf("%s.[%d]", key, i), value.Slice().At(i), redactedKeys, maskedKeys, allowedKeys, ignoredKeys)
232+
}
233+
default:
234+
strVal := value.Str()
235+
if s.shouldAllowValue(strVal) {
236+
*allowedKeys = append(*allowedKeys, key)
237+
return
238+
}
239+
processedValue := s.processStringValue(strVal)
240+
if strVal != processedValue {
241+
*maskedKeys = append(*maskedKeys, key)
242+
value.SetStr(processedValue)
147243
}
148244
}
149245
}
@@ -192,10 +288,7 @@ func (s *redaction) processResourceMetric(ctx context.Context, rm pmetric.Resour
192288
// processAttrs redacts the attributes of a resource span or a span
193289
func (s *redaction) processAttrs(_ context.Context, attributes pcommon.Map) {
194290
// TODO: Use the context for recording metrics
195-
var toDelete []string
196-
var toBlock []string
197-
var allowed []string
198-
var ignoring []string
291+
var redactedKeys, maskedKeys, allowedKeys, ignoredKeys []string
199292

200293
// Identify attributes to redact and mask in the following sequence
201294
// 1. Make a list of attribute keys to redact
@@ -206,67 +299,42 @@ func (s *redaction) processAttrs(_ context.Context, attributes pcommon.Map) {
206299
// - Only range through all attributes once
207300
// - Don't mask any values if the whole attribute is slated for deletion
208301
attributes.Range(func(k string, value pcommon.Value) bool {
209-
// don't delete or redact the attribute if it should be ignored
210-
if _, ignored := s.ignoreList[k]; ignored {
211-
ignoring = append(ignoring, k)
212-
// Skip to the next attribute
302+
if s.shouldIgnoreKey(k) {
303+
ignoredKeys = append(ignoredKeys, k)
213304
return true
214305
}
215-
216-
// Make a list of attribute keys to redact
217-
if !s.config.AllowAllKeys {
218-
if _, allowed := s.allowList[k]; !allowed {
219-
toDelete = append(toDelete, k)
220-
// Skip to the next attribute
221-
return true
222-
}
306+
if s.shouldRedactKey(k) {
307+
redactedKeys = append(redactedKeys, k)
308+
return true
223309
}
224-
225310
strVal := value.Str()
226-
// Allow any values matching the allowed list regex
227-
for _, compiledRE := range s.allowRegexList {
228-
if match := compiledRE.MatchString(strVal); match {
229-
allowed = append(allowed, k)
230-
return true
231-
}
311+
if s.shouldAllowValue(strVal) {
312+
allowedKeys = append(allowedKeys, k)
313+
return true
232314
}
233-
234-
// Mask any blocked keys for the other attributes
235-
for _, compiledRE := range s.blockKeyRegexList {
236-
if match := compiledRE.MatchString(k); match {
237-
toBlock = append(toBlock, k)
238-
maskedValue := s.maskValue(strVal, regexp.MustCompile(".*"))
239-
value.SetStr(maskedValue)
240-
return true
241-
}
315+
if s.shouldMaskKey(k) {
316+
maskedKeys = append(maskedKeys, k)
317+
maskedValue := s.maskValue(strVal, regexp.MustCompile(".*"))
318+
value.SetStr(maskedValue)
319+
return true
242320
}
243-
244-
// Mask any blocked values for the other attributes
245-
var matched bool
246-
for _, compiledRE := range s.blockRegexList {
247-
if compiledRE.MatchString(strVal) {
248-
if !matched {
249-
matched = true
250-
toBlock = append(toBlock, k)
251-
}
252-
253-
maskedValue := s.maskValue(strVal, compiledRE)
254-
value.SetStr(maskedValue)
255-
strVal = maskedValue
256-
}
321+
processedString := s.processStringValue(strVal)
322+
if processedString != strVal {
323+
maskedKeys = append(maskedKeys, k)
324+
value.SetStr(processedString)
257325
}
258326
return true
259327
})
260328

261329
// Delete the attributes on the redaction list
262-
for _, k := range toDelete {
330+
for _, k := range redactedKeys {
263331
attributes.Remove(k)
264332
}
265333
// Add diagnostic information to the span
266-
s.addMetaAttrs(toDelete, attributes, redactedKeys, redactedKeyCount)
267-
s.addMetaAttrs(toBlock, attributes, maskedValues, maskedValueCount)
268-
s.addMetaAttrs(allowed, attributes, allowedValues, allowedValueCount)
269-
s.addMetaAttrs(ignoring, attributes, "", ignoredKeyCount)
334+
s.addMetaAttrs(redactedKeys, attributes, redactionRedactedKeys, redactionRedactedCount)
335+
s.addMetaAttrs(maskedKeys, attributes, redactionMaskedKeys, redactionMaskedCount)
336+
s.addMetaAttrs(allowedKeys, attributes, redactionAllowedKeys, redactionAllowedCount)
337+
s.addMetaAttrs(ignoredKeys, attributes, "", redactionIgnoredCount)
270338
}
271339

272340
//nolint:gosec
@@ -314,16 +382,70 @@ func (s *redaction) addMetaAttrs(redactedAttrs []string, attributes pcommon.Map,
314382
}
315383
}
316384

385+
func (s *redaction) processStringValue(strVal string) string {
386+
// Mask any blocked values for the other attributes
387+
for _, compiledRE := range s.blockRegexList {
388+
match := compiledRE.MatchString(strVal)
389+
if match {
390+
strVal = s.maskValue(strVal, compiledRE)
391+
}
392+
}
393+
return strVal
394+
}
395+
396+
func (s *redaction) shouldMaskKey(k string) bool {
397+
// Mask any blocked keys for the other attributes
398+
for _, compiledRE := range s.blockKeyRegexList {
399+
if match := compiledRE.MatchString(k); match {
400+
return true
401+
}
402+
}
403+
return false
404+
}
405+
406+
func (s *redaction) shouldAllowValue(strVal string) bool {
407+
// Allow any values matching the allowed list regex
408+
for _, compiledRE := range s.allowRegexList {
409+
if match := compiledRE.MatchString(strVal); match {
410+
return true
411+
}
412+
}
413+
return false
414+
}
415+
416+
func (s *redaction) shouldIgnoreKey(k string) bool {
417+
if _, ignored := s.ignoreList[k]; ignored {
418+
return true
419+
}
420+
return false
421+
}
422+
423+
func (s *redaction) shouldRedactKey(k string) bool {
424+
if !s.config.AllowAllKeys {
425+
if _, found := s.allowList[k]; !found {
426+
return true
427+
}
428+
}
429+
return false
430+
}
431+
317432
const (
318-
debug = "debug"
319-
info = "info"
320-
redactedKeys = "redaction.redacted.keys"
321-
redactedKeyCount = "redaction.redacted.count"
322-
maskedValues = "redaction.masked.keys"
323-
maskedValueCount = "redaction.masked.count"
324-
allowedValues = "redaction.allowed.keys"
325-
allowedValueCount = "redaction.allowed.count"
326-
ignoredKeyCount = "redaction.ignored.count"
433+
debug = "debug"
434+
info = "info"
435+
redactionRedactedKeys = "redaction.redacted.keys"
436+
redactionRedactedCount = "redaction.redacted.count"
437+
redactionMaskedKeys = "redaction.masked.keys"
438+
redactionMaskedCount = "redaction.masked.count"
439+
redactionAllowedKeys = "redaction.allowed.keys"
440+
redactionAllowedCount = "redaction.allowed.count"
441+
redactionIgnoredCount = "redaction.ignored.count"
442+
redactionBodyRedactedKeys = "redaction.body.redacted.keys"
443+
redactionBodyRedactedCount = "redaction.body.redacted.count"
444+
redactionBodyMaskedKeys = "redaction.body.masked.keys"
445+
redactionBodyMaskedCount = "redaction.body.masked.count"
446+
redactionBodyAllowedKeys = "redaction.body.allowed.keys"
447+
redactionBodyAllowedCount = "redaction.body.allowed.count"
448+
redactionBodyIgnoredCount = "redaction.body.ignored.count"
327449
)
328450

329451
// makeAllowList sets up a lookup table of allowed span attribute keys
@@ -338,7 +460,7 @@ func makeAllowList(c *Config) map[string]string {
338460
// span attributes (e.g. `notes`, `description`), then it will those
339461
// attribute keys in `redaction.masked.keys` and set the
340462
// `redaction.masked.count` to 2
341-
redactionKeys := []string{redactedKeys, redactedKeyCount, maskedValues, maskedValueCount, ignoredKeyCount}
463+
redactionKeys := []string{redactionRedactedKeys, redactionRedactedCount, redactionMaskedKeys, redactionMaskedCount, redactionIgnoredCount}
342464
// allowList consists of the keys explicitly allowed by the configuration
343465
// as well as of the new span attributes that the processor creates to
344466
// summarize its changes

0 commit comments

Comments
 (0)