Feature/mask query 3 (#506)

* mask query

* mask query

* mask query

* mask query

* mask query

* mask query

---------

Co-authored-by: k.torgaev <[email protected]>

Author: kasimtj

config/config.go

@@ -64,6 +64,8 @@ type Config struct {
 	// Whether to print debug logs
 	LogDebug bool `yaml:"log_debug,omitempty"`
 
+	LogMasks []LogMask `yaml:"log_masks,omitempty"`
+
 	// Whether to ignore security warnings
 	HackMePlease bool `yaml:"hack_me_please,omitempty"`
 
@@ -1216,3 +1218,8 @@ func (c Config) checkVulnerabilities() error {
 	}
 	return nil
 }
+
+type LogMask struct {
+	Regex       string `yaml:"regex"`
+	Replacement string `yaml:"replacement"`
+}

config/config_test.go

@@ -93,6 +93,13 @@ var fullConfig = Config{
 	},
 	LogDebug: true,
 
+	LogMasks: []LogMask{
+		{
+			Regex:       `(s3\(\s*'(?:(?:\\'|[^'])*)'\s*,\s*'(?:(?:\\'|[^'])*)'\s*,\s*')((?:\\'|[^'])*)(')`,
+			Replacement: "$1******$3",
+		},
+	},
+
 	Clusters: []Cluster{
 		{
 			Name:   "first cluster",
@@ -879,6 +886,9 @@ users:
   - 1.2.3.0/24
   deny_https: true
 log_debug: true
+log_masks:
+- regex: (s3\(\s*'(?:(?:\\'|[^'])*)'\s*,\s*'(?:(?:\\'|[^'])*)'\s*,\s*')((?:\\'|[^'])*)(')
+  replacement: $1******$3
 hack_me_please: true
 network_groups:
 - name: office

config/testdata/full.yml

@@ -3,6 +3,11 @@
 # By default debug logs are disabled.
 log_debug: true
 
+# Optional log masks, for example replace s3('', '', 'secret', '') -> s3('', '', '******', '')
+log_masks:
+  - regex: (s3\(\s*'(?:(?:\\'|[^'])*)'\s*,\s*'(?:(?:\\'|[^'])*)'\s*,\s*')((?:\\'|[^'])*)(')
+    replacement: $1******$3
+
 # Whether to ignore security checks during config parsing.
 #
 # By default security checks are enabled.

docs/src/content/docs/cn/index.md

@@ -352,6 +352,11 @@ caches:
 # By default debug logs are disabled.
 log_debug: true
 
+# Optional log masks, for example replace s3('', '', 'secret', '') -> s3('', '', '******', '')
+log_masks:
+  - regex: (s3\(\s*'(?:(?:\\'|[^'])*)'\s*,\s*'(?:(?:\\'|[^'])*)'\s*,\s*')((?:\\'|[^'])*)(')
+    replacement: $1******$3
+
 # Whether to ignore security checks during config parsing.
 #
 # By default security checks are enabled.

docs/src/content/docs/configuration/default.md

@@ -15,6 +15,11 @@ Example of [full](https://github.com/contentsquare/chproxy/blob/master/config/te
 # By default debug logs are disabled.
 log_debug: true
 
+# Optional log masks, for example replace s3('', '', 'secret', '') -> s3('', '', '******', '')
+log_masks:
+  - regex: (s3\(\s*'(?:(?:\\'|[^'])*)'\s*,\s*'(?:(?:\\'|[^'])*)'\s*,\s*')((?:\\'|[^'])*)(')
+    replacement: $1******$3
+
 # Whether to ignore security checks during config parsing.
 #
 # By default security checks are enabled.

log/log.go

@@ -5,7 +5,10 @@ import (
 	"io"
 	"log"
 	"os"
+	"regexp"
 	"sync/atomic"
+
+	"github.com/contentsquare/chproxy/config"
 )
 
 var (
@@ -52,30 +55,58 @@ func Debugf(format string, args ...interface{}) {
 		return
 	}
 	s := fmt.Sprintf(format, args...)
-	debugLogger.Output(outputCallDepth, s) // nolint
+	debugLogger.Output(outputCallDepth, mask(s)) // nolint
 }
 
 // Infof prints info message according to a format
 func Infof(format string, args ...interface{}) {
 	s := fmt.Sprintf(format, args...)
-	infoLogger.Output(outputCallDepth, s) // nolint
+	infoLogger.Output(outputCallDepth, mask(s)) // nolint
 }
 
 // Errorf prints warning message according to a format
 func Errorf(format string, args ...interface{}) {
 	s := fmt.Sprintf(format, args...)
-	errorLogger.Output(outputCallDepth, s) // nolint
+	errorLogger.Output(outputCallDepth, mask(s)) // nolint
 }
 
 // ErrorWithCallDepth prints err into error log using the given callDepth.
 func ErrorWithCallDepth(err error, callDepth int) {
 	s := err.Error()
-	errorLogger.Output(outputCallDepth+callDepth, s) //nolint
+	errorLogger.Output(outputCallDepth+callDepth, mask(s)) //nolint
 }
 
 // Fatalf prints fatal message according to a format and exits program
 func Fatalf(format string, args ...interface{}) {
 	s := fmt.Sprintf(format, args...)
-	fatalLogger.Output(outputCallDepth, s) // nolint
+	fatalLogger.Output(outputCallDepth, mask(s)) // nolint
 	os.Exit(1)
 }
+
+type regexReplacer struct {
+	regex       *regexp.Regexp
+	replacement string
+}
+
+var replacers []regexReplacer
+
+func InitReplacer(logMasks []config.LogMask) error {
+	for _, logMask := range logMasks {
+		re, err := regexp.Compile(logMask.Regex)
+		if err != nil {
+			return fmt.Errorf("error compiling regex %s: %w", logMask.Regex, err)
+		}
+		replacers = append(replacers, regexReplacer{
+			regex:       re,
+			replacement: logMask.Replacement,
+		})
+	}
+	return nil
+}
+
+func mask(s string) string {
+	for _, r := range replacers {
+		s = r.regex.ReplaceAllString(s, r.replacement)
+	}
+	return s
+}

log/log_test.go

@@ -0,0 +1,28 @@
+package log
+
+import (
+	"bytes"
+	"log"
+	"testing"
+
+	"github.com/contentsquare/chproxy/config"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLogMask(t *testing.T) {
+	err := InitReplacer([]config.LogMask{
+		{
+			Regex:       `(s3\(\s*'(?:(?:\\'|[^'])*)'\s*,\s*'(?:(?:\\'|[^'])*)'\s*,\s*')((?:\\'|[^'])*)(')`,
+			Replacement: "$1******$3",
+		},
+	})
+	assert.NoError(t, err)
+	var b bytes.Buffer
+	testLogger := log.New(&b, "DEBUG: ", stdLogFlags)
+	err = testLogger.Output(outputCallDepth,
+		mask("select * from s3('http://s3server/bucket', 'access-key', 'seret-key', 'CSVWithNamesAndTypes')"))
+	assert.NoError(t, err)
+	res, err := b.ReadString('\n')
+	assert.NoError(t, err)
+	assert.Contains(t, res, "select * from s3('http://s3server/bucket', 'access-key', '******', 'CSVWithNamesAndTypes')")
+}

main.go

@@ -53,6 +53,9 @@ func main() {
 	if err != nil {
 		log.Fatalf("error while loading config: %s", err)
 	}
+	if err = log.InitReplacer(cfg.LogMasks); err != nil {
+		log.Fatalf("error while log replacer init: %s", err)
+	}
 	registerMetrics(cfg)
 	if err = applyConfig(cfg); err != nil {
 		log.Fatalf("error while applying config: %s", err)