Skip to content

Commit c4aece8

Browse files
giladl-neosecgiladl-neosec
authored
Issue-128 | Allow param_ in query parameters (#175)
* Issue-128 | Allow param_ in query parameters * created func to hash map * using in newParamsRegistry * using in calcQueryParamsHash * go mod tidy changes * Change nil and empty to return 0 for Key with default value * Fix golang-lint * Added tests for calcQueryParamsHash
1 parent 8739522 commit c4aece8

File tree

10 files changed

+198
-39
lines changed

10 files changed

+198
-39
lines changed

‎cache/key.go

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,13 @@ type Key struct {
4545

4646
// Version represents data encoding version number
4747
Version int
48+
49+
// QueryParamsHash must contain hashed value of query params
50+
QueryParamsHash uint32
4851
}
4952

5053
// NewKey construct cache key from provided parameters with default version number
51-
func NewKey(query []byte, originParams url.Values, acceptEncoding string, paramsHash uint32) *Key {
54+
func NewKey(query []byte, originParams url.Values, acceptEncoding string, userParamsHash uint32, queryParamsHash uint32) *Key {
5255
return &Key{
5356
Query: query,
5457
AcceptEncoding: acceptEncoding,
@@ -60,8 +63,9 @@ func NewKey(query []byte, originParams url.Values, acceptEncoding string, params
6063
Extremes: originParams.Get("extremes"),
6164
MaxResultRows: originParams.Get("max_result_rows"),
6265
ResultOverflowMode: originParams.Get("result_overflow_mode"),
63-
UserParamsHash: paramsHash,
66+
UserParamsHash: userParamsHash,
6467
Version: Version,
68+
QueryParamsHash: queryParamsHash,
6569
}
6670
}
6771

@@ -71,9 +75,9 @@ func (k *Key) filePath(dir string) string {
7175

7276
// String returns string representation of the key.
7377
func (k *Key) String() string {
74-
s := fmt.Sprintf("V%d; Query=%q; AcceptEncoding=%q; DefaultFormat=%q; Database=%q; Compress=%q; EnableHTTPCompression=%q; Namespace=%q; MaxResultRows=%q; Extremes=%q; ResultOverflowMode=%q; UserParams=%d",
78+
s := fmt.Sprintf("V%d; Query=%q; AcceptEncoding=%q; DefaultFormat=%q; Database=%q; Compress=%q; EnableHTTPCompression=%q; Namespace=%q; MaxResultRows=%q; Extremes=%q; ResultOverflowMode=%q; UserParams=%d; QueryParams=%d",
7579
k.Version, k.Query, k.AcceptEncoding, k.DefaultFormat, k.Database, k.Compress, k.EnableHTTPCompression, k.Namespace,
76-
k.MaxResultRows, k.Extremes, k.ResultOverflowMode, k.UserParamsHash)
80+
k.MaxResultRows, k.Extremes, k.ResultOverflowMode, k.UserParamsHash, k.QueryParamsHash)
7781
h := sha256.Sum256([]byte(s))
7882

7983
// The first 16 bytes of the hash should be enough

‎cache/key_test.go

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,15 @@ func TestKeyString(t *testing.T) {
1515
Query: []byte("SELECT 1 FROM system.numbers LIMIT 10"),
1616
Version: 2,
1717
},
18-
expected: "bebe3382e36ffdeea479b45d827b208a",
18+
expected: "ef4c039ea06ce6fd95f4ffef551ba029",
1919
},
2020
{
2121
key: &Key{
2222
Query: []byte("SELECT 1 FROM system.numbers LIMIT 10"),
2323
AcceptEncoding: "gzip",
2424
Version: 2,
2525
},
26-
expected: "498c1af30fb94280fd7c7225c0c8fb39",
26+
expected: "cb83c486eea079a87a6e567ba9869111",
2727
},
2828
{
2929
key: &Key{
@@ -32,7 +32,7 @@ func TestKeyString(t *testing.T) {
3232
DefaultFormat: "JSON",
3333
Version: 2,
3434
},
35-
expected: "720292aa0647cc5e53e0b6e6033eef34",
35+
expected: "89edc4ac678557d80063d1060b712808",
3636
},
3737
{
3838
key: &Key{
@@ -42,7 +42,7 @@ func TestKeyString(t *testing.T) {
4242
Database: "foobar",
4343
Version: 2,
4444
},
45-
expected: "5c6a70736d71e570faca739c4557780c",
45+
expected: "120d73469183ace3a31c941cfcc8dc13",
4646
},
4747
{
4848
key: &Key{
@@ -53,7 +53,7 @@ func TestKeyString(t *testing.T) {
5353
Namespace: "ns123",
5454
Version: 2,
5555
},
56-
expected: "08b4baf6825e53bbd18136a88abda4f8",
56+
expected: "8441149c2cba1503e201aa94cda949f7",
5757
},
5858
{
5959
key: &Key{
@@ -65,7 +65,23 @@ func TestKeyString(t *testing.T) {
6565
Namespace: "ns123",
6666
Version: 2,
6767
},
68-
expected: "0e043f23ccd1b9039b33623b3b7c114a",
68+
expected: "882a1cfc54f86e75a3ee89757bd33672",
69+
},
70+
{
71+
key: &Key{
72+
Query: []byte("SELECT * FROM {table_name:Identifier} LIMIT 10"),
73+
QueryParamsHash: 3825709,
74+
Version: 3,
75+
},
76+
expected: "9d7a76630ca453d120a7349c4b6fa23d",
77+
},
78+
{
79+
key: &Key{
80+
Query: []byte("SELECT * FROM {table_name:Identifier} LIMIT 10"),
81+
QueryParamsHash: 3825710,
82+
Version: 3,
83+
},
84+
expected: "1899cf94d4c5a3dda9575df7d8734e9b",
6985
},
7086
}
7187

‎config/config_test.go

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -178,18 +178,18 @@ var fullConfig = Config{
178178

179179
Users: []User{
180180
{
181-
Name: "web",
182-
Password: "****",
183-
ToCluster: "first cluster",
184-
ToUser: "web",
185-
DenyHTTP: true,
186-
AllowCORS: true,
187-
ReqPerMin: 4,
188-
MaxQueueSize: 100,
189-
MaxQueueTime: Duration(35 * time.Second),
181+
Name: "web",
182+
Password: "****",
183+
ToCluster: "first cluster",
184+
ToUser: "web",
185+
DenyHTTP: true,
186+
AllowCORS: true,
187+
ReqPerMin: 4,
188+
MaxQueueSize: 100,
189+
MaxQueueTime: Duration(35 * time.Second),
190190
MaxExecutionTime: Duration(30 * time.Second),
191-
Cache: "longterm",
192-
Params: "web",
191+
Cache: "longterm",
192+
Params: "web",
193193
},
194194
{
195195
Name: "default",
@@ -275,9 +275,9 @@ func TestLoadConfig(t *testing.T) {
275275
},
276276
Users: []User{
277277
{
278-
Name: "default",
279-
ToCluster: "cluster",
280-
ToUser: "default",
278+
Name: "default",
279+
ToCluster: "cluster",
280+
ToUser: "default",
281281
MaxExecutionTime: Duration(30 * time.Second),
282282
},
283283
},

‎go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ go 1.17
44

55
require (
66
github.com/DataDog/zstd v1.5.0
7-
github.com/alicebob/miniredis/v2 v2.18.0
7+
github.com/alicebob/miniredis/v2 v2.21.0
88
github.com/go-redis/redis/v8 v8.11.4
99
github.com/google/go-cmp v0.5.7
1010
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
@@ -28,7 +28,7 @@ require (
2828
github.com/prometheus/client_model v0.1.0 // indirect
2929
github.com/prometheus/common v0.7.0 // indirect
3030
github.com/prometheus/procfs v0.0.8 // indirect
31-
github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da // indirect
31+
github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 // indirect
3232
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect
3333
golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
3434
golang.org/x/text v0.3.6 // indirect

‎go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
66
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
77
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
88
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
9-
github.com/alicebob/miniredis/v2 v2.18.0 h1:EPUGD69ou4Uw4c81t9NLh0+dSou46k4tFEvf498FJ0g=
10-
github.com/alicebob/miniredis/v2 v2.18.0/go.mod h1:gquAfGbzn92jvtrSC69+6zZnwSODVXVpYDRaGhWaL6I=
9+
github.com/alicebob/miniredis/v2 v2.21.0 h1:CdmwIlKUWFBDS+4464GtQiQ0R1vpzOgu4Vnd74rBL7M=
10+
github.com/alicebob/miniredis/v2 v2.21.0/go.mod h1:XNqvJdQJv5mSuVMc0ynneafpnL/zv52acZ6kqeS0t88=
1111
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
1212
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
1313
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -119,8 +119,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
119119
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
120120
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
121121
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
122-
github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg=
123-
github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
122+
github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 h1:k/gmLsJDWwWqbLCur2yWnJzwQEKRcAHXo6seXGuSwWw=
123+
github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA=
124124
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
125125
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
126126
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

‎proxy.go

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"net/http/httputil"
1111
"net/url"
1212
"strconv"
13+
"strings"
1314
"sync"
1415
"time"
1516

@@ -254,11 +255,14 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
254255
"cluster_user": s.labels["cluster_user"],
255256
}
256257

257-
var paramsHash uint32
258+
var userParamsHash uint32
258259
if s.user.params != nil {
259-
paramsHash = s.user.params.key
260+
userParamsHash = s.user.params.key
260261
}
261-
key := cache.NewKey(skipLeadingComments(q), origParams, sortHeader(req.Header.Get("Accept-Encoding")), paramsHash)
262+
263+
queryParamsHash := calcQueryParamsHash(origParams)
264+
265+
key := cache.NewKey(skipLeadingComments(q), origParams, sortHeader(req.Header.Get("Accept-Encoding")), userParamsHash, queryParamsHash)
262266

263267
startTime := time.Now()
264268
userCache := s.user.cache
@@ -358,6 +362,21 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
358362
}
359363
}
360364

365+
func calcQueryParamsHash(origParams url.Values) uint32 {
366+
queryParams := make(map[string]string)
367+
for param := range origParams {
368+
if strings.HasPrefix(param, "param_") {
369+
queryParams[param] = origParams.Get(param)
370+
}
371+
}
372+
var queryParamsHash, err = calcMapHash(queryParams)
373+
if err != nil {
374+
log.Errorf("fail to calc hash for params %s; %s", origParams, err)
375+
return 0
376+
}
377+
return queryParamsHash
378+
}
379+
361380
// applyConfig applies the given cfg to reverseProxy.
362381
//
363382
// New config is applied only if non-nil error returned.

‎proxy_test.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"crypto/tls"
66
"fmt"
7+
"github.com/stretchr/testify/assert"
78
"io"
89
"io/ioutil"
910
"math/rand"
@@ -761,3 +762,59 @@ func (r *requestRegistry) get(key string) (bool, error) {
761762
}
762763
return v, nil
763764
}
765+
766+
func TestCalcQueryParamsHash(t *testing.T) {
767+
testCases := []struct {
768+
name string
769+
input url.Values
770+
expectedResult uint32
771+
}{
772+
{
773+
"nil Value",
774+
nil,
775+
0,
776+
},
777+
{
778+
"empty calcQueryParamsHash",
779+
url.Values{},
780+
0,
781+
},
782+
{
783+
"map with non param_ value",
784+
url.Values{"session_id": {"foo", "bar"}},
785+
0,
786+
},
787+
{
788+
"map with only param_ value",
789+
url.Values{"param_limit": {"1"}},
790+
0x94a386,
791+
},
792+
{
793+
"map with only param_ value. value affects result",
794+
url.Values{"param_limit": {"2"}},
795+
0x329bae01,
796+
},
797+
{
798+
"map with mix of param_ and non-param_ value",
799+
url.Values{"param_limit": {"1"}, "session_id": {"foo", "bar"}},
800+
0x94a386,
801+
},
802+
{
803+
"map with multiple param_ values",
804+
url.Values{"param_limit": {"1", "2"}, "param_table": {"foo"}},
805+
0x3a8a5c31,
806+
},
807+
{
808+
"map with multiple param_ values and only first value in array affects result",
809+
url.Values{"param_limit": {"1"}, "param_table": {"foo", "bar"}},
810+
0x3a8a5c31,
811+
},
812+
}
813+
814+
for _, tc := range testCases {
815+
t.Run(tc.name, func(t *testing.T) {
816+
r := calcQueryParamsHash(tc.input)
817+
assert.Equal(t, r, tc.expectedResult)
818+
})
819+
}
820+
}

‎scope.go

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package main
33
import (
44
"context"
55
"fmt"
6-
"hash/fnv"
76
"io/ioutil"
87
"net"
98
"net/http"
@@ -343,6 +342,13 @@ func (s *scope) decorateRequest(req *http.Request) (*http.Request, url.Values) {
343342
}
344343
}
345344

345+
// Keep parametrized queries params
346+
for param := range origParams {
347+
if strings.HasPrefix(param, "param_") {
348+
params.Set(param, origParams.Get(param))
349+
}
350+
}
351+
346352
// Keep external_data params
347353
if req.Method == "POST" {
348354
ct := req.Header.Get("Content-Type")
@@ -426,13 +432,18 @@ func newParamsRegistry(params []config.Param) (*paramsRegistry, error) {
426432
if len(params) == 0 {
427433
return nil, fmt.Errorf("params can't be empty")
428434
}
429-
h := fnv.New32a()
430-
for _, p := range params {
431-
str := fmt.Sprintf("%s=%s&", p.Key, p.Value)
432-
h.Write([]byte(str))
435+
436+
var paramsMap map[string]string
437+
for _, k := range params {
438+
paramsMap[k.Key] = k.Value
433439
}
440+
key, err := calcMapHash(paramsMap)
441+
if err != nil {
442+
return nil, err
443+
}
444+
434445
return &paramsRegistry{
435-
key: h.Sum32(),
446+
key: key,
436447
params: params,
437448
}, nil
438449
}

‎utils.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,3 +268,18 @@ func (dc chDecompressor) decompress(r io.Reader) ([]byte, error) {
268268
lr := chdecompressor.NewReader(r)
269269
return ioutil.ReadAll(lr)
270270
}
271+
272+
func calcMapHash(m map[string]string) (uint32, error) {
273+
if len(m) == 0 {
274+
return 0, nil
275+
}
276+
h := fnv.New32a()
277+
for k, v := range m {
278+
str := fmt.Sprintf("%s=%s&", k, v)
279+
_, err := h.Write([]byte(str))
280+
if err != nil {
281+
return 0, err
282+
}
283+
}
284+
return h.Sum32(), nil
285+
}

0 commit comments

Comments
 (0)