cache per user (#258)

* make cache not anymore shared per user

* make cache shareable by user configurable

Author: mga-chka

cache/async_cache.go

@@ -20,7 +20,8 @@ type AsyncCache struct {
 
 	graceTime time.Duration
 
-	MaxPayloadSize config.ByteSize
+	MaxPayloadSize     config.ByteSize
+	SharedWithAllUsers bool
 }
 
 func (c *AsyncCache) Close() error {
@@ -112,5 +113,6 @@ func NewAsyncCache(cfg config.Cache, maxExecutionTime time.Duration) (*AsyncCach
 		TransactionRegistry: transaction,
 		graceTime:           graceTime,
 		MaxPayloadSize:      maxPayloadSize,
+		SharedWithAllUsers:  cfg.SharedWithAllUsers,
 	}, nil
 }

cache/key.go

@@ -52,10 +52,13 @@ type Key struct {
 
 	// QueryParamsHash must contain hashed value of query params
 	QueryParamsHash uint32
+
+	// UserCredentialHash must contain hashed value of username & password
+	UserCredentialHash uint32
 }
 
 // NewKey construct cache key from provided parameters with default version number
-func NewKey(query []byte, originParams url.Values, acceptEncoding string, userParamsHash uint32, queryParamsHash uint32) *Key {
+func NewKey(query []byte, originParams url.Values, acceptEncoding string, userParamsHash uint32, queryParamsHash uint32, userCredentialHash uint32) *Key {
 	return &Key{
 		Query:                 query,
 		AcceptEncoding:        acceptEncoding,
@@ -70,6 +73,7 @@ func NewKey(query []byte, originParams url.Values, acceptEncoding string, userPa
 		UserParamsHash:        userParamsHash,
 		Version:               Version,
 		QueryParamsHash:       queryParamsHash,
+		UserCredentialHash:    userCredentialHash,
 	}
 }
 
@@ -79,9 +83,9 @@ func (k *Key) filePath(dir string) string {
 
 // String returns string representation of the key.
 func (k *Key) String() string {
-	s := fmt.Sprintf("V%d; Query=%q; AcceptEncoding=%q; DefaultFormat=%q; Database=%q; Compress=%q; EnableHTTPCompression=%q; Namespace=%q; MaxResultRows=%q; Extremes=%q; ResultOverflowMode=%q; UserParams=%d; QueryParams=%d",
+	s := fmt.Sprintf("V%d; Query=%q; AcceptEncoding=%q; DefaultFormat=%q; Database=%q; Compress=%q; EnableHTTPCompression=%q; Namespace=%q; MaxResultRows=%q; Extremes=%q; ResultOverflowMode=%q; UserParams=%d; QueryParams=%d; UserCredentialHash=%d",
 		k.Version, k.Query, k.AcceptEncoding, k.DefaultFormat, k.Database, k.Compress, k.EnableHTTPCompression, k.Namespace,
-		k.MaxResultRows, k.Extremes, k.ResultOverflowMode, k.UserParamsHash, k.QueryParamsHash)
+		k.MaxResultRows, k.Extremes, k.ResultOverflowMode, k.UserParamsHash, k.QueryParamsHash, k.UserCredentialHash)
 	h := sha256.Sum256([]byte(s))
 
 	// The first 16 bytes of the hash should be enough

cache/key_test.go

@@ -15,15 +15,15 @@ func TestKeyString(t *testing.T) {
 				Query:   []byte("SELECT 1 FROM system.numbers LIMIT 10"),
 				Version: 2,
 			},
-			expected: "ef4c039ea06ce6fd95f4ffef551ba029",
+			expected: "f11e8438adeeb325881c9a4da01925b3",
 		},
 		{
 			key: &Key{
 				Query:          []byte("SELECT 1 FROM system.numbers LIMIT 10"),
 				AcceptEncoding: "gzip",
 				Version:        2,
 			},
-			expected: "cb83c486eea079a87a6e567ba9869111",
+			expected: "045cbb29a40a81c42378569cf0bc4078",
 		},
 		{
 			key: &Key{
@@ -32,7 +32,7 @@ func TestKeyString(t *testing.T) {
 				DefaultFormat:  "JSON",
 				Version:        2,
 			},
-			expected: "89edc4ac678557d80063d1060b712808",
+			expected: "186386850c49c60a49dbf7af89c671c9",
 		},
 		{
 			key: &Key{
@@ -42,7 +42,7 @@ func TestKeyString(t *testing.T) {
 				Database:       "foobar",
 				Version:        2,
 			},
-			expected: "120d73469183ace3a31c941cfcc8dc13",
+			expected: "68f3231d17cad0a3473e63f419e07580",
 		},
 		{
 			key: &Key{
@@ -53,7 +53,7 @@ func TestKeyString(t *testing.T) {
 				Namespace:      "ns123",
 				Version:        2,
 			},
-			expected: "8441149c2cba1503e201aa94cda949f7",
+			expected: "8f5e765e69df7c24a58f13cdf752ad2f",
 		},
 		{
 			key: &Key{
@@ -65,23 +65,32 @@ func TestKeyString(t *testing.T) {
 				Namespace:      "ns123",
 				Version:        2,
 			},
-			expected: "882a1cfc54f86e75a3ee89757bd33672",
+			expected: "93a121f03f438ef7969540c78e943e2c",
 		},
 		{
 			key: &Key{
 				Query:           []byte("SELECT * FROM {table_name:Identifier} LIMIT 10"),
 				QueryParamsHash: 3825709,
 				Version:         3,
 			},
-			expected: "9d7a76630ca453d120a7349c4b6fa23d",
+			expected: "7edddc7d9db4bc4036dee36893f57cb1",
 		},
 		{
 			key: &Key{
 				Query:           []byte("SELECT * FROM {table_name:Identifier} LIMIT 10"),
 				QueryParamsHash: 3825710,
 				Version:         3,
 			},
-			expected: "1899cf94d4c5a3dda9575df7d8734e9b",
+			expected: "68ba76fb53a6fa71ba8fe63dd34a2201",
+		},
+		{
+			key: &Key{
+				Query:              []byte("SELECT * FROM {table_name:Identifier} LIMIT 10"),
+				QueryParamsHash:    3825710,
+				Version:            3,
+				UserCredentialHash: 234324,
+			},
+			expected: "c5b58ecb4ff026e62ee846dc63c749d5",
 		},
 	}
 

config/README.md

@@ -82,7 +82,11 @@ grace_time: <duration>
 
 # Maximum total size of request payload for caching. The default value
 # is set to 1 Petabyte.
+# The default value set so high is to allow users who do not use response size limitations virtually unlimited cache.
 max_payload_size: <byte_size>
+
+# Whether a query cached by a user can be used by another user
+shared_with_all_users: <bool> | default = false [optional]
 ```
 
 ### <distributed_cache_config>
@@ -122,6 +126,9 @@ grace_time: <duration>
 # is set to 1 Petabyte.
 # The default value set so high is to allow users who do not use response size limitations virtually unlimited cache.
 max_payload_size: <byte_size>
+
+# Whether a query cached by a user can be used by another user
+shared_with_all_users: <bool> | default = false [optional]
 ```
 
 ### <param_groups_config>

config/config.go

@@ -665,6 +665,9 @@ type Cache struct {
 
 	// Maximum total size of request payload for caching
 	MaxPayloadSize ByteSize `yaml:"max_payload_size,omitempty"`
+
+	// Whether a query cached by a user could be used by another user
+	SharedWithAllUsers bool `yaml:"shared_with_all_users,omitempty"`
 }
 
 type FileSystemCacheConfig struct {

config/config_test.go

@@ -21,9 +21,10 @@ var fullConfig = Config{
 				Dir:     "/path/to/longterm/cachedir",
 				MaxSize: ByteSize(100 << 30),
 			},
-			Expire:         Duration(time.Hour),
-			GraceTime:      Duration(20 * time.Second),
-			MaxPayloadSize: ByteSize(100 << 30),
+			Expire:             Duration(time.Hour),
+			GraceTime:          Duration(20 * time.Second),
+			MaxPayloadSize:     ByteSize(100 << 30),
+			SharedWithAllUsers: false,
 		},
 		{
 			Name: "shortterm",
@@ -32,8 +33,9 @@ var fullConfig = Config{
 				Dir:     "/path/to/shortterm/cachedir",
 				MaxSize: ByteSize(100 << 20),
 			},
-			Expire:         Duration(10 * time.Second),
-			MaxPayloadSize: ByteSize(100 << 20),
+			Expire:             Duration(10 * time.Second),
+			MaxPayloadSize:     ByteSize(100 << 20),
+			SharedWithAllUsers: true,
 		},
 	},
 	HackMePlease: true,
@@ -851,6 +853,7 @@ caches:
     dir: /path/to/shortterm/cachedir
     max_size: 104857600
   max_payload_size: 104857600
+  shared_with_all_users: true
 param_groups:
 - name: cron-job
   params:

config/testdata/full.yml

@@ -47,6 +47,7 @@ caches:
       max_size: 100Mb
       dir: "/path/to/shortterm/cachedir"
     max_payload_size: 100Mb
+    shared_with_all_users: true
     expire: 10s
 
 # Optional network lists, might be used as values for `allowed_networks`.

docs/content/en/configuration/caching.md

@@ -48,4 +48,15 @@ When Clickhouse responds to the firstly arrived query, existing key is updated a
 - if succeeded, as completed
 - if failed, as failed along with the exception message prepended with `[concurrent query failed]`.
 
-Transaction is kept for the duration of 2 * grace_time or 2 * max_execution_time, depending if grace time is specified.
+Transaction is kept for the duration of 2 * grace_time or 2 * max_execution_time, depending if grace time is specified.
+
+#### Cache shared with all users
+Until version 1.19.0, the cache is shared with all users.
+It means that if:
+- a user X does a query A,
+- then the result is cached,
+- then a user Y does the same query A
+User Y will get the cached response from user X's query.
+
+Since 1.20.0, the cache is specific for each user by default since it's better in terms of security.
+It's possible to use the previous behavior by setting the following property of the cache in the config file `shared_with_all_users = true`