fix: improve memory-usage for inmemory-store

common-server/.gitignore

@@ -10,6 +10,8 @@
 *.dylib
 bin/*
 Dockerfile.cross
+data/*
+*.tmp.yaml
 
 # Test binary, built with `go test -c`
 *.test

common-server/helm/templates/deployment.yaml

@@ -57,6 +57,8 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
+          - name: db-data
+            mountPath: /data
           - name: app-config
             mountPath: /etc/common-server
             readOnly: true
@@ -84,6 +86,8 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+        - name: db-data
+          emptyDir: {}
         - name: app-config
           configMap:
             name: {{ .Release.Name }}-config

common-server/internal/config/config.go

@@ -56,6 +56,9 @@ type SecurityConfig struct {
 type ServerConfig struct {
 	Address  string `json:"address"`
 	BasePath string `json:"basepath"`
+	// Filepath is used to configure the database to use the filesystem.
+	// It is used as base path for the database files.
+	Filepath string `json:"filepath"`
 
 	AddGroupToPath bool               `yaml:"addGroupToPath" json:"addGroupToPath"`
 	Resources      []ResourceConfig   `json:"resources"`
@@ -166,6 +169,9 @@ func (c *ServerConfig) BuildServer(ctx context.Context, dynamicClient dynamic.In
 				GVR:          crd.GVR,
 				GVK:          crd.GVK,
 				AllowedSorts: resource.AllowedSorts,
+				Database: inmemory.DatabaseOpts{
+					Filepath: c.Filepath,
+				},
 			}
 
 			resourceId := strings.ToLower(crd.GVR.Resource)

common-server/internal/informer/informer.go

@@ -96,6 +96,10 @@ func SanitizeObject(obj *unstructured.Unstructured) {
 	}
 
 	delete(metadata, "managedFields")
+	annotations, ok := metadata["annotations"].(map[string]any)
+	if ok {
+		delete(annotations, "kubectl.kubernetes.io/last-applied-configuration")
+	}
 }
 
 func wrapEventHandler(ctx context.Context, log logr.Logger, eh EventHandler) cache.ResourceEventHandlerFuncs {

common-server/pkg/store/inmemory/inmemory_store.go

@@ -6,10 +6,14 @@ package inmemory
 
 import (
 	"context"
+	"fmt"
+	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/bytedance/sonic"
 	"github.com/dgraph-io/badger/v4"
+	"github.com/dgraph-io/badger/v4/options"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	"github.com/telekom/controlplane/common-server/internal/informer"
@@ -35,6 +39,12 @@ type StoreOpts struct {
 	GVR          schema.GroupVersionResource
 	GVK          schema.GroupVersionKind
 	AllowedSorts []string
+
+	Database DatabaseOpts
+}
+
+type DatabaseOpts struct {
+	Filepath string
 }
 
 type InmemoryObjectStore[T store.Object] struct {
@@ -49,9 +59,28 @@ type InmemoryObjectStore[T store.Object] struct {
 	sortValueCache sync.Map
 }
 
-func newDbOrDie(log logr.Logger) *badger.DB {
-	opts := badger.DefaultOptions("").WithInMemory(true)
-	opts.IndexCacheSize = 100 << 20
+func newDbOrDie(storeOpts StoreOpts, log logr.Logger) *badger.DB {
+	useFilesystem := storeOpts.Database.Filepath != ""
+	path := ""
+	if useFilesystem {
+		dbName := fmt.Sprintf("db-%s-%s-%s", strings.ToLower(storeOpts.GVR.Group), strings.ToLower(storeOpts.GVR.Version), strings.ToLower(storeOpts.GVR.Resource))
+		path = filepath.Join(storeOpts.Database.Filepath, dbName)
+	}
+
+	log.Info("initializing badger DB", "inMemory", !useFilesystem, "path", path)
+
+	opts := badger.DefaultOptions(path).
+		WithInMemory(!useFilesystem).
+		WithMetricsEnabled(false).
+		WithIndexCacheSize(0).
+		WithMemTableSize(32 << 20).     // 32 MB
+		WithValueLogFileSize(64 << 20). // 64 MB
+		WithBlockCacheSize(64 << 20).   // 64 MB
+		WithBlockSize(4 << 10).         // 4 KB
+		WithValueThreshold(1 << 20).    // 1 MB
+		WithBloomFalsePositive(0.01).
+		WithCompression(options.Snappy)
+
 	opts.Logger = NewLoggerShim(log)
 	db, err := badger.Open(opts)
 	if err != nil {
@@ -69,7 +98,7 @@ func NewOrDie[T store.Object](ctx context.Context, storeOpts StoreOpts) store.Ob
 		k8sClient: storeOpts.Client.Resource(storeOpts.GVR),
 	}
 	var err error
-	store.db = newDbOrDie(store.log)
+	store.db = newDbOrDie(storeOpts, store.log)
 	store.informer = informer.New(ctx, store.gvr, storeOpts.Client, store)
 
 	if err = store.informer.Start(); err != nil {

common-server/pkg/store/inmemory/sorted_store_test.go

@@ -26,7 +26,7 @@ var _ = Describe("Sorted Store", func() {
 		sortedStore := Sortable(
 			&InmemoryObjectStore[*unstructured.Unstructured]{
 				ctx: ctx,
-				db:  newDbOrDie(logr.Discard()),
+				db:  newDbOrDie(StoreOpts{}, logr.Discard()),
 				log: logr.Discard(),
 			},
 			StoreOpts{
@@ -136,7 +136,7 @@ func BenchmarkStore(b *testing.B) {
 	s := Sortable(
 		&InmemoryObjectStore[*unstructured.Unstructured]{
 			ctx: ctx,
-			db:  newDbOrDie(logr.Discard()),
+			db:  newDbOrDie(StoreOpts{}, logr.Discard()),
 			log: logr.Discard(),
 		},
 		StoreOpts{

common-server/pkg/store/secrets/resolver.go

@@ -107,7 +107,7 @@ func (s *SecretManagerResolver) ReplaceAllFromBytes(ctx context.Context, b []byt
 			log.V(1).Info("Replacing secrets in array", "jsonPath", paths)
 			b, err = s.ReplaceAllFromBytes(ctx, b, paths)
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to replace all from bytes for json path %s", jsonPath)
+				return nil, errors.Wrapf(err, "failed to replace all from bytes for json path %q", jsonPath)
 			}
 			continue
 		}
@@ -124,7 +124,7 @@ func (s *SecretManagerResolver) ReplaceAllFromBytes(ctx context.Context, b []byt
 		log.V(1).Info("Replacing secret", "jsonPath", jsonPath, "secretRef", secretRef)
 		secretValue, err := s.M.Get(ctx, secretRef)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to get secret value")
+			return nil, errors.Wrapf(err, "failed to get secret value for reference %q", secretRef)
 		}
 
 		b, err = sjson.SetBytes(b, jsonPath, secretValue)

rover-server/.gitignore

@@ -10,6 +10,7 @@
 *.dylib
 bin/*
 Dockerfile.cross
+data
 
 # Test binary, built with `go test -c`
 *.test

rover-server/README.md

@@ -35,6 +35,7 @@ The server can be configured using environment variables or configuration files:
 - `SECURITY_TRUSTEDISSUERS`: Comma-separated list of trusted issuers for JWT validation
 - `SECURITY_LMS_BASEPATH`: Base path for the LMS (Last Mile Security) checking
 - `SECURITY_DEFAULTSCOPE`: Default scope if token does not contain one
+- `DATABASE_FILEPATH`: This enables the database to store data also in the filesystem. If empty, the database will be in-memory only.
 
 # Installation
 

rover-server/config/default/kustomization.yaml

@@ -29,6 +29,7 @@ configMapGenerator:
     literals:
       - SECURITY_TRUSTEDISSUERS=""
       - SECURITY_LMS_BASEPATH=""
+      - DATABASE_FILEPATH=""
 
 patches:
   # [SECRET_MANAGER] The following patch will add the secret manager to the deployment.

rover-server/config/server/server.yaml

@@ -57,8 +57,12 @@ spec:
             requests:
               cpu: 10m
               memory: 64Mi
-          volumeMounts: []
-      volumes: []
+          volumeMounts:
+            - name: db-data
+              mountPath: /data
+      volumes:
+        - name: db-data
+          emptyDir: {}
       serviceAccountName: rover-server
       terminationGracePeriodSeconds: 10
 ---

rover-server/go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
 	go.uber.org/zap v1.27.0
+	go.yaml.in/yaml/v4 v4.0.0-rc.2
 	golang.org/x/text v0.29.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apimachinery v0.34.1
@@ -162,7 +163,6 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect
 	golang.org/x/arch v0.4.0 // indirect
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
 	golang.org/x/mod v0.27.0 // indirect

rover-server/internal/config/config.go

@@ -69,6 +69,9 @@ func setDefaults() {
 	// LMS
 	viper.SetDefault("security.lms.basePath", "")
 
-	//FileManager
+	// FileManager
 	viper.SetDefault("fileManager.skipTLS", true)
+
+	// Database
+	viper.SetDefault("database.filepath", "") // empty string means in-memory only
 }

rover-server/pkg/store/stores.go

@@ -7,6 +7,7 @@ package store
 import (
 	"context"
 
+	"github.com/spf13/viper"
 	adminv1 "github.com/telekom/controlplane/admin/api/v1"
 	apiv1 "github.com/telekom/controlplane/api/api/v1"
 	applicationv1 "github.com/telekom/controlplane/application/api/v1"
@@ -64,10 +65,13 @@ var InitOrDie = func(ctx context.Context, cfg *rest.Config) {
 
 func NewOrDie[T store.Object](ctx context.Context, gvr schema.GroupVersionResource, gvk schema.GroupVersionKind) store.ObjectStore[T] {
 	storeOpts := inmemory.StoreOpts{
+		Client:       dynamicClient,
 		GVR:          gvr,
 		GVK:          gvk,
 		AllowedSorts: []string{},
-		Client:       dynamicClient,
+		Database: inmemory.DatabaseOpts{
+			Filepath: viper.GetString("database.filepath"),
+		},
 	}
 
 	return inmemory.NewSortableOrDie[T](ctx, storeOpts)