Use separate config struct for persistent storage (#4067)

Author: pmm-sumo

exporter/exporterhelper/README.md

@@ -17,12 +17,10 @@ The following configuration options can be modified:
 - `sending_queue`
   - `enabled` (default = true)
   - `num_consumers` (default = 10): Number of consumers that dequeue batches; ignored if `enabled` is `false`
-  - `queue_size` (default = 5000): Maximum number of batches kept in memory or on disk (for persistent storage) before dropping; ignored if `enabled` is `false`
+  - `queue_size` (default = 5000): Maximum number of batches kept in memory before dropping; ignored if `enabled` is `false`
   User should calculate this as `num_seconds * requests_per_second` where:
     - `num_seconds` is the number of seconds to buffer in case of a backend outage
     - `requests_per_second` is the average number of requests per seconds.
-  - `persistent_storage_enabled` (default = false): When set, enables persistence via a file storage extension 
-    (note, `enable_unstable` build tag needs to be enabled first, see below for more details)
 - `resource_to_telemetry_conversion`
   - `enabled` (default = false): If `enabled` is `true`, all the resource attributes will be converted to metric labels by default.
 - `timeout` (default = 5s): Time to wait per individual attempt to send data to a backend.
@@ -36,6 +34,15 @@ The full list of settings exposed for this helper exporter are documented [here]
 > :warning: The capability is under development and currently can be enabled only in OpenTelemetry
 > Collector Contrib with `enable_unstable` build tag set. 
 
+With this build tag set, additional configuration option can be enabled:
+
+- `sending_queue`
+  - `persistent_storage_enabled` (default = false): When set, enables persistence via a file storage extension
+    (note, `enable_unstable` build tag needs to be enabled first, see below for more details)
+
+The maximum number of batches stored to disk can be controlled using `sending_queue.queue_size` parameter (which,
+similarly as for in-memory buffering, defaults to 5000 batches).
+
 When `persistent_storage_enabled` is set to true, the queue is being buffered to disk using 
 [file storage extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/storage/filestorage).
 If collector instance is killed while having some items in the persistent queue, on restart the items are being picked and

exporter/exporterhelper/queued_retry.go

@@ -49,32 +49,6 @@ func init() {
 	metricproducer.GlobalManager().AddProducer(r)
 }
 
-// QueueSettings defines configuration for queueing batches before sending to the consumerSender.
-type QueueSettings struct {
-	// Enabled indicates whether to not enqueue batches before sending to the consumerSender.
-	Enabled bool `mapstructure:"enabled"`
-	// NumConsumers is the number of consumers from the queue.
-	NumConsumers int `mapstructure:"num_consumers"`
-	// QueueSize is the maximum number of batches allowed in queue at a given time.
-	QueueSize int `mapstructure:"queue_size"`
-	// PersistentStorageEnabled describes whether persistence via a file storage extension is enabled
-	PersistentStorageEnabled bool `mapstructure:"persistent_storage_enabled"`
-}
-
-// DefaultQueueSettings returns the default settings for QueueSettings.
-func DefaultQueueSettings() QueueSettings {
-	return QueueSettings{
-		Enabled:      true,
-		NumConsumers: 10,
-		// For 5000 queue elements at 100 requests/sec gives about 50 sec of survival of destination outage.
-		// This is a pretty decent value for production.
-		// User should calculate this from the perspective of how many seconds to buffer in case of a backend outage,
-		// multiply that by the number of requests per seconds.
-		QueueSize:                5000,
-		PersistentStorageEnabled: false,
-	}
-}
-
 // RetrySettings defines configuration for retrying batches in case of export failure.
 // The current supported strategy is exponential backoff.
 type RetrySettings struct {

exporter/exporterhelper/queued_retry_experimental.go

@@ -37,6 +37,32 @@ import (
 // queued_retry_experimental includes the code for both memory-backed and persistent-storage backed queued retry helpers
 // enabled by setting "enable_unstable" build tag
 
+// QueueSettings defines configuration for queueing batches before sending to the consumerSender.
+type QueueSettings struct {
+	// Enabled indicates whether to not enqueue batches before sending to the consumerSender.
+	Enabled bool `mapstructure:"enabled"`
+	// NumConsumers is the number of consumers from the queue.
+	NumConsumers int `mapstructure:"num_consumers"`
+	// QueueSize is the maximum number of batches allowed in queue at a given time.
+	QueueSize int `mapstructure:"queue_size"`
+	// PersistentStorageEnabled describes whether persistence via a file storage extension is enabled
+	PersistentStorageEnabled bool `mapstructure:"persistent_storage_enabled"`
+}
+
+// DefaultQueueSettings returns the default settings for QueueSettings.
+func DefaultQueueSettings() QueueSettings {
+	return QueueSettings{
+		Enabled:      true,
+		NumConsumers: 10,
+		// For 5000 queue elements at 100 requests/sec gives about 50 sec of survival of destination outage.
+		// This is a pretty decent value for production.
+		// User should calculate this from the perspective of how many seconds to buffer in case of a backend outage,
+		// multiply that by the number of requests per seconds.
+		QueueSize:                5000,
+		PersistentStorageEnabled: false,
+	}
+}
+
 var (
 	currentlyDispatchedBatchesGauge, _ = r.AddInt64DerivedGauge(
 		obsmetrics.ExporterKey+"/currently_dispatched_batches",

exporter/exporterhelper/queued_retry_inmemory.go

@@ -34,6 +34,29 @@ import (
 // queued_retry_inmemory includes the code for memory-backed (original) queued retry helper only
 // enabled when "enable_unstable" build tag is not set
 
+// QueueSettings defines configuration for queueing batches before sending to the consumerSender.
+type QueueSettings struct {
+	// Enabled indicates whether to not enqueue batches before sending to the consumerSender.
+	Enabled bool `mapstructure:"enabled"`
+	// NumConsumers is the number of consumers from the queue.
+	NumConsumers int `mapstructure:"num_consumers"`
+	// QueueSize is the maximum number of batches allowed in queue at a given time.
+	QueueSize int `mapstructure:"queue_size"`
+}
+
+// DefaultQueueSettings returns the default settings for QueueSettings.
+func DefaultQueueSettings() QueueSettings {
+	return QueueSettings{
+		Enabled:      true,
+		NumConsumers: 10,
+		// For 5000 queue elements at 100 requests/sec gives about 50 sec of survival of destination outage.
+		// This is a pretty decent value for production.
+		// User should calculate this from the perspective of how many seconds to buffer in case of a backend outage,
+		// multiply that by the number of requests per seconds.
+		QueueSize: 5000,
+	}
+}
+
 type queuedRetrySender struct {
 	fullName        string
 	cfg             QueueSettings