Allow users to configure initial buffer size

Signed-off-by: Bogdan Drutu <[email protected]>

Author: bogdandrutu

.chloggen/add-initial-buffer.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/stanza
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow users to configure initial buffer size
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [37786]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

pkg/stanza/fileconsumer/config.go

@@ -59,6 +59,7 @@ func NewConfig() *Config {
 		MaxConcurrentFiles: defaultMaxConcurrentFiles,
 		StartAt:            "end",
 		FingerprintSize:    fingerprint.DefaultSize,
+		InitialBufferSize:  scanner.DefaultBufferSize,
 		MaxLogSize:         reader.DefaultMaxLogSize,
 		Encoding:           defaultEncoding,
 		FlushPeriod:        reader.DefaultFlushPeriod,
@@ -70,13 +71,15 @@ func NewConfig() *Config {
 
 // Config is the configuration of a file input operator
 type Config struct {
-	matcher.Criteria        `mapstructure:",squash"`
-	attrs.Resolver          `mapstructure:",squash"`
-	PollInterval            time.Duration   `mapstructure:"poll_interval,omitempty"`
-	MaxConcurrentFiles      int             `mapstructure:"max_concurrent_files,omitempty"`
-	MaxBatches              int             `mapstructure:"max_batches,omitempty"`
-	StartAt                 string          `mapstructure:"start_at,omitempty"`
-	FingerprintSize         helper.ByteSize `mapstructure:"fingerprint_size,omitempty"`
+	matcher.Criteria   `mapstructure:",squash"`
+	attrs.Resolver     `mapstructure:",squash"`
+	PollInterval       time.Duration   `mapstructure:"poll_interval,omitempty"`
+	MaxConcurrentFiles int             `mapstructure:"max_concurrent_files,omitempty"`
+	MaxBatches         int             `mapstructure:"max_batches,omitempty"`
+	StartAt            string          `mapstructure:"start_at,omitempty"`
+	FingerprintSize    helper.ByteSize `mapstructure:"fingerprint_size,omitempty"`
+	// InitialBufferSize represents the initial size of the read buffer for headers and body.
+	InitialBufferSize       helper.ByteSize `mapstructure:"initial_buffer_size,omitempty"`
 	MaxLogSize              helper.ByteSize `mapstructure:"max_log_size,omitempty"`
 	Encoding                string          `mapstructure:"encoding,omitempty"`
 	SplitConfig             split.Config    `mapstructure:"multiline,omitempty"`
@@ -154,7 +157,7 @@ func (c Config) Build(set component.TelemetrySettings, emit emit.Callback, opts
 		TelemetrySettings:       set,
 		FromBeginning:           startAtBeginning,
 		FingerprintSize:         int(c.FingerprintSize),
-		InitialBufferSize:       scanner.DefaultBufferSize,
+		InitialBufferSize:       int(c.InitialBufferSize),
 		MaxLogSize:              int(c.MaxLogSize),
 		Encoding:                enc,
 		SplitFunc:               splitFunc,

pkg/stanza/fileconsumer/internal/reader/reader.go

@@ -167,7 +167,6 @@ func (r *Reader) readHeader(ctx context.Context) (doneReadingFile bool) {
 	}
 	r.headerReader = nil
 	r.HeaderFinalized = true
-	r.initialBufferSize = scanner.DefaultBufferSize
 
 	// Reset position in file to r.Offest after the header scanner might have moved it past a content token.
 	if _, err := r.file.Seek(r.Offset, 0); err != nil {

receiver/otlpjsonfilereceiver/file_test.go

@@ -206,6 +206,7 @@ func testdataConfigYamlAsMap() *Config {
 			Encoding:           "utf-8",
 			StartAt:            "end",
 			FingerprintSize:    1000,
+			InitialBufferSize:  16 * 1024,
 			MaxLogSize:         1024 * 1024,
 			MaxConcurrentFiles: 1024,
 			FlushPeriod:        500 * time.Millisecond,