[exporter/awss3] feat!: partition format

.chloggen/custom-partition.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'breaking'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awss3exporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Replaced the `s3_partition` option with `s3_partition_format` to provide more flexibility to users.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [37915, 37503]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Users can provide custom file partitions using [strftime](https://linux.die.net/man/3/strftime) formatting.
+  The default value of `year=%Y/month=%m/day=%d/hour=%H/minute=%M` matches the older pattern (with `s3_partition: minute`)
+
+  If users do not provide a value for `s3_prefix`, the exporter will not create a `/` folder in the bucket.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/awss3exporter/README.md

@@ -24,7 +24,7 @@ The following exporter configuration parameters are supported.
 | `region`                  | AWS region.                                                                                                                                | "us-east-1" |
 | `s3_bucket`               | S3 bucket                                                                                                                                  |             |
 | `s3_prefix`               | prefix for the S3 key (root directory inside bucket).                                                                                      |             |
-| `s3_partition`            | time granularity of S3 key: hour or minute                                                                                                 | "minute"    |
+| `s3_partition_format`     | filepath formatting for the partition; See [strftime](https://linux.die.net/man/3/strftime) for format specification.                      | "year=%Y/month=%m/day=%d/hour=%H/minute=%M" |
 | `role_arn`                | the Role ARN to be assumed                                                                                                                 |             |
 | `file_prefix`             | file prefix defined by user                                                                                                                |             |
 | `marshaler`               | marshaler used to produce output data                                                                                                      | `otlp_json` |
@@ -58,30 +58,50 @@ See https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/
 - `none` (default): No compression will be applied
 - `gzip`: Files will be compressed with gzip. **This does not support `sumo_ic`marshaler.**
 
-# Example Configuration
+# Example Configurations
 
 Following example configuration defines to store output in 'eu-central' region and bucket named 'databucket'.
 
 ```yaml
 exporters:
   awss3:
     s3uploader:
-        region: 'eu-central-1'
-        s3_bucket: 'databucket'
-        s3_prefix: 'metric'
-        s3_partition: 'minute'
-
-  # Optional (disabled by default)
-  sending_queue:
-    enabled: true
-    num_consumers: 10
-    queue_size: 100
+      region: 'eu-central-1'
+      s3_bucket: 'databucket'
+      s3_prefix: 'metric'
+
+    # Optional (disabled by default)
+    sending_queue:
+      enabled: true
+      num_consumers: 10
+      queue_size: 100
 ```
 
 Logs and traces will be stored inside 'databucket' in the following path format.
 
 ```console
-metric/year=XXXX/month=XX/day=XX/hour=XX/minute=XX
+metric/year=YYYY/month=MM/day=DD/hour=HH/minute=mm
+```
+
+## Partition Formatting
+
+By setting the `s3_partition_format` option, users can specify the file path for their logs.
+See the [strftime](https://linux.die.net/man/3/strftime) reference for more formatting options.
+
+```yaml
+exporters:
+  awss3:
+    s3uploader:
+      region: 'eu-central-1'
+      s3_bucket: 'databucket'
+      s3_prefix: 'metric'
+      s3_partition_format: '%Y/%m/%d/%H/%M'
+```
+
+In this case, logs and traces would be stored in the following path format.
+
+```console
+metric/YYYY/MM/DD/HH/mm
 ```
 
 ## AWS Credential Configuration

exporter/awss3exporter/config.go

@@ -20,9 +20,8 @@ type S3UploaderConfig struct {
 	S3Bucket string `mapstructure:"s3_bucket"`
 	// S3Prefix is the key (directory) prefix to written to inside the bucket
 	S3Prefix string `mapstructure:"s3_prefix"`
-	// S3Partition is used to provide the rollup on how data is written.
-	// Valid values are: [hour,minute]
-	S3Partition string `mapstructure:"s3_partition"`
+	// S3PartitionFormat is used to provide the rollup on how data is written. Uses [strftime](https://linux.die.net/man/3/strftime) formatting.
+	S3PartitionFormat string `mapstructure:"s3_partition_format"`
 	// FilePrefix is the filename prefix used for the file to avoid any potential collisions.
 	FilePrefix string `mapstructure:"file_prefix"`
 	// Endpoint is the URL used for communicated with S3.

exporter/awss3exporter/config_test.go

@@ -40,10 +40,10 @@ func TestLoadConfig(t *testing.T) {
 		Encoding:              &encoding,
 		EncodingFileExtension: "baz",
 		S3Uploader: S3UploaderConfig{
-			Region:       "us-east-1",
-			S3Bucket:     "foo",
-			S3Partition:  "minute",
-			StorageClass: "STANDARD",
+			Region:            "us-east-1",
+			S3Bucket:          "foo",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			StorageClass:      "STANDARD",
 		},
 		MarshalerName: "otlp_json",
 	}, e,
@@ -73,12 +73,12 @@ func TestConfig(t *testing.T) {
 	assert.Equal(t, &Config{
 		QueueSettings: queueCfg,
 		S3Uploader: S3UploaderConfig{
-			Region:       "us-east-1",
-			S3Bucket:     "foo",
-			S3Prefix:     "bar",
-			S3Partition:  "minute",
-			Endpoint:     "http://endpoint.com",
-			StorageClass: "STANDARD",
+			Region:            "us-east-1",
+			S3Bucket:          "foo",
+			S3Prefix:          "bar",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			Endpoint:          "http://endpoint.com",
+			StorageClass:      "STANDARD",
 		},
 		MarshalerName: "otlp_json",
 	}, e,
@@ -104,12 +104,12 @@ func TestConfigS3StorageClaas(t *testing.T) {
 
 	assert.Equal(t, &Config{
 		S3Uploader: S3UploaderConfig{
-			Region:       "us-east-1",
-			S3Bucket:     "foo",
-			S3Prefix:     "bar",
-			S3Partition:  "minute",
-			Endpoint:     "http://endpoint.com",
-			StorageClass: "STANDARD_IA",
+			Region:            "us-east-1",
+			S3Bucket:          "foo",
+			S3Prefix:          "bar",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			Endpoint:          "http://endpoint.com",
+			StorageClass:      "STANDARD_IA",
 		},
 		QueueSettings: queueCfg,
 		MarshalerName: "otlp_json",
@@ -137,14 +137,14 @@ func TestConfigForS3CompatibleSystems(t *testing.T) {
 	assert.Equal(t, &Config{
 		QueueSettings: queueCfg,
 		S3Uploader: S3UploaderConfig{
-			Region:           "us-east-1",
-			S3Bucket:         "foo",
-			S3Prefix:         "bar",
-			S3Partition:      "minute",
-			Endpoint:         "alternative-s3-system.example.com",
-			S3ForcePathStyle: true,
-			DisableSSL:       true,
-			StorageClass:     "STANDARD",
+			Region:            "us-east-1",
+			S3Bucket:          "foo",
+			S3Prefix:          "bar",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			Endpoint:          "alternative-s3-system.example.com",
+			S3ForcePathStyle:  true,
+			DisableSSL:        true,
+			StorageClass:      "STANDARD",
 		},
 		MarshalerName: "otlp_json",
 	}, e,
@@ -252,10 +252,10 @@ func TestMarshallerName(t *testing.T) {
 	assert.Equal(t, &Config{
 		QueueSettings: queueCfg,
 		S3Uploader: S3UploaderConfig{
-			Region:       "us-east-1",
-			S3Bucket:     "foo",
-			S3Partition:  "minute",
-			StorageClass: "STANDARD",
+			Region:            "us-east-1",
+			S3Bucket:          "foo",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			StorageClass:      "STANDARD",
 		},
 		MarshalerName: "sumo_ic",
 	}, e,
@@ -266,10 +266,10 @@ func TestMarshallerName(t *testing.T) {
 	assert.Equal(t, &Config{
 		QueueSettings: queueCfg,
 		S3Uploader: S3UploaderConfig{
-			Region:       "us-east-1",
-			S3Bucket:     "bar",
-			S3Partition:  "minute",
-			StorageClass: "STANDARD",
+			Region:            "us-east-1",
+			S3Bucket:          "bar",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			StorageClass:      "STANDARD",
 		},
 		MarshalerName: "otlp_proto",
 	}, e,
@@ -296,11 +296,11 @@ func TestCompressionName(t *testing.T) {
 	assert.Equal(t, &Config{
 		QueueSettings: queueCfg,
 		S3Uploader: S3UploaderConfig{
-			Region:       "us-east-1",
-			S3Bucket:     "foo",
-			S3Partition:  "minute",
-			Compression:  "gzip",
-			StorageClass: "STANDARD",
+			Region:            "us-east-1",
+			S3Bucket:          "foo",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			Compression:       "gzip",
+			StorageClass:      "STANDARD",
 		},
 		MarshalerName: "otlp_json",
 	}, e,
@@ -311,11 +311,11 @@ func TestCompressionName(t *testing.T) {
 	assert.Equal(t, &Config{
 		QueueSettings: queueCfg,
 		S3Uploader: S3UploaderConfig{
-			Region:       "us-east-1",
-			S3Bucket:     "bar",
-			S3Partition:  "minute",
-			Compression:  "none",
-			StorageClass: "STANDARD",
+			Region:            "us-east-1",
+			S3Bucket:          "bar",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			Compression:       "none",
+			StorageClass:      "STANDARD",
 		},
 		MarshalerName: "otlp_proto",
 	}, e,

exporter/awss3exporter/factory.go

@@ -32,9 +32,9 @@ func createDefaultConfig() component.Config {
 	return &Config{
 		QueueSettings: queueCfg,
 		S3Uploader: S3UploaderConfig{
-			Region:       "us-east-1",
-			S3Partition:  "minute",
-			StorageClass: "STANDARD",
+			Region:            "us-east-1",
+			S3PartitionFormat: "year=%Y/month=%m/day=%d/hour=%H/minute=%M",
+			StorageClass:      "STANDARD",
 		},
 		MarshalerName: "otlp_json",
 	}

exporter/awss3exporter/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.61
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.1
 	github.com/aws/aws-sdk-go-v2/service/sts v1.33.14
+	github.com/itchyny/timefmt-go v0.1.6
 	github.com/stretchr/testify v1.10.0
 	github.com/tilinna/clock v1.1.0
 	go.opentelemetry.io/collector/component v0.119.1-0.20250210123122-44b3eeda354c

exporter/awss3exporter/internal/upload/partition.go

@@ -4,11 +4,11 @@
 package upload // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter/internal/upload"
 
 import (
-	"fmt"
 	"math/rand/v2"
 	"strconv"
 	"time"
 
+	"github.com/itchyny/timefmt-go"
 	"go.opentelemetry.io/collector/config/configcompression"
 )
 
@@ -20,10 +20,10 @@ type PartitionKeyBuilder struct {
 	// PartitionPrefix defines the S3 directory (key)
 	// prefix used to write the file
 	PartitionPrefix string
-	// PartitionTruncation is used to truncate values into
+	// PartitionFormat is used to separate values into
 	// different time buckets.
-	// Currently hourly or minutely is supported
-	PartitionTruncation string
+	// Uses [strftime](https://linux.die.net/man/3/strftime) formatting.
+	PartitionFormat string
 	// FilePrefix is used to define the prefix of the file written
 	// to the directory in S3.
 	FilePrefix string
@@ -49,16 +49,10 @@ func (pki *PartitionKeyBuilder) Build(ts time.Time) string {
 }
 
 func (pki *PartitionKeyBuilder) bucketKeyPrefix(ts time.Time) string {
-	key := fmt.Sprintf("year=%d/month=%02d/day=%02d/hour=%02d", ts.Year(), ts.Month(), ts.Day(), ts.Hour())
-
-	switch pki.PartitionTruncation {
-	case "minute":
-		key += "/" + fmt.Sprintf("minute=%02d", ts.Minute())
-	default:
-		// Nothing to do, key defaults to hourly
+	if pki.PartitionPrefix != "" {
+		pki.PartitionPrefix += "/"
 	}
-
-	return pki.PartitionPrefix + "/" + key
+	return pki.PartitionPrefix + timefmt.Format(ts, pki.PartitionFormat)
 }
 
 func (pki *PartitionKeyBuilder) fileName() string {