[receiver/awss3receiver] Introduce the AWS S3 Receiver

.chloggen/awss3receiver_intro.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awss3receiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "introduce the AWS S3 receiver"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [ 30750 ]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [ user ]

.github/CODEOWNERS

@@ -186,6 +186,7 @@ receiver/awscloudwatchmetricsreceiver/                   @open-telemetry/collect
 receiver/awscloudwatchreceiver/                          @open-telemetry/collector-contrib-approvers @djaglowski @schmikei
 receiver/awscontainerinsightreceiver/                    @open-telemetry/collector-contrib-approvers @Aneurysm9 @pxaws
 receiver/awsecscontainermetricsreceiver/                 @open-telemetry/collector-contrib-approvers @Aneurysm9
+receiver/awss3receiver/                                  @open-telemetry/collector-contrib-approvers @atoulme @adcharre
 receiver/awsfirehosereceiver/                            @open-telemetry/collector-contrib-approvers @Aneurysm9
 receiver/awsxrayreceiver/                                @open-telemetry/collector-contrib-approvers @wangzlei @srprash
 receiver/azureblobreceiver/                              @open-telemetry/collector-contrib-approvers @eedorenko @mx-psi

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -183,6 +183,7 @@ body:
       - receiver/awscontainerinsight
       - receiver/awsecscontainermetrics
       - receiver/awsfirehose
+      - receiver/awss3
       - receiver/awsxray
       - receiver/azureblob
       - receiver/azureeventhub

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -177,6 +177,7 @@ body:
       - receiver/awscontainerinsight
       - receiver/awsecscontainermetrics
       - receiver/awsfirehose
+      - receiver/awss3
       - receiver/awsxray
       - receiver/azureblob
       - receiver/azureeventhub

.github/ISSUE_TEMPLATE/other.yaml

@@ -177,6 +177,7 @@ body:
       - receiver/awscontainerinsight
       - receiver/awsecscontainermetrics
       - receiver/awsfirehose
+      - receiver/awss3
       - receiver/awsxray
       - receiver/azureblob
       - receiver/azureeventhub

cmd/otelcontribcol/builder-config.yaml

@@ -124,6 +124,7 @@ receivers:
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.96.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver v0.96.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver v0.96.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver v0.96.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.96.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azureblobreceiver v0.96.0
   - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azureeventhubreceiver v0.96.0
@@ -444,4 +445,5 @@ replaces:
   - github.com/open-telemetry/opentelemetry-collector-contrib/extension/solarwindsapmsettingsextension => ../../extension/solarwindsapmsettingsextension
   - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/namedpipereceiver => ../../receiver/namedpipereceiver
   - github.com/open-telemetry/opentelemetry-collector-contrib/internal/sqlquery => ../../internal/sqlquery
+  - github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver => ../../receiver/awss3receiver
 

cmd/otelcontribcol/go.mod

@@ -112,6 +112,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.96.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsecscontainermetricsreceiver v0.96.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsfirehosereceiver v0.96.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver v0.96.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.96.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azureblobreceiver v0.96.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/azureeventhubreceiver v0.96.0
@@ -1191,3 +1192,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/sola
 replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/namedpipereceiver => ../../receiver/namedpipereceiver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/sqlquery => ../../internal/sqlquery
+
+replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver => ../../receiver/awss3receiver

receiver/awss3receiver/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

receiver/awss3receiver/README.md

@@ -0,0 +1,49 @@
+# AWS S3 Receiver
+<!-- status autogenerated section -->
+| Status        |           |
+| ------------- |-----------|
+| Stability     | [development]: traces   |
+| Distributions | [contrib] |
+| Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fawss3%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fawss3) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fawss3%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fawss3) |
+| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@atoulme](https://www.github.com/atoulme), [@adcharre](https://www.github.com/adcharre) |
+
+[development]: https://github.com/open-telemetry/opentelemetry-collector#development
+[contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
+<!-- end autogenerated section -->
+
+## Overview
+Receiver for retrieving trace previously stored in S3 by the [AWS S3 Exporter](../../exporter/awss3exporter/README.md).
+
+## Configuration
+The following exporter configuration parameters are supported.
+
+| Name                  | Description                                                                                                                                | Default     |
+|:----------------------|:-------------------------------------------------------------------------------------------------------------------------------------------|-------------|
+| `starttime`           | The time at which to start retrieving data.                                                                                                |             |
+| `endtime`             | The time at which to stop retrieving data.                                                                                                 |             |
+| `s3downloader:`       |                                                                                                                                            |             |
+| `region`              | AWS region.                                                                                                                                | "us-east-1" |
+| `s3_bucket`           | S3 bucket                                                                                                                                  |             |
+| `s3_prefix`           | prefix for the S3 key (root directory inside bucket).                                                                                      |             |
+| `s3_partition`        | time granularity of S3 key: hour or minute                                                                                                 | "minute"    |
+| `file_prefix`         | file prefix defined by user                                                                                                                |             |
+| `endpoint`            | overrides the endpoint used by the exporter instead of constructing it from `region` and `s3_bucket`                                       |             |
+| `s3_force_path_style` | [set this to `true` to force the request to use path-style addressing](http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html) | false       |
+
+### Time format for `starttime` and `endtime`
+The `starttime` and `endtime` fields are used to specify the time range for which to retrieve data. 
+The time format is either `YYYY-MM-DD HH:MM` or simply `YYYY-MM-DD`, in which case the time is assumed to be `00:00`.
+
+### Example Configuration
+
+```yaml
+receivers:
+  awss3:
+    starttime: "2024-01-01 01:00"
+    endtime: "2024-01-02"
+    s3downloader:
+        region: "us-west-1"
+        s3_bucket: "mybucket"
+        s3_prefix: "trace"
+        s3_partition: "minute"
+```

receiver/awss3receiver/config.go

@@ -0,0 +1,72 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awss3receiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver"
+
+import (
+	"errors"
+	"time"
+
+	"go.opentelemetry.io/collector/component"
+	"go.uber.org/multierr"
+)
+
+// S3DownloaderConfig contains aws s3 downloader related config to controls things
+// like bucket, prefix, batching, connections, retries, etc.
+type S3DownloaderConfig struct {
+	Region           string `mapstructure:"region"`
+	S3Bucket         string `mapstructure:"s3_bucket"`
+	S3Prefix         string `mapstructure:"s3_prefix"`
+	S3Partition      string `mapstructure:"s3_partition"`
+	FilePrefix       string `mapstructure:"file_prefix"`
+	Endpoint         string `mapstructure:"endpoint"`
+	S3ForcePathStyle bool   `mapstructure:"s3_force_path_style"`
+}
+
+// Config defines the configuration for the file receiver.
+type Config struct {
+	S3Downloader S3DownloaderConfig `mapstructure:"s3downloader"`
+	StartTime    string             `mapstructure:"starttime"`
+	EndTime      string             `mapstructure:"endtime"`
+}
+
+func createDefaultConfig() component.Config {
+	return &Config{
+		S3Downloader: S3DownloaderConfig{
+			Region:      "us-east-1",
+			S3Partition: "minute",
+		},
+	}
+}
+
+func (c Config) Validate() error {
+	var errs error
+	if c.S3Downloader.S3Bucket == "" {
+		errs = multierr.Append(errs, errors.New("bucket is required"))
+	}
+	if c.StartTime == "" {
+		errs = multierr.Append(errs, errors.New("start time is required"))
+	} else {
+		if err := validateTime(c.StartTime); err != nil {
+			errs = multierr.Append(errs, errors.New("unable to parse start date"))
+		}
+	}
+	if c.EndTime == "" {
+		errs = multierr.Append(errs, errors.New("end time is required"))
+	} else {
+		if err := validateTime(c.EndTime); err != nil {
+			errs = multierr.Append(errs, errors.New("unable to parse end time"))
+		}
+	}
+	return errs
+}
+
+func validateTime(str string) error {
+	layouts := []string{"2006-01-02 15:04", time.DateOnly}
+	for _, layout := range layouts {
+		if _, err := time.Parse(layout, str); err == nil {
+			return nil
+		}
+	}
+	return errors.New("unable to parse time string")
+}

receiver/awss3receiver/config_test.go

@@ -0,0 +1,89 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awss3receiver
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/confmap/confmaptest"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver/internal/metadata"
+)
+
+func TestLoadConfig_Validate_Invalid(t *testing.T) {
+	cfg := Config{}
+	assert.Error(t, cfg.Validate())
+}
+
+func TestConfig_Validate_Valid(t *testing.T) {
+	cfg := Config{
+		S3Downloader: S3DownloaderConfig{
+			Region:           "",
+			S3Bucket:         "abucket",
+			S3Prefix:         "",
+			S3Partition:      "",
+			FilePrefix:       "",
+			Endpoint:         "",
+			S3ForcePathStyle: false,
+		},
+		StartTime: "2024-01-01",
+		EndTime:   "2024-01-01",
+	}
+	assert.NoError(t, cfg.Validate())
+}
+
+func TestLoadConfig(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml"))
+	require.NoError(t, err)
+
+	tests := []struct {
+		id           component.ID
+		expected     component.Config
+		errorMessage string
+	}{
+		{
+			id:           component.NewIDWithName(metadata.Type, ""),
+			errorMessage: "bucket is required; start time is required; end time is required",
+		},
+		{
+			id:           component.NewIDWithName(metadata.Type, "1"),
+			errorMessage: "unable to parse start date; unable to parse end time",
+		},
+		{
+			id: component.NewIDWithName(metadata.Type, "2"),
+			expected: &Config{
+				S3Downloader: S3DownloaderConfig{
+					Region:      "us-east-1",
+					S3Bucket:    "abucket",
+					S3Partition: "minute",
+				},
+				StartTime: "2024-01-31 15:00",
+				EndTime:   "2024-02-03",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.id.String(), func(t *testing.T) {
+			factory := NewFactory()
+			cfg := factory.CreateDefaultConfig()
+
+			sub, err := cm.Sub(tt.id.String())
+			require.NoError(t, err)
+			require.NoError(t, component.UnmarshalConfig(sub, cfg))
+
+			if tt.errorMessage != "" {
+				assert.EqualError(t, component.ValidateConfig(cfg), tt.errorMessage)
+				return
+			}
+
+			assert.NoError(t, component.ValidateConfig(cfg))
+			assert.Equal(t, tt.expected, cfg)
+		})
+	}
+}

receiver/awss3receiver/doc.go

@@ -0,0 +1,9 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//go:generate mdatagen metadata.yaml
+
+// Package awss3receiver implements a receiver that can be used by the
+// Opentelemetry collector to retrieve traces previously stored in S3 by the
+// AWS S3 Exporter.
+package awss3receiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver"

receiver/awss3receiver/factory.go

@@ -0,0 +1,26 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awss3receiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver"
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/consumer"
+	"go.opentelemetry.io/collector/receiver"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awss3receiver/internal/metadata"
+)
+
+func NewFactory() receiver.Factory {
+	return receiver.NewFactory(
+		metadata.Type,
+		createDefaultConfig,
+		receiver.WithTraces(createTracesReceiver, metadata.TracesStability),
+	)
+}
+
+func createTracesReceiver(_ context.Context, settings receiver.CreateSettings, cc component.Config, consumer consumer.Traces) (receiver.Traces, error) {
+	return newAWSS3TraceReceiver(cc.(*Config), consumer, settings.Logger)
+}

receiver/awss3receiver/factory_test.go

@@ -0,0 +1,24 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awss3receiver
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/consumer/consumertest"
+	"go.opentelemetry.io/collector/receiver/receivertest"
+)
+
+func TestNewFactory(t *testing.T) {
+	f := NewFactory()
+	_, err := f.CreateTracesReceiver(
+		context.Background(),
+		receivertest.NewNopCreateSettings(),
+		f.CreateDefaultConfig(),
+		consumertest.NewNop(),
+	)
+	require.NoError(t, err)
+}