New component: awslogsencodingextension (open-telemetry#38816)

#### Description

This is a new encoding extension for decoding logs from various AWS
services. We will start with the
CloudWatch Logs subscription filter format and expand to other formats,
including CloudFront access logs, VPC flow logs (text/parquet), and
more.

#### Link to tracking issue

First part of
open-telemetry#38627

#### Testing

Added basic unit tests.

#### Documentation

README.

Author: axw

.chloggen/add-awslogsencodingextension.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: new_component
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: awslogsencodingextension
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add AWS Logs Encoding Extension to support decoding logs produced by AWS services
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [38627]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

.github/CODEOWNERS

@@ -94,6 +94,7 @@ extension/cgroupruntimeextension/                                @open-telemetry
 extension/encoding/                                              @open-telemetry/collector-contrib-approvers @atoulme @dao-jun @dmitryax @MovieStoreGuy @VihasMakwana
 extension/encoding/avrologencodingextension/                     @open-telemetry/collector-contrib-approvers @thmshmm
 extension/encoding/awscloudwatchmetricstreamsencodingextension/  @open-telemetry/collector-contrib-approvers @axw @constanca-m
+extension/encoding/awslogsencodingextension/                     @open-telemetry/collector-contrib-approvers @axw @constanca-m
 extension/encoding/googlecloudlogentryencodingextension/         @open-telemetry/collector-contrib-approvers @alexvanboxel
 extension/encoding/jaegerencodingextension/                      @open-telemetry/collector-contrib-approvers @MovieStoreGuy @atoulme
 extension/encoding/jsonlogencodingextension/                     @open-telemetry/collector-contrib-approvers @VihasMakwana @atoulme

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -97,6 +97,7 @@ body:
       - extension/encoding
       - extension/encoding/avrologencoding
       - extension/encoding/awscloudwatchmetricstreamsencoding
+      - extension/encoding/awslogsencoding
       - extension/encoding/googlecloudlogentryencoding
       - extension/encoding/jaegerencoding
       - extension/encoding/jsonlogencoding

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -91,6 +91,7 @@ body:
       - extension/encoding
       - extension/encoding/avrologencoding
       - extension/encoding/awscloudwatchmetricstreamsencoding
+      - extension/encoding/awslogsencoding
       - extension/encoding/googlecloudlogentryencoding
       - extension/encoding/jaegerencoding
       - extension/encoding/jsonlogencoding

.github/ISSUE_TEMPLATE/other.yaml

@@ -91,6 +91,7 @@ body:
       - extension/encoding
       - extension/encoding/avrologencoding
       - extension/encoding/awscloudwatchmetricstreamsencoding
+      - extension/encoding/awslogsencoding
       - extension/encoding/googlecloudlogentryencoding
       - extension/encoding/jaegerencoding
       - extension/encoding/jsonlogencoding

.github/ISSUE_TEMPLATE/unmaintained.yaml

@@ -96,6 +96,7 @@ body:
       - extension/encoding
       - extension/encoding/avrologencoding
       - extension/encoding/awscloudwatchmetricstreamsencoding
+      - extension/encoding/awslogsencoding
       - extension/encoding/googlecloudlogentryencoding
       - extension/encoding/jaegerencoding
       - extension/encoding/jsonlogencoding

.github/component_labels.txt

@@ -75,6 +75,7 @@ extension/cgroupruntimeextension extension/cgroupruntime
 extension/encoding extension/encoding
 extension/encoding/avrologencodingextension extension/encoding/avrologencoding
 extension/encoding/awscloudwatchmetricstreamsencodingextension extension/encoding/awscloudwatchmetricstreams
+extension/encoding/awslogsencodingextension extension/encoding/awslogsencoding
 extension/encoding/googlecloudlogentryencodingextension extension/encoding/googlecloudlogentryencoding
 extension/encoding/jaegerencodingextension extension/encoding/jaegerencoding
 extension/encoding/jsonlogencodingextension extension/encoding/jsonlogencoding

extension/encoding/awslogsencodingextension/Makefile

@@ -0,0 +1 @@
+include ../../../Makefile.Common

extension/encoding/awslogsencodingextension/README.md

@@ -0,0 +1,28 @@
+# AWS Logs encoding extension
+
+<!-- status autogenerated section -->
+| Status        |           |
+| ------------- |-----------|
+| Stability     | [development]  |
+| Distributions | [] |
+| Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aextension%2Fawslogsencoding%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aextension%2Fawslogsencoding) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aextension%2Fawslogsencoding%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aextension%2Fawslogsencoding) |
+| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@axw](https://www.github.com/axw), [@constanca-m](https://www.github.com/constanca-m) |
+
+[development]: https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/component-stability.md#development
+<!-- end autogenerated section -->
+
+This extension unmarshals logs encoded in formats produced by AWS services, including:
+ - [Amazon CloudWatch Logs Subscription Filters](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/SubscriptionFilters.html).
+ - (More to be added later.)
+
+Example for Amazon CloudWatch Logs Subscription Filters:
+```yaml
+extensions:
+  awslogs_encoding/cloudwatch:
+    format: cloudwatch_logs_subscription_filter
+
+receivers:
+  awsfirehose:
+    endpoint: :1234
+    encoding: awslogs_encoding/cloudwatch
+```

extension/encoding/awslogsencodingextension/cloudwatchlogssubscriptionfilter.go

@@ -0,0 +1,17 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awslogsencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/awslogsencodingextension"
+
+import (
+	"errors"
+
+	"go.opentelemetry.io/collector/pdata/plog"
+)
+
+type cloudWatchLogsSubscriptionFilterUnmarshaler struct{}
+
+func (cloudWatchLogsSubscriptionFilterUnmarshaler) UnmarshalLogs(_ []byte) (plog.Logs, error) {
+	// TODO implement this
+	return plog.Logs{}, errors.New("not implemented")
+}

extension/encoding/awslogsencodingextension/config.go

@@ -0,0 +1,39 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awslogsencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/awslogsencodingextension"
+
+import (
+	"fmt"
+
+	"go.opentelemetry.io/collector/confmap/xconfmap"
+)
+
+var _ xconfmap.Validator = (*Config)(nil)
+
+const (
+	formatCloudWatchLogsSubscriptionFilter = "cloudwatch_logs_subscription_filter"
+)
+
+var supportedFormats = []string{formatCloudWatchLogsSubscriptionFilter}
+
+type Config struct {
+	// Format defines the AWS logs format.
+	//
+	// Valid values are:
+	// - cloudwatch_logs_subscription_filter
+	//
+	Format string `mapstructure:"format"`
+}
+
+func (cfg *Config) Validate() error {
+	switch cfg.Format {
+	case "":
+		return fmt.Errorf("format unspecified, expected one of %q", supportedFormats)
+	case formatCloudWatchLogsSubscriptionFilter:
+		// valid
+	default:
+		return fmt.Errorf("unsupported format %q, expected one of %q", cfg.Format, supportedFormats)
+	}
+	return nil
+}

extension/encoding/awslogsencodingextension/config_test.go

@@ -0,0 +1,63 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awslogsencodingextension
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/confmap/confmaptest"
+	"go.opentelemetry.io/collector/confmap/xconfmap"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/awslogsencodingextension/internal/metadata"
+)
+
+func TestLoadConfig(t *testing.T) {
+	t.Parallel()
+
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml"))
+	require.NoError(t, err)
+
+	tests := []struct {
+		id          component.ID
+		expected    component.Config
+		expectedErr string
+	}{
+		{
+			id:          component.NewIDWithName(metadata.Type, ""),
+			expectedErr: `format unspecified, expected one of ["cloudwatch_logs_subscription_filter"]`,
+		},
+		{
+			id: component.NewIDWithName(metadata.Type, "cloudwatch_logs_subscription_filter"),
+			expected: &Config{
+				Format: formatCloudWatchLogsSubscriptionFilter,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		name := strings.ReplaceAll(tt.id.String(), "/", "_")
+		t.Run(name, func(t *testing.T) {
+			factory := NewFactory()
+			cfg := factory.CreateDefaultConfig()
+
+			sub, err := cm.Sub(tt.id.String())
+			require.NoError(t, err)
+			require.NoError(t, sub.Unmarshal(cfg))
+
+			err = xconfmap.Validate(cfg)
+			if tt.expectedErr != "" {
+				assert.Error(t, err)
+				assert.EqualError(t, err, tt.expectedErr)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, tt.expected, cfg)
+			}
+		})
+	}
+}

extension/encoding/awslogsencodingextension/doc.go

@@ -0,0 +1,8 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+//go:generate mdatagen metadata.yaml
+
+// Package awslogsencodingextension provides an encoding extension
+// for unmarshalling logs produced by various AWS services.
+package awslogsencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/awslogsencodingextension"

extension/encoding/awslogsencodingextension/extension.go

@@ -0,0 +1,53 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awslogsencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/awslogsencodingextension"
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/extension"
+	"go.opentelemetry.io/collector/pdata/plog"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding"
+)
+
+var _ encoding.LogsUnmarshalerExtension = (*encodingExtension)(nil)
+
+type encodingExtension struct {
+	unmarshaler plog.Unmarshaler
+	format      string
+}
+
+func newExtension(cfg *Config, _ extension.Settings) (*encodingExtension, error) {
+	switch cfg.Format {
+	case formatCloudWatchLogsSubscriptionFilter:
+		return &encodingExtension{
+			unmarshaler: cloudWatchLogsSubscriptionFilterUnmarshaler{},
+			format:      cfg.Format,
+		}, nil
+	default:
+		// Format will have been validated by Config.Validate,
+		// so we'll only get here if we haven't handled a valid
+		// format.
+		return nil, fmt.Errorf("unimplemented format %q", cfg.Format)
+	}
+}
+
+func (*encodingExtension) Start(_ context.Context, _ component.Host) error {
+	return nil
+}
+
+func (*encodingExtension) Shutdown(_ context.Context) error {
+	return nil
+}
+
+func (e *encodingExtension) UnmarshalLogs(record []byte) (plog.Logs, error) {
+	logs, err := e.unmarshaler.UnmarshalLogs(record)
+	if err != nil {
+		return plog.Logs{}, fmt.Errorf("failed to unmarshal logs as '%s' format: %w", e.format, err)
+	}
+	return logs, nil
+}

extension/encoding/awslogsencodingextension/extension_test.go

@@ -0,0 +1,25 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awslogsencodingextension
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/extension/extensiontest"
+)
+
+func TestNew_CloudWatchLogsSubscriptionFilter(t *testing.T) {
+	e, err := newExtension(&Config{Format: formatCloudWatchLogsSubscriptionFilter}, extensiontest.NewNopSettings(extensiontest.NopType))
+	require.NoError(t, err)
+	require.NotNil(t, e)
+}
+
+func TestNew_Unimplemented(t *testing.T) {
+	e, err := newExtension(&Config{Format: "invalid"}, extensiontest.NewNopSettings(extensiontest.NopType))
+	require.Error(t, err)
+	require.Nil(t, e)
+	assert.EqualError(t, err, `unimplemented format "invalid"`)
+}

extension/encoding/awslogsencodingextension/factory.go

@@ -0,0 +1,30 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package awslogsencodingextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/awslogsencodingextension"
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/extension"
+
+	"github.com/open-telemetry/opentelemetry-collector-contrib/extension/encoding/awslogsencodingextension/internal/metadata"
+)
+
+func NewFactory() extension.Factory {
+	return extension.NewFactory(
+		metadata.Type,
+		createDefaultConfig,
+		createExtension,
+		metadata.ExtensionStability,
+	)
+}
+
+func createExtension(_ context.Context, settings extension.Settings, cfg component.Config) (extension.Extension, error) {
+	return newExtension(cfg.(*Config), settings)
+}
+
+func createDefaultConfig() component.Config {
+	return &Config{}
+}