Merge pull request open-telemetry#52 from sethAmazon/retry-imds

Retry IMDS Calls

Author: sethAmazon

exporter/datadogexporter/internal/hostmetadata/internal/ec2/ec2.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -63,12 +64,9 @@ func GetHostInfo(logger *zap.Logger) (hostInfo *HostInfo) {
 		return
 	}
 
-	meta := ec2metadata.New(sess)
-
-	if !meta.Available() {
-		logger.Debug("EC2 Metadata not available")
-		return
-	}
+	meta := ec2metadata.New(sess, &aws.Config{
+		Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+	})
 
 	if idDoc, err := meta.GetInstanceIdentityDocument(); err == nil {
 		hostInfo.InstanceID = idDoc.InstanceID

internal/aws/awsutil/conn.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
@@ -44,7 +45,9 @@ type ConnAttr interface {
 type Conn struct{}
 
 func (c *Conn) getEC2Region(s *session.Session) (string, error) {
-	return ec2metadata.New(s).Region()
+	return ec2metadata.New(s, &aws.Config{
+		Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+	}).Region()
 }
 
 // AWS STS endpoint constants

internal/aws/proxy/conn.go

@@ -28,6 +28,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
@@ -79,7 +80,9 @@ var newAWSSession = func(roleArn string, region string, log *zap.Logger) (*sessi
 }
 
 var getEC2Region = func(s *session.Session) (string, error) {
-	return ec2metadata.New(s).Region()
+	return ec2metadata.New(s, &aws.Config{
+		Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+	}).Region()
 }
 
 func getAWSConfigSession(c *Config, logger *zap.Logger) (*aws.Config, *session.Session, error) {

internal/aws/xray/telemetry/sender.go

@@ -19,6 +19,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/xray"
@@ -187,7 +189,9 @@ func ToOptions(cfg Config, sess *session.Session, settings *awsutil.AWSSessionSe
 		envMetadataProvider{envKey: envAWSInstanceID},
 	}
 	if !settings.LocalMode {
-		metadataClient := ec2metadata.New(sess)
+		metadataClient := ec2metadata.New(sess, &aws.Config{
+			Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+		})
 		hostnameProviders = append(hostnameProviders, ec2MetadataProvider{
 			client:      metadataClient,
 			metadataKey: metadataHostname,

internal/metadataproviders/aws/ec2/metadata.go

@@ -17,6 +17,8 @@ package ec2 // import "github.com/open-telemetry/opentelemetry-collector-contrib
 import (
 	"context"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 )
@@ -35,7 +37,9 @@ var _ Provider = (*metadataClient)(nil)
 
 func NewProvider(sess *session.Session) Provider {
 	return &metadataClient{
-		metadata: ec2metadata.New(sess),
+		metadata: ec2metadata.New(sess, &aws.Config{
+			Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+		}),
 	}
 }
 

receiver/awscontainerinsightreceiver/internal/host/ec2metadata.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
 	awsec2metadata "github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"go.uber.org/zap"
@@ -52,7 +54,9 @@ type ec2MetadataOption func(*ec2Metadata)
 func newEC2Metadata(ctx context.Context, session *session.Session, refreshInterval time.Duration,
 	instanceIDReadyC chan bool, instanceIPReadyC chan bool, localMode bool, logger *zap.Logger, options ...ec2MetadataOption) ec2MetadataProvider {
 	emd := &ec2Metadata{
-		client:           awsec2metadata.New(session),
+		client: awsec2metadata.New(session, &aws.Config{
+			Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+		}),
 		refreshInterval:  refreshInterval,
 		instanceIDReadyC: instanceIDReadyC,
 		instanceIPReadyC: instanceIPReadyC,