Fix metrics registration failure after panic (#830)

Author: Sneagan

kafka/main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	awsConfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/brave-intl/challenge-bypass-server/server"
+	"github.com/brave-intl/challenge-bypass-server/utils/metrics"
 	uuid "github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 	kafkaGo "github.com/segmentio/kafka-go"
@@ -103,14 +104,17 @@ func StartConsumers(ctx context.Context, providedServer *server.Server, logger *
 		prometheusRegistry = prometheus.DefaultRegisterer
 	}
 
-	prometheusRegistry.MustRegister(tokenIssuanceRequestTotal)
-	prometheusRegistry.MustRegister(tokenIssuanceFailureTotal)
-	prometheusRegistry.MustRegister(tokenRedeemRequestTotal)
-	prometheusRegistry.MustRegister(tokenRedeemFailureTotal)
-	prometheusRegistry.MustRegister(duplicateRedemptionTotal)
-	prometheusRegistry.MustRegister(idempotentRedemptionTotal)
-	prometheusRegistry.MustRegister(rebootFromPanicTotal)
-	prometheusRegistry.MustRegister(kafkaErrorTotal)
+	metrics.MustRegisterIfNotRegistered(
+		prometheusRegistry,
+		tokenIssuanceRequestTotal,
+		tokenIssuanceFailureTotal,
+		tokenRedeemRequestTotal,
+		tokenRedeemFailureTotal,
+		duplicateRedemptionTotal,
+		idempotentRedemptionTotal,
+		rebootFromPanicTotal,
+		kafkaErrorTotal,
+	)
 
 	if len(brokers) < 1 {
 		brokers = strings.Split(os.Getenv("VPC_KAFKA_BROKERS"), ",")
@@ -386,8 +390,8 @@ func getDialer(ctx context.Context, logger *slog.Logger) (*kafkaGo.Dialer, error
 			DualStack:     true,
 			SASLMechanism: aws_msk_iam_v2.NewMechanism(cfg),
 			TLS: &tls.Config{
-			    MinVersion: tls.VersionTLS12,
-		    },
+				MinVersion: tls.VersionTLS12,
+			},
 		}, nil
 	} else {
 		logger.Debug("generating Dialer")

server/server.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/dynamodb"
 	"github.com/brave-intl/bat-go/libs/middleware"
+	"github.com/brave-intl/challenge-bypass-server/utils/metrics"
 	"github.com/go-chi/chi/v5"
 	chiware "github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/httplog/v3"
@@ -77,23 +78,26 @@ var (
 
 // init - Register Metrics for Server
 func init() {
-	// DB
-	prometheus.MustRegister(fetchIssuerTotal)
-	prometheus.MustRegister(createIssuerTotal)
-	prometheus.MustRegister(redeemTokenTotal)
-	prometheus.MustRegister(fetchRedemptionTotal)
-	// DB latency
-	prometheus.MustRegister(fetchIssuerByTypeDBDuration)
-	prometheus.MustRegister(createIssuerDBDuration)
-	prometheus.MustRegister(createRedemptionDBDuration)
-	prometheus.MustRegister(fetchRedemptionDBDuration)
-	// API Calls
-	prometheus.MustRegister(v1BlindedTokenCallTotal)
-	prometheus.MustRegister(v1IssuerCallTotal)
-	prometheus.MustRegister(v2BlindedTokenCallTotal)
-	prometheus.MustRegister(v2IssuerCallTotal)
-	prometheus.MustRegister(v3BlindedTokenCallTotal)
-	prometheus.MustRegister(v3IssuerCallTotal)
+	metrics.MustRegisterIfNotRegistered(
+		prometheus.DefaultRegisterer,
+		// DB
+		fetchIssuerTotal,
+		createIssuerTotal,
+		redeemTokenTotal,
+		fetchRedemptionTotal,
+		// DB latency
+		fetchIssuerByTypeDBDuration,
+		createIssuerDBDuration,
+		createRedemptionDBDuration,
+		fetchRedemptionDBDuration,
+		// API Calls
+		v1BlindedTokenCallTotal,
+		v1IssuerCallTotal,
+		v2BlindedTokenCallTotal,
+		v2IssuerCallTotal,
+		v3BlindedTokenCallTotal,
+		v3IssuerCallTotal,
+	)
 }
 
 // Server - base server type

utils/metrics/register.go

@@ -0,0 +1,27 @@
+package metrics
+
+import (
+	"errors"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// When the service panics and restarts we get a prometheus error indicating that our
+// collectors are already registered. To handle this case, we check if a registration
+// failure is the result of an already registered collector and use that one if so.
+// This is all side effects due to the way that prometheus handles collectors as
+// interfaces with pointers etc.
+func MustRegisterIfNotRegistered(
+	registry prometheus.Registerer,
+	collectors ...prometheus.Collector,
+) {
+	for _, collector := range collectors {
+		err := registry.Register(collector)
+		if err != nil {
+			var alreadyRegisteredError prometheus.AlreadyRegisteredError
+			if !errors.As(err, &alreadyRegisteredError) {
+				panic(err)
+			}
+		}
+	}
+}