feat(o11y): add hibp request count and metrics endpoint

package.json

@@ -92,6 +92,7 @@
     "next-auth": "^4.24.13",
     "nodemailer": "^7.0.11",
     "pg": "^8.16.3",
+    "prom-client": "^15.1.3",
     "raw-loader": "^4.0.2",
     "react": "^19.2.3",
     "react-aria": "^3.45.0",

src/app/api/v1/hibp/notify/route.ts

@@ -1,20 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export const runtime = "nodejs";
+
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 import { NextRequest, NextResponse } from "next/server";
-import { captureMessage } from "@sentry/node";
 
 import { bearerToken } from "../../../utils/auth";
 import { logger } from "../../../../functions/server/logging";
 
 import { PubSub } from "@google-cloud/pubsub";
 import { isValidBearer } from "../../../../../utils/hibp";
 import { config } from "../../../../../config";
-
-const projectId = process.env.GCP_PUBSUB_PROJECT_ID;
-const topicName = process.env.GCP_PUBSUB_TOPIC_NAME;
-const subscriptionName = process.env.GCP_PUBSUB_SUBSCRIPTION_NAME;
+import {
+  hibpNotifyRequestsTotal,
+  incHibpNotifyFailure,
+} from "../../../../../instrumentation.node";
 
 export type PostHibpNotificationRequestBody = {
   breachName: string;
@@ -29,23 +34,18 @@ export type PostHibpNotificationRequestBody = {
  * @param req
  */
 export async function POST(req: NextRequest) {
+  hibpNotifyRequestsTotal.inc();
+
   let pubsub: PubSub;
   let json: PostHibpNotificationRequestBody;
-  const hibpNotifyToken = process.env.HIBP_NOTIFY_TOKEN;
+  const hibpNotifyToken = config.hibpNotifyToken;
+  const projectId = config.gcp.projectId;
+  const topicName = config.gcp.pubsub.hibpTopic;
   try {
-    if (!projectId) {
-      throw new Error("GCP_PUBSUB_PROJECT_ID env var not set");
-    }
-    if (!topicName) {
-      throw new Error("GCP_PUBSUB_TOPIC_NAME env var not set");
-    }
-    if (!hibpNotifyToken) {
-      throw new Error("HIBP_NOTIFY_TOKEN env var not set");
-    }
-
     const headerToken = bearerToken(req);
     if (!isValidBearer(headerToken, hibpNotifyToken)) {
       logger.error(`Received invalid header token: [${headerToken}]`);
+      incHibpNotifyFailure("unauthorized");
       return NextResponse.json({ success: false }, { status: 401 });
     }
 
@@ -55,44 +55,44 @@ export async function POST(req: NextRequest) {
       logger.error(
         "HIBP breach notification: requires breachName, hashPrefix, and hashSuffixes.",
       );
+      incHibpNotifyFailure("bad-request");
       return NextResponse.json({ success: false }, { status: 400 });
     }
   } catch (ex) {
     logger.error("error_processing_breach_alert_request:", {
       exception: ex as string,
     });
+    incHibpNotifyFailure("server-error");
     return NextResponse.json({ success: false }, { status: 500 });
   }
 
   try {
     pubsub = new PubSub({ projectId });
   } catch (ex) {
     logger.error("error_connecting_to_pubsub:", { exception: ex as string });
-    captureMessage(`error_connecting_to_pubsub:  ${ex as string}`);
+    incHibpNotifyFailure("pubsub-error");
     return NextResponse.json({ success: false }, { status: 429 });
   }
 
   try {
     const topic = pubsub.topic(topicName);
+    const [exists] = await topic.exists();
+    if (!exists) {
+      logger.error("error_connecting_to_pubsub: topic does not exist", {
+        topic: topicName,
+      });
+      incHibpNotifyFailure("pubsub-error");
+      return NextResponse.json({ success: false }, { status: 500 });
+    }
     await topic.publishMessage({ json });
     logger.info("queued_breach_notification_success", {
       json,
       topic: topicName,
     });
     return NextResponse.json({ success: true }, { status: 200 });
   } catch {
-    if (config.nodeEnv === "development") {
-      if (!subscriptionName) {
-        throw new Error("GCP_PUBSUB_SUBSCRIPTION_NAME env var not set");
-      }
-      await pubsub.createTopic(topicName);
-      await pubsub.topic(topicName).createSubscription(subscriptionName);
-    } else {
-      logger.error("pubsub_topic_not_found:", { topicName });
-      captureMessage(`pubsub_topic_not_found:  ${topicName}`);
-      return NextResponse.json({ success: false }, { status: 429 });
-    }
     logger.error("error_queuing_hibp_breach:", { topicName });
+    incHibpNotifyFailure("pubsub-error");
     return NextResponse.json({ success: false }, { status: 429 });
   }
 }

src/app/api/v1/metrics/route.ts

@@ -0,0 +1,17 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { NextResponse } from "next/server";
+import { registry } from "../../../../instrumentation.node";
+
+export async function GET() {
+  const metrics = await registry.metrics();
+  return new NextResponse(metrics, {
+    status: 200,
+    headers: {
+      "Content-Type":
+        registry.contentType || "text/plain; version=0.0.4; charset=utf-8",
+    },
+  });
+}

src/config.ts

@@ -6,6 +6,11 @@
 import "./app/functions/server/notInClientComponent";
 import "./initializeEnvVars";
 
+// Don't need to have coverage on config object
+/* c8 ignore start */
+const isLocalOrTest =
+  process.env.NODE_ENV === "test" || process.env.APP_ENV === "local";
+
 /**
  * Environment-specific values
  *
@@ -76,7 +81,19 @@ export const config = {
   fxRemoteSettingsWriterUser: process.env.FX_REMOTE_SETTINGS_WRITER_USER,
   fxRemoteSettingsWriterPass: process.env.FX_REMOTE_SETTINGS_WRITER_PASS,
   fxRemoteSettingsWriterServer: process.env.FX_REMOTE_SETTINGS_WRITER_SERVER,
+
+  gcp: {
+    projectId: getEnvString("GCP_PUBSUB_PROJECT_ID", {
+      fallbackValue: isLocalOrTest ? "your-project-name" : undefined,
+    }),
+    pubsub: {
+      hibpTopic: getEnvString("GCP_PUBSUB_TOPIC_NAME", {
+        fallbackValue: isLocalOrTest ? "hibp-breaches" : undefined,
+      }),
+    },
+  },
 } as const;
+/* c8 ignore end */
 
 /**
  * Like {@link getEnvString}, but also ensures the value is a valid integer

src/instrumentation.node.ts

@@ -0,0 +1,86 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export const runtime = "nodejs";
+
+import client from "prom-client";
+
+type MetricsState = {
+  registry: Readonly<client.Registry>;
+  hibpNotifyRequestsTotal: client.Counter;
+  hibpNotifyRequestFailuresTotal: client.Counter<"error">;
+};
+
+declare global {
+  var metrics: Readonly<MetricsState>;
+}
+
+function getOrInitMetrics(): MetricsState {
+  // Return cached state
+  if (globalThis.metrics !== undefined) return globalThis.metrics;
+  const registry = new client.Registry();
+  client.collectDefaultMetrics({ register: registry });
+
+  /**
+   * hibp_notify_requests_total
+   * Metric to instrument the number of requests from HIBP notifying of breaches
+   * Scope: "/hibp/notify"
+   */
+  const hibpNotifyRequestsTotal = new client.Counter({
+    name: "hibp_notify_requests_total",
+    help: "Metric to instrument the number of requests from HIBP notifying of breaches",
+    registers: [registry],
+  });
+
+  /**
+   * hibp_notify_request_failures_total{error="..."}
+   * Metric to instrument the number of failed requests on HIBP notify endpoint
+   * Labels:
+   *  - error: one of "timeout", "bad-request", "rate-limited"
+   * Scope: "/hibp/notify"
+   */
+  const hibpNotifyRequestFailuresTotal = new client.Counter<"error">({
+    name: "hibp_notify_request_failures_total",
+    help: "Metric to instrument the number of failed requests on HIBP notify endpoint",
+    labelNames: ["error"],
+    registers: [registry],
+  });
+
+  const state: MetricsState = {
+    registry,
+    hibpNotifyRequestFailuresTotal,
+    hibpNotifyRequestsTotal,
+  };
+
+  // Make it readonly
+  Object.defineProperty(globalThis, "metrics", {
+    value: state,
+    writable: false,
+    configurable: false,
+    enumerable: false,
+  });
+  return state;
+}
+
+export const {
+  registry,
+  hibpNotifyRequestsTotal,
+  hibpNotifyRequestFailuresTotal,
+} = getOrInitMetrics();
+
+export type HibpNotifyFailureError =
+  | "server-error"
+  | "pubsub-error"
+  | "bad-request"
+  | "rate-limited"
+  | "unauthorized"
+  | "invalid-config";
+
+/**
+ * Increment helper to keep label values consistent
+ * on hibpNotifyRequestFailuresTotal
+ */
+export function incHibpNotifyFailure(error: HibpNotifyFailureError, by = 1) {
+  hibpNotifyRequestFailuresTotal.inc({ error }, by);
+}

src/instrumentation.ts

@@ -4,7 +4,7 @@
 
 import * as Sentry from "@sentry/nextjs";
 
-export function register() {
+export async function register() {
   if (process.env.NEXT_RUNTIME === "nodejs") {
     Sentry.init({
       environment: process.env.APP_ENV,
@@ -21,6 +21,7 @@ export function register() {
       // Setting this option to true will print useful information to the console while you're setting up Sentry.
       debug: false,
     });
+    await import("./instrumentation.node");
   }
 }
 

src/telemetry/prom_metrics.yaml

@@ -0,0 +1,32 @@
+# This documents custom metrics that are implemented in the app
+---
+version: 1.0
+
+service: Monitor
+
+hibp:
+  hibp_notify_requests_total:
+    description: |
+      Metric to instrument the number of requests from HIBP notifying of breaches
+    type: counter
+    scope: |
+      The "/hibp/notify" endpoint
+    # No alerting
+    alert_policy: []
+
+  hibp_notify_request_failures_total:
+    description: |
+      Metric to instrument the number of failed requests on HIBP notify endpoint
+    type: counter
+    labels:
+      - name: error
+        description: |
+          Record the error. Any of "server-error", "pubsub-error", "bad-request",
+          "rate-limited", "unauthorized", "invalid-config"
+    scope: |
+      The "/hibp/notify" endpoint
+    alert_policy:
+      - severity_level: S1
+        trigger_condition: |
+          Trigger when the aggregate failure rate is greater than
+          20 req/sec.