feat(exporter-otlp-*)!: support custom HTTP agents

experimental/CHANGELOG.md

@@ -9,6 +9,10 @@ For notes on migrating to 2.x / 0.200.x see [the upgrade guide](doc/upgrade-to-2
 ### :boom: Breaking Changes
 
 * feat(api-logs)!: Marked private methods as "conventionally private". [#5789](https://github.com/open-telemetry/opentelemetry-js/pull/5789)
+* feat(exporter-otlp-\*): support custom HTTP agents [#5719](https://github.com/open-telemetry/opentelemetry-js/pull/5719) @raphael-theriault-swi
+  * `OtlpHttpConfiguration.agentOptions` has been removed and functionality has been rolled into `OtlpHttpConfiguration.agentFactory`
+    * (old) `{ agentOptions: myOptions }`
+    * (new) `{ agentFactory: httpAgentFactoryFromOptions(myOptions) }`
 
 ### :rocket: Features
 

experimental/packages/otlp-exporter-base/src/configuration/convert-legacy-node-http-options.ts

@@ -16,35 +16,32 @@
 import { OTLPExporterNodeConfigBase } from './legacy-node-configuration';
 import {
   getHttpConfigurationDefaults,
+  HttpAgentFactory,
+  httpAgentFactoryFromOptions,
   mergeOtlpHttpConfigurationWithDefaults,
   OtlpHttpConfiguration,
 } from './otlp-http-configuration';
 import { getHttpConfigurationFromEnvironment } from './otlp-http-env-configuration';
-import type * as http from 'http';
-import type * as https from 'https';
 import { diag } from '@opentelemetry/api';
 import { wrapStaticHeadersInFunction } from './shared-configuration';
 
 function convertLegacyAgentOptions(
   config: OTLPExporterNodeConfigBase
-): http.AgentOptions | https.AgentOptions | undefined {
-  // populate keepAlive for use with new settings
-  if (config?.keepAlive != null) {
-    if (config.httpAgentOptions != null) {
-      if (config.httpAgentOptions.keepAlive == null) {
-        // specific setting is not set, populate with non-specific setting.
-        config.httpAgentOptions.keepAlive = config.keepAlive;
-      }
-      // do nothing, use specific setting otherwise
-    } else {
-      // populate specific option if AgentOptions does not exist.
-      config.httpAgentOptions = {
-        keepAlive: config.keepAlive,
-      };
-    }
+): HttpAgentFactory | undefined {
+  if (typeof config.httpAgentOptions === 'function') {
+    return config.httpAgentOptions;
   }
 
-  return config.httpAgentOptions;
+  let legacy = config.httpAgentOptions;
+  if (config.keepAlive != null) {
+    legacy = { keepAlive: config.keepAlive, ...legacy };
+  }
+
+  if (legacy != null) {
+    return httpAgentFactoryFromOptions(legacy);
+  } else {
+    return undefined;
+  }
 }
 
 /**
@@ -72,7 +69,7 @@ export function convertLegacyHttpOptions(
       concurrencyLimit: config.concurrencyLimit,
       timeoutMillis: config.timeoutMillis,
       compression: config.compression,
-      agentOptions: convertLegacyAgentOptions(config),
+      agentFactory: convertLegacyAgentOptions(config),
     },
     getHttpConfigurationFromEnvironment(signalIdentifier, signalResourcePath),
     getHttpConfigurationDefaults(requiredHeaders, signalResourcePath)

experimental/packages/otlp-exporter-base/src/configuration/legacy-node-configuration.ts

@@ -18,15 +18,38 @@
 import type * as http from 'http';
 import type * as https from 'https';
 
-import { OTLPExporterConfigBase } from './legacy-base-configuration';
+import type { OTLPExporterConfigBase } from './legacy-base-configuration';
+import type { HttpAgentFactory } from './otlp-http-configuration';
 
 /**
  * Collector Exporter node base config
  */
 export interface OTLPExporterNodeConfigBase extends OTLPExporterConfigBase {
   keepAlive?: boolean;
   compression?: CompressionAlgorithm;
-  httpAgentOptions?: http.AgentOptions | https.AgentOptions;
+  /**
+   * Custom HTTP agent options or a factory function for creating agents.
+   *
+   * @remarks
+   * Prefer using `http.AgentOptions` or `https.AgentOptions` over a factory function wherever possible.
+   * If using a factory function (`HttpAgentFactory`), **do not import `http.Agent` or `https.Agent`
+   * statically at the top of the file**.
+   * Instead, use dynamic `import()` or `require()` to load the module. This ensures that the `http` or `https`
+   * module is not loaded before `@opentelemetry/instrumentation-http` can instrument it.
+   *
+   * @example <caption> Using agent options directly: </caption>
+   * httpAgentOptions: {
+   *   keepAlive: true,
+   *   maxSockets: 10
+   * }
+   *
+   * @example <caption> Using a factory with dynamic import: </caption>
+   * httpAgentOptions: async (protocol) => {
+   *   const module = protocol === 'http:' ? await import('http') : await import('https');
+   *   return new module.Agent({ keepAlive: true });
+   * }
+   */
+  httpAgentOptions?: http.AgentOptions | https.AgentOptions | HttpAgentFactory;
 }
 
 export enum CompressionAlgorithm {

experimental/packages/otlp-exporter-base/src/configuration/otlp-http-configuration.ts

@@ -25,10 +25,24 @@ import { validateAndNormalizeHeaders } from '../util';
 import type * as http from 'http';
 import type * as https from 'https';
 
+export type HttpAgentFactory = (
+  protocol: string
+) => http.Agent | https.Agent | Promise<http.Agent> | Promise<https.Agent>;
+
 export interface OtlpHttpConfiguration extends OtlpSharedConfiguration {
   url: string;
   headers: () => Record<string, string>;
-  agentOptions: http.AgentOptions | https.AgentOptions;
+  /**
+   * Factory function for creating agents.
+   *
+   * @remarks
+   * Prefer using {@link httpAgentFactoryFromOptions} over manually writing a factory function wherever possible.
+   * If using a factory function (`HttpAgentFactory`), **do not import `http.Agent` or `https.Agent`
+   * statically at the top of the file**.
+   * Instead, use dynamic `import()` or `require()` to load the module. This ensures that the `http` or `https`
+   * module is not loaded before `@opentelemetry/instrumentation-http` can instrument it.
+   */
+  agentFactory: HttpAgentFactory;
 }
 
 function mergeHeaders(
@@ -71,6 +85,16 @@ function validateUserProvidedUrl(url: string | undefined): string | undefined {
   }
 }
 
+export function httpAgentFactoryFromOptions(
+  options: http.AgentOptions | https.AgentOptions
+): HttpAgentFactory {
+  return async protocol => {
+    const module = protocol === 'http:' ? import('http') : import('https');
+    const { Agent } = await module;
+    return new Agent(options);
+  };
+}
+
 /**
  * @param userProvidedConfiguration  Configuration options provided by the user in code.
  * @param fallbackConfiguration Fallback to use when the {@link userProvidedConfiguration} does not specify an option.
@@ -96,10 +120,10 @@ export function mergeOtlpHttpConfigurationWithDefaults(
       validateUserProvidedUrl(userProvidedConfiguration.url) ??
       fallbackConfiguration.url ??
       defaultConfiguration.url,
-    agentOptions:
-      userProvidedConfiguration.agentOptions ??
-      fallbackConfiguration.agentOptions ??
-      defaultConfiguration.agentOptions,
+    agentFactory:
+      userProvidedConfiguration.agentFactory ??
+      fallbackConfiguration.agentFactory ??
+      defaultConfiguration.agentFactory,
   };
 }
 
@@ -111,6 +135,6 @@ export function getHttpConfigurationDefaults(
     ...getSharedConfigurationDefaults(),
     headers: () => requiredHeaders,
     url: 'http://localhost:4318/' + signalResourcePath,
-    agentOptions: { keepAlive: true },
+    agentFactory: httpAgentFactoryFromOptions({ keepAlive: true }),
   };
 }

experimental/packages/otlp-exporter-base/src/index-node-http.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+export { httpAgentFactoryFromOptions } from './configuration/otlp-http-configuration';
 export { createOtlpHttpExportDelegate } from './otlp-http-export-delegate';
 export { getSharedConfigurationFromEnvironment } from './configuration/shared-env-configuration';
 export { convertLegacyHttpOptions } from './configuration/convert-legacy-node-http-options';

experimental/packages/otlp-exporter-base/src/transport/http-exporter-transport.ts

@@ -14,21 +14,19 @@
  * limitations under the License.
  */
 
-import type {
-  HttpRequestParameters,
-  sendWithHttp,
-} from './http-transport-types';
+import type { HttpRequestParameters } from './http-transport-types';
 
 // NOTE: do not change these type imports to actual imports. Doing so WILL break `@opentelemetry/instrumentation-http`,
 // as they'd be imported before the http/https modules can be wrapped.
 import type * as https from 'https';
 import type * as http from 'http';
 import { ExportResponse } from '../export-response';
 import { IExporterTransport } from '../exporter-transport';
+import { sendWithHttp } from './http-transport-utils';
 
 interface Utils {
   agent: http.Agent | https.Agent;
-  send: sendWithHttp;
+  request: typeof http.request | typeof https.request;
 }
 
 class HttpExporterTransport implements IExporterTransport {
@@ -37,10 +35,11 @@ class HttpExporterTransport implements IExporterTransport {
   constructor(private _parameters: HttpRequestParameters) {}
 
   async send(data: Uint8Array, timeoutMillis: number): Promise<ExportResponse> {
-    const { agent, send } = this._loadUtils();
+    const { agent, request } = await this._loadUtils();
 
     return new Promise<ExportResponse>(resolve => {
-      send(
+      sendWithHttp(
+        request,
         this._parameters,
         agent,
         data,
@@ -56,30 +55,30 @@ class HttpExporterTransport implements IExporterTransport {
     // intentionally left empty, nothing to do.
   }
 
-  private _loadUtils(): Utils {
+  private async _loadUtils(): Promise<Utils> {
     let utils = this._utils;
 
     if (utils === null) {
-      // Lazy require to ensure that http/https is not required before instrumentations can wrap it.
-      const {
-        sendWithHttp,
-        createHttpAgent,
-        // eslint-disable-next-line @typescript-eslint/no-require-imports
-      } = require('./http-transport-utils');
-
-      utils = this._utils = {
-        agent: createHttpAgent(
-          this._parameters.url,
-          this._parameters.agentOptions
-        ),
-        send: sendWithHttp,
-      };
+      const protocol = new URL(this._parameters.url).protocol;
+      const [agent, request] = await Promise.all([
+        this._parameters.agentFactory(protocol),
+        requestFunctionFactory(protocol),
+      ]);
+      utils = this._utils = { agent, request };
     }
 
     return utils;
   }
 }
 
+async function requestFunctionFactory(
+  protocol: string
+): Promise<typeof http.request | typeof https.request> {
+  const module = protocol === 'http:' ? import('http') : import('https');
+  const { request } = await module;
+  return request;
+}
+
 export function createHttpExporterTransport(
   parameters: HttpRequestParameters
 ): IExporterTransport {

experimental/packages/otlp-exporter-base/src/transport/http-transport-types.ts

@@ -14,21 +14,11 @@
  * limitations under the License.
  */
 
-import type * as http from 'http';
-import type * as https from 'https';
-import { ExportResponse } from '../export-response';
-
-export type sendWithHttp = (
-  params: HttpRequestParameters,
-  agent: http.Agent | https.Agent,
-  data: Uint8Array,
-  onDone: (response: ExportResponse) => void,
-  timeoutMillis: number
-) => void;
+import { HttpAgentFactory } from '../configuration/otlp-http-configuration';
 
 export interface HttpRequestParameters {
   url: string;
   headers: () => Record<string, string>;
   compression: 'gzip' | 'none';
-  agentOptions: http.AgentOptions | https.AgentOptions;
+  agentFactory: HttpAgentFactory;
 }

experimental/packages/otlp-exporter-base/src/transport/http-transport-utils.ts

@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import * as http from 'http';
-import * as https from 'https';
+import type * as http from 'http';
+import type * as https from 'https';
 import * as zlib from 'zlib';
 import { Readable } from 'stream';
 import { HttpRequestParameters } from './http-transport-types';
@@ -27,13 +27,15 @@ import { OTLPExporterError } from '../types';
 
 /**
  * Sends data using http
+ * @param requestFunction
  * @param params
  * @param agent
  * @param data
  * @param onDone
  * @param timeoutMillis
  */
 export function sendWithHttp(
+  request: typeof https.request | typeof http.request,
   params: HttpRequestParameters,
   agent: http.Agent | https.Agent,
   data: Uint8Array,
@@ -53,8 +55,6 @@ export function sendWithHttp(
     agent: agent,
   };
 
-  const request = parsedUrl.protocol === 'http:' ? http.request : https.request;
-
   const req = request(options, (res: http.IncomingMessage) => {
     const responseData: Buffer[] = [];
     res.on('data', chunk => responseData.push(chunk));
@@ -133,12 +133,3 @@ function readableFromUint8Array(buff: string | Uint8Array): Readable {
 
   return readable;
 }
-
-export function createHttpAgent(
-  rawUrl: string,
-  agentOptions: http.AgentOptions | https.AgentOptions
-) {
-  const parsedUrl = new URL(rawUrl);
-  const Agent = parsedUrl.protocol === 'http:' ? http.Agent : https.Agent;
-  return new Agent(agentOptions);
-}

experimental/packages/otlp-exporter-base/test/common/configuration/otlp-http-configuration.test.ts

@@ -28,7 +28,7 @@ describe('mergeOtlpHttpConfigurationWithDefaults', function () {
     compression: 'none',
     concurrencyLimit: 2,
     headers: () => ({ 'User-Agent': 'default-user-agent' }),
-    agentOptions: { keepAlive: true },
+    agentFactory: () => null!,
   };
 
   describe('headers', function () {