Prevent ANR during SDK initialization (#709)

* Prevent ANR during SDK initialization

Description
When initializing the OpenTelemetry Android SDK with disk buffering enabled, we
discovered that synchronous disk space checks were causing ANRs in production.
These checks occur during the creation of disk buffering exporters, specifically
in `DiskManager.getMaxFolderSize()`, which makes blocking IPC calls through
`StorageManager.getAllocatableBytes()` on the main thread. The issue manifests
in the following ANR stacktrace:
```
android.os.BinderProxy.transact (BinderProxy.java:662)
android.os.storage.IStorageManager$Stub$Proxy.getAllocatableBytes (IStorageManager.java:2837)
android.os.storage.StorageManager.getAllocatableBytes (StorageManager.java:2414)
android.os.storage.StorageManager.getAllocatableBytes (StorageManager.java:2404)
io.opentelemetry.android.internal.services.CacheStorage.getAvailableSpace (CacheStorage.java:66)
io.opentelemetry.android.internal.services.CacheStorage.ensureCacheSpaceAvailable (CacheStorage.java:50)
io.opentelemetry.android.internal.features.persistence.DiskManager.getMaxFolderSize (DiskManager.kt:58)
io.opentelemetry.android.OpenTelemetryRumBuilder.createStorageConfiguration (OpenTelemetryRumBuilder.java:338)
io.opentelemetry.android.OpenTelemetryRumBuilder.build (OpenTelemetryRumBuilder.java:286)
```

Our Solution
To fix this we moved initialization to run on a background executor and
buffer the data in memory until it completes.

The process works like this:
1. Initialize the SDK with `BufferDelegatingExporter` instances that can
   immediately accept telemetry data.
2. Move exporter initialization off the main thread.
3. Once async initialization completes, flush buffered signals to initialized
   exporters and delegate all future signals.

The primary goal of this solution is to be unobtrusive and prevent ANRs caused
by initialization of disk exporters, while preventing signals from being
dropped.

Testing
We have added unit tests to cover the buffering, delevation, and RUM
building. We've also verified this with both disk enabled and disk
disabled.

* Address the comments from the reviews

* Add some logging when the buffer overflows

* Fix the formatting

Author: ansman

core/src/main/java/io/opentelemetry/android/OpenTelemetryRumBuilder.java

@@ -10,8 +10,11 @@
 import android.app.Application;
 import android.util.Log;
 import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
 import io.opentelemetry.android.common.RumConstants;
 import io.opentelemetry.android.config.OtelRumConfig;
+import io.opentelemetry.android.export.BufferDelegatingLogExporter;
+import io.opentelemetry.android.export.BufferDelegatingSpanExporter;
 import io.opentelemetry.android.features.diskbuffering.DiskBufferingConfiguration;
 import io.opentelemetry.android.features.diskbuffering.SignalFromDiskExporter;
 import io.opentelemetry.android.features.diskbuffering.scheduler.DefaultExportScheduleHandler;
@@ -63,7 +66,6 @@
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
-import javax.annotation.Nullable;
 import kotlin.jvm.functions.Function0;
 
 /**
@@ -94,7 +96,10 @@ public final class OpenTelemetryRumBuilder {
 
     private Resource resource;
 
+    private boolean isBuilt = false;
+
     @Nullable private ServiceManager serviceManager;
+
     @Nullable private ExportScheduleHandler exportScheduleHandler;
 
     private static TextMapPropagator buildDefaultPropagator() {
@@ -122,6 +127,7 @@ public static OpenTelemetryRumBuilder create(Application application, OtelRumCon
      * @return {@code this}
      */
     public OpenTelemetryRumBuilder setResource(Resource resource) {
+        checkNotBuilt();
         this.resource = resource;
         return this;
     }
@@ -134,6 +140,7 @@ public OpenTelemetryRumBuilder setResource(Resource resource) {
      * @return {@code this}
      */
     public OpenTelemetryRumBuilder mergeResource(Resource resource) {
+        checkNotBuilt();
         this.resource = this.resource.merge(resource);
         return this;
     }
@@ -173,6 +180,7 @@ public OpenTelemetryRumBuilder addTracerProviderCustomizer(
      */
     public OpenTelemetryRumBuilder addMeterProviderCustomizer(
             BiFunction<SdkMeterProviderBuilder, Application, SdkMeterProviderBuilder> customizer) {
+        checkNotBuilt();
         meterProviderCustomizers.add(customizer);
         return this;
     }
@@ -193,6 +201,7 @@ public OpenTelemetryRumBuilder addMeterProviderCustomizer(
     public OpenTelemetryRumBuilder addLoggerProviderCustomizer(
             BiFunction<SdkLoggerProviderBuilder, Application, SdkLoggerProviderBuilder>
                     customizer) {
+        checkNotBuilt();
         loggerProviderCustomizers.add(customizer);
         return this;
     }
@@ -204,6 +213,7 @@ public OpenTelemetryRumBuilder addLoggerProviderCustomizer(
      */
     public OpenTelemetryRumBuilder addInstrumentation(AndroidInstrumentation instrumentation) {
         instrumentations.add(instrumentation);
+        checkNotBuilt();
         return this;
     }
 
@@ -218,6 +228,7 @@ public OpenTelemetryRumBuilder addInstrumentation(AndroidInstrumentation instrum
     public OpenTelemetryRumBuilder addPropagatorCustomizer(
             Function<? super TextMapPropagator, ? extends TextMapPropagator> propagatorCustomizer) {
         requireNonNull(propagatorCustomizer, "propagatorCustomizer");
+        checkNotBuilt();
         Function<? super TextMapPropagator, ? extends TextMapPropagator> existing =
                 this.propagatorCustomizer;
         this.propagatorCustomizer =
@@ -237,6 +248,7 @@ public OpenTelemetryRumBuilder addPropagatorCustomizer(
     public OpenTelemetryRumBuilder addSpanExporterCustomizer(
             Function<? super SpanExporter, ? extends SpanExporter> spanExporterCustomizer) {
         requireNonNull(spanExporterCustomizer, "spanExporterCustomizer");
+        checkNotBuilt();
         Function<? super SpanExporter, ? extends SpanExporter> existing =
                 this.spanExporterCustomizer;
         this.spanExporterCustomizer =
@@ -256,6 +268,7 @@ public OpenTelemetryRumBuilder addSpanExporterCustomizer(
     public OpenTelemetryRumBuilder addLogRecordExporterCustomizer(
             Function<? super LogRecordExporter, ? extends LogRecordExporter>
                     logRecordExporterCustomizer) {
+        checkNotBuilt();
         Function<? super LogRecordExporter, ? extends LogRecordExporter> existing =
                 this.logRecordExporterCustomizer;
         this.logRecordExporterCustomizer =
@@ -276,9 +289,63 @@ public OpenTelemetryRumBuilder addLogRecordExporterCustomizer(
      * @return A new {@link OpenTelemetryRum} instance.
      */
     public OpenTelemetryRum build() {
+        if (isBuilt) {
+            throw new IllegalStateException("You cannot call build multiple times");
+        }
+        isBuilt = true;
         InitializationEvents initializationEvents = InitializationEvents.get();
         applyConfiguration(initializationEvents);
 
+        BufferDelegatingLogExporter bufferDelegatingLogExporter = new BufferDelegatingLogExporter();
+
+        BufferDelegatingSpanExporter bufferDelegatingSpanExporter =
+                new BufferDelegatingSpanExporter();
+
+        SessionManager sessionManager =
+                SessionManager.create(timeoutHandler, config.getSessionTimeout().toNanos());
+
+        OpenTelemetrySdk sdk =
+                OpenTelemetrySdk.builder()
+                        .setTracerProvider(
+                                buildTracerProvider(
+                                        sessionManager, application, bufferDelegatingSpanExporter))
+                        .setLoggerProvider(
+                                buildLoggerProvider(
+                                        sessionManager, application, bufferDelegatingLogExporter))
+                        .setMeterProvider(buildMeterProvider(application))
+                        .setPropagators(buildFinalPropagators())
+                        .build();
+
+        otelSdkReadyListeners.forEach(listener -> listener.accept(sdk));
+
+        SdkPreconfiguredRumBuilder delegate =
+                new SdkPreconfiguredRumBuilder(
+                        application,
+                        sdk,
+                        timeoutHandler,
+                        sessionManager,
+                        config.shouldDiscoverInstrumentations(),
+                        getServiceManager());
+
+        // AsyncTask is deprecated but the thread pool is still used all over the Android SDK
+        // and it provides a way to get a background thread without having to create a new one.
+        android.os.AsyncTask.THREAD_POOL_EXECUTOR.execute(
+                () ->
+                        initializeExporters(
+                                initializationEvents,
+                                bufferDelegatingSpanExporter,
+                                bufferDelegatingLogExporter));
+
+        instrumentations.forEach(delegate::addInstrumentation);
+
+        return delegate.build();
+    }
+
+    private void initializeExporters(
+            InitializationEvents initializationEvents,
+            BufferDelegatingSpanExporter bufferDelegatingSpanExporter,
+            BufferDelegatingLogExporter bufferedDelegatingLogExporter) {
+
         DiskBufferingConfiguration diskBufferingConfiguration =
                 config.getDiskBufferingConfiguration();
         SpanExporter spanExporter = buildSpanExporter();
@@ -306,44 +373,25 @@ public OpenTelemetryRum build() {
         }
         initializationEvents.spanExporterInitialized(spanExporter);
 
-        SessionManager sessionManager =
-                SessionManager.create(timeoutHandler, config.getSessionTimeout().toNanos());
+        bufferedDelegatingLogExporter.setDelegate(logsExporter);
 
-        OpenTelemetrySdk sdk =
-                OpenTelemetrySdk.builder()
-                        .setTracerProvider(
-                                buildTracerProvider(sessionManager, application, spanExporter))
-                        .setLoggerProvider(
-                                buildLoggerProvider(sessionManager, application, logsExporter))
-                        .setMeterProvider(buildMeterProvider(application))
-                        .setPropagators(buildFinalPropagators())
-                        .build();
-
-        otelSdkReadyListeners.forEach(listener -> listener.accept(sdk));
+        bufferDelegatingSpanExporter.setDelegate(spanExporter);
 
         scheduleDiskTelemetryReader(signalFromDiskExporter);
-
-        SdkPreconfiguredRumBuilder delegate =
-                new SdkPreconfiguredRumBuilder(
-                        application,
-                        sdk,
-                        timeoutHandler,
-                        sessionManager,
-                        config.shouldDiscoverInstrumentations(),
-                        getServiceManager());
-        instrumentations.forEach(delegate::addInstrumentation);
-        return delegate.build();
     }
 
     @NonNull
     private ServiceManager getServiceManager() {
         if (serviceManager == null) {
             serviceManager = ServiceManagerImpl.Companion.create(application);
         }
-        return serviceManager;
+        // This can never be null since we never write `null` to it
+        return requireNonNull(serviceManager);
     }
 
-    public OpenTelemetryRumBuilder setServiceManager(ServiceManager serviceManager) {
+    public OpenTelemetryRumBuilder setServiceManager(@NonNull ServiceManager serviceManager) {
+        requireNonNull(serviceManager, "serviceManager cannot be null");
+        checkNotBuilt();
         this.serviceManager = serviceManager;
         return this;
     }
@@ -353,7 +401,9 @@ public OpenTelemetryRumBuilder setServiceManager(ServiceManager serviceManager)
      * If not specified, the default schedule exporter will be used.
      */
     public OpenTelemetryRumBuilder setExportScheduleHandler(
-            ExportScheduleHandler exportScheduleHandler) {
+            @NonNull ExportScheduleHandler exportScheduleHandler) {
+        requireNonNull(exportScheduleHandler, "exportScheduleHandler cannot be null");
+        checkNotBuilt();
         this.exportScheduleHandler = exportScheduleHandler;
         return this;
     }
@@ -376,7 +426,6 @@ private StorageConfiguration createStorageConfiguration() throws IOException {
     }
 
     private void scheduleDiskTelemetryReader(@Nullable SignalFromDiskExporter signalExporter) {
-
         if (exportScheduleHandler == null) {
             ServiceManager serviceManager = getServiceManager();
             // TODO: Is it safe to get the work service yet here? If so, we can
@@ -387,6 +436,9 @@ private void scheduleDiskTelemetryReader(@Nullable SignalFromDiskExporter signal
                             new DefaultExportScheduler(getWorkService), getWorkService);
         }
 
+        final ExportScheduleHandler exportScheduleHandler =
+                requireNonNull(this.exportScheduleHandler);
+
         if (signalExporter == null) {
             // Disabling here allows to cancel previously scheduled exports using tools that
             // can run even after the app has been terminated (such as WorkManager).
@@ -408,6 +460,7 @@ private void scheduleDiskTelemetryReader(@Nullable SignalFromDiskExporter signal
      * @return this
      */
     public OpenTelemetryRumBuilder addOtelSdkReadyListener(Consumer<OpenTelemetrySdk> callback) {
+        checkNotBuilt();
         otelSdkReadyListeners.add(callback);
         return this;
     }
@@ -521,4 +574,10 @@ private ContextPropagators buildFinalPropagators() {
         TextMapPropagator defaultPropagator = buildDefaultPropagator();
         return ContextPropagators.create(propagatorCustomizer.apply(defaultPropagator));
     }
+
+    private void checkNotBuilt() {
+        if (isBuilt) {
+            throw new IllegalStateException("This method cannot be called after calling build");
+        }
+    }
 }

core/src/main/java/io/opentelemetry/android/export/BufferDelegatingLogExporter.kt

@@ -0,0 +1,39 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.android.export
+
+import io.opentelemetry.sdk.common.CompletableResultCode
+import io.opentelemetry.sdk.logs.data.LogRecordData
+import io.opentelemetry.sdk.logs.export.LogRecordExporter
+
+/**
+ * An in-memory buffer delegating log exporter that buffers log records in memory until a delegate is set.
+ * Once a delegate is set, the buffered log records are exported to the delegate.
+ *
+ * The buffer size is set to 5,000 log entries by default. If the buffer is full, the exporter will drop new log records.
+ */
+internal class BufferDelegatingLogExporter(
+    maxBufferedLogs: Int = 5_000,
+) : LogRecordExporter {
+    private val delegatingExporter =
+        DelegatingExporter<LogRecordExporter, LogRecordData>(
+            doExport = LogRecordExporter::export,
+            doFlush = LogRecordExporter::flush,
+            doShutdown = LogRecordExporter::shutdown,
+            maxBufferedData = maxBufferedLogs,
+            logType = "log records",
+        )
+
+    fun setDelegate(delegate: LogRecordExporter) {
+        delegatingExporter.setDelegate(delegate)
+    }
+
+    override fun export(logs: Collection<LogRecordData>): CompletableResultCode = delegatingExporter.export(logs)
+
+    override fun flush(): CompletableResultCode = delegatingExporter.flush()
+
+    override fun shutdown(): CompletableResultCode = delegatingExporter.shutdown()
+}

core/src/main/java/io/opentelemetry/android/export/BufferDelegatingSpanExporter.kt

@@ -0,0 +1,39 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.android.export
+
+import io.opentelemetry.sdk.common.CompletableResultCode
+import io.opentelemetry.sdk.trace.data.SpanData
+import io.opentelemetry.sdk.trace.export.SpanExporter
+
+/**
+ * An in-memory buffer delegating span exporter that buffers span data in memory until a delegate is set.
+ * Once a delegate is set, the buffered span data is exported to the delegate.
+ *
+ * The buffer size is set to 5,000 spans by default. If the buffer is full, the exporter will drop new span data.
+ */
+internal class BufferDelegatingSpanExporter(
+    maxBufferedSpans: Int = 5_000,
+) : SpanExporter {
+    private val delegatingExporter =
+        DelegatingExporter<SpanExporter, SpanData>(
+            doExport = SpanExporter::export,
+            doFlush = SpanExporter::flush,
+            doShutdown = SpanExporter::shutdown,
+            maxBufferedData = maxBufferedSpans,
+            logType = "span data",
+        )
+
+    fun setDelegate(delegate: SpanExporter) {
+        delegatingExporter.setDelegate(delegate)
+    }
+
+    override fun export(spans: Collection<SpanData>): CompletableResultCode = delegatingExporter.export(spans)
+
+    override fun flush(): CompletableResultCode = delegatingExporter.flush()
+
+    override fun shutdown(): CompletableResultCode = delegatingExporter.shutdown()
+}