Wait for sidecar in DaprSecretStoreConfiguration (#838)

The secret store configuration provider was trying to access Dapr
during the app startup. If the app started faster than Dapr, it would
get an error trying to access the secrets. This commit lets the
provider wait for the sidecar to come up before making any requests.
If the sidecar does not come up at all, we will still fail.

#779

Signed-off-by: Hal Spang <[email protected]>

Author: halspang

examples/AspNetCore/SecretStoreConfigurationProviderSample/Program.cs

@@ -19,6 +19,7 @@
     using Dapr.Extensions.Configuration;
     using Microsoft.Extensions.DependencyInjection;
     using System.Collections.Generic;
+    using System;
 
     /// <summary>
     /// Secret Store Configuration Provider Sample.
@@ -62,7 +63,8 @@ public static IHostBuilder CreateHostBuilder(string[] args)
                     // configBuilder.AddDaprSecretStore("demosecrets", secretDescriptors, client);
 
                     // Add the secret store Configuration Provider to the configuration builder.
-                    configBuilder.AddDaprSecretStore("demosecrets", client);
+                    // Including a TimeSpan allows us to dictate how long we should wait for the Sidecar to start.
+                    configBuilder.AddDaprSecretStore("demosecrets", client, TimeSpan.FromSeconds(10));
                 })
                 .ConfigureWebHostDefaults(webBuilder =>
                 {

src/Dapr.Client/DaprClient.cs

@@ -313,6 +313,23 @@ public HttpRequestMessage CreateInvokeMethodRequest<TRequest>(string appId, stri
         /// <param name="cancellationToken">A <see cref="CancellationToken" /> that can be used to cancel the operation.</param>
         /// <returns>A <see cref="Task{T}" /> that will return the value when the operation has completed.</returns>
         public abstract Task<bool> CheckHealthAsync(CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Perform health-check of Dapr sidecar's outbound APIs. Return 'true' if the sidecar is healthy. Otherwise false. This method should
+        /// be used over <see cref="CheckHealthAsync(CancellationToken)"/> when the health of Dapr is being checked before it starts. This
+        /// health endpoint indicates that Dapr has stood up its APIs and is currently waiting on this application to report fully healthy.
+        /// </summary>
+        /// <param name="cancellationToken">A <see cref="CancellationToken" /> that can be used to cancel the operation.</param>
+        /// <returns>A <see cref="Task{T}" /> that will return the value when the operation has completed.</returns>
+        public abstract Task<bool> CheckOutboundHealthAsync(CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Calls <see cref="CheckOutboundHealthAsync(CancellationToken)"/> until the sidecar is reporting as healthy. If the sidecar
+        /// does not become healthy, an exception will be thrown.
+        /// </summary>
+        /// <param name="cancellationToken">A <see cref="CancellationToken" /> that can be used to cancel the operation.</param>
+        /// <returns>A <see cref="Task" /> that will return when the operation has completed.</returns>
+        public abstract Task WaitForSidecarAsync(CancellationToken cancellationToken = default);
 
         /// <summary>
         /// Perform service invocation using the request provided by <paramref name="request" />. The response will

src/Dapr.Client/DaprClientGrpc.cs

@@ -305,6 +305,34 @@ public override async Task<bool> CheckHealthAsync(CancellationToken cancellation
             }
         }
 
+        public override async Task<bool> CheckOutboundHealthAsync(CancellationToken cancellationToken = default)
+        {
+            var path = "/v1.0/healthz/outbound";
+            var request = new HttpRequestMessage(HttpMethod.Get, new Uri(this.httpEndpoint, path));
+            try
+            {
+                var response = await this.httpClient.SendAsync(request, cancellationToken);
+                return response.IsSuccessStatusCode;
+            }
+            catch (HttpRequestException)
+            {
+                return false;
+            }
+        }
+
+        public override async Task WaitForSidecarAsync(CancellationToken cancellationToken = default)
+        {
+            while (true)
+            { 
+                var response = await CheckOutboundHealthAsync(cancellationToken);
+                if (response)
+                {
+                    break;
+                }
+                await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
+            }
+        }
+
         public override async Task<HttpResponseMessage> InvokeMethodWithResponseAsync(HttpRequestMessage request, CancellationToken cancellationToken = default)
         {
             ArgumentVerifier.ThrowIfNull(request, nameof(request));

src/Dapr.Extensions.Configuration/DaprSecretStoreConfigurationExtensions.cs

@@ -17,6 +17,7 @@
 using Microsoft.Extensions.Configuration;
 using Dapr.Extensions.Configuration.DaprSecretStore;
 using System.Linq;
+using System.Threading;
 
 namespace Dapr.Extensions.Configuration
 {
@@ -53,6 +54,37 @@ public static IConfigurationBuilder AddDaprSecretStore(
             return configurationBuilder;
         }
 
+        /// <summary>
+        /// Adds an <see cref="IConfigurationProvider"/> that reads configuration values from the Dapr Secret Store.
+        /// </summary>
+        /// <param name="configurationBuilder">The <see cref="IConfigurationBuilder"/> to add to.</param>
+        /// <param name="store">Dapr secret store name.</param>
+        /// <param name="secretDescriptors">The secrets to retrieve.</param>
+        /// <param name="client">The Dapr client.</param>
+        /// <param name="sidecarWaitTimeout">The <see cref="TimeSpan"/> used to configure the timeout waiting for Dapr.</param>
+        /// <returns>The <see cref="IConfigurationBuilder"/>.</returns>
+        public static IConfigurationBuilder AddDaprSecretStore(
+            this IConfigurationBuilder configurationBuilder,
+            string store,
+            IEnumerable<DaprSecretDescriptor> secretDescriptors,
+            DaprClient client,
+            TimeSpan sidecarWaitTimeout)
+        {
+            ArgumentVerifier.ThrowIfNullOrEmpty(store, nameof(store));
+            ArgumentVerifier.ThrowIfNull(secretDescriptors, nameof(secretDescriptors));
+            ArgumentVerifier.ThrowIfNull(client, nameof(client));
+
+            configurationBuilder.Add(new DaprSecretStoreConfigurationSource()
+            {
+                Store = store,
+                SecretDescriptors = secretDescriptors,
+                Client = client,
+                SidecarWaitTimeout = sidecarWaitTimeout
+            });
+
+            return configurationBuilder;
+        }
+
         /// <summary>
         /// Adds an <see cref="IConfigurationProvider"/> that reads configuration values from the Dapr Secret Store.
         /// </summary>
@@ -80,6 +112,36 @@ public static IConfigurationBuilder AddDaprSecretStore(
             return configurationBuilder;
         }
 
+        /// <summary>
+        /// Adds an <see cref="IConfigurationProvider"/> that reads configuration values from the Dapr Secret Store.
+        /// </summary>
+        /// <param name="configurationBuilder">The <see cref="IConfigurationBuilder"/> to add to.</param>
+        /// <param name="store">Dapr secret store name.</param>
+        /// <param name="metadata">A collection of metadata key-value pairs that will be provided to the secret store. The valid metadata keys and values are determined by the type of secret store used.</param>
+        /// <param name="client">The Dapr client</param>
+        /// <param name="sidecarWaitTimeout">The <see cref="TimeSpan"/> used to configure the timeout waiting for Dapr.</param>
+        /// <returns>The <see cref="IConfigurationBuilder"/>.</returns>
+        public static IConfigurationBuilder AddDaprSecretStore(
+            this IConfigurationBuilder configurationBuilder,
+            string store,
+            DaprClient client,
+            TimeSpan sidecarWaitTimeout,
+            IReadOnlyDictionary<string, string>? metadata = null)
+        {
+            ArgumentVerifier.ThrowIfNullOrEmpty(store, nameof(store));
+            ArgumentVerifier.ThrowIfNull(client, nameof(client));
+
+            configurationBuilder.Add(new DaprSecretStoreConfigurationSource()
+            {
+                Store = store,
+                Metadata = metadata,
+                Client = client,
+                SidecarWaitTimeout = sidecarWaitTimeout
+            });
+
+            return configurationBuilder;
+        }
+
         /// <summary>
         /// Adds an <see cref="IConfigurationProvider"/> that reads configuration values from the Dapr Secret Store.
         /// </summary>
@@ -113,6 +175,42 @@ public static IConfigurationBuilder AddDaprSecretStore(
             return configurationBuilder;
         }
 
+        /// <summary>
+        /// Adds an <see cref="IConfigurationProvider"/> that reads configuration values from the Dapr Secret Store.
+        /// </summary>
+        /// <param name="configurationBuilder">The <see cref="IConfigurationBuilder"/> to add to.</param>
+        /// <param name="store">Dapr secret store name.</param>
+        /// <param name="keyDelimiters">A collection of delimiters that will be replaced by ':' in the key of every secret.</param>
+        /// <param name="client">The Dapr client</param>
+        /// <param name="sidecarWaitTimeout">The <see cref="TimeSpan"/> used to configure the timeout waiting for Dapr.</param>
+        /// <returns>The <see cref="IConfigurationBuilder"/>.</returns>
+        public static IConfigurationBuilder AddDaprSecretStore(
+            this IConfigurationBuilder configurationBuilder,
+            string store,
+            DaprClient client,
+            IEnumerable<string>? keyDelimiters,
+            TimeSpan sidecarWaitTimeout)
+        {
+            ArgumentVerifier.ThrowIfNullOrEmpty(store, nameof(store));
+            ArgumentVerifier.ThrowIfNull(client, nameof(client));
+
+            var source = new DaprSecretStoreConfigurationSource
+            {
+                Store = store,
+                Client = client,
+                SidecarWaitTimeout = sidecarWaitTimeout
+            };
+
+            if (keyDelimiters != null)
+            {
+                source.KeyDelimiters = keyDelimiters.ToList();
+            }
+
+            configurationBuilder.Add(source);
+
+            return configurationBuilder;
+        }
+
         /// <summary>
         /// Adds an <see cref="IConfigurationProvider"/> that reads configuration values from the command line.
         /// </summary>

src/Dapr.Extensions.Configuration/DaprSecretStoreConfigurationProvider.cs

@@ -14,6 +14,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Dapr.Client;
 using Microsoft.Extensions.Configuration;
@@ -25,6 +26,8 @@ namespace Dapr.Extensions.Configuration.DaprSecretStore
     /// </summary>
     internal class DaprSecretStoreConfigurationProvider : ConfigurationProvider
     {
+        internal static readonly TimeSpan DefaultSidecarWaitTimeout = TimeSpan.FromSeconds(5);
+
         private readonly string store;
 
         private readonly bool normalizeKey;
@@ -37,39 +40,56 @@ internal class DaprSecretStoreConfigurationProvider : ConfigurationProvider
 
         private readonly DaprClient client;
 
+        private readonly TimeSpan sidecarWaitTimeout;
+
         /// <summary>
         /// Creates a new instance of <see cref="DaprSecretStoreConfigurationProvider"/>.
         /// </summary>
-        /// <param name="store">Dapr Secre Store name.</param>
+        /// <param name="store">Dapr Secret Store name.</param>
         /// <param name="normalizeKey">Indicates whether any key delimiters should be replaced with the delimiter ":".</param>
         /// <param name="secretDescriptors">The secrets to retrieve.</param>
         /// <param name="client">Dapr client used to retrieve Secrets</param>
-        public DaprSecretStoreConfigurationProvider(string store, bool normalizeKey, IEnumerable<DaprSecretDescriptor> secretDescriptors, DaprClient client)
-        {
-            ArgumentVerifier.ThrowIfNullOrEmpty(store, nameof(store));
-            ArgumentVerifier.ThrowIfNull(secretDescriptors, nameof(secretDescriptors));
-            ArgumentVerifier.ThrowIfNull(client, nameof(client));
-
-            if (secretDescriptors.Count() == 0)
-            {
-                throw new ArgumentException("No secret descriptor was provided", nameof(secretDescriptors));
-            }
+        public DaprSecretStoreConfigurationProvider(
+            string store,
+            bool normalizeKey,
+            IEnumerable<DaprSecretDescriptor> secretDescriptors,
+            DaprClient client) : this(store, normalizeKey, null, secretDescriptors, client, DefaultSidecarWaitTimeout)
+        {            
+        }
 
-            this.store = store;
-            this.normalizeKey = normalizeKey;
-            this.secretDescriptors = secretDescriptors;
-            this.client = client;
+        /// <summary>
+        /// Creates a new instance of <see cref="DaprSecretStoreConfigurationProvider"/>.
+        /// </summary>
+        /// <param name="store">Dapr Secret Store name.</param>
+        /// <param name="normalizeKey">Indicates whether any key delimiters should be replaced with the delimiter ":".</param>
+        /// <param name="keyDelimiters">A collection of delimiters that will be replaced by ':' in the key of every secret.</param>
+        /// <param name="secretDescriptors">The secrets to retrieve.</param>
+        /// <param name="client">Dapr client used to retrieve Secrets</param>
+        public DaprSecretStoreConfigurationProvider(
+            string store,
+            bool normalizeKey,
+            IList<string>? keyDelimiters,
+            IEnumerable<DaprSecretDescriptor> secretDescriptors,
+            DaprClient client) : this(store, normalizeKey, keyDelimiters, secretDescriptors, client, DefaultSidecarWaitTimeout)
+        {
         }
 
         /// <summary>
         /// Creates a new instance of <see cref="DaprSecretStoreConfigurationProvider"/>.
         /// </summary>
-        /// <param name="store">Dapr Secre Store name.</param>
+        /// <param name="store">Dapr Secret Store name.</param>
         /// <param name="normalizeKey">Indicates whether any key delimiters should be replaced with the delimiter ":".</param>
         /// <param name="keyDelimiters">A collection of delimiters that will be replaced by ':' in the key of every secret.</param>
         /// <param name="secretDescriptors">The secrets to retrieve.</param>
         /// <param name="client">Dapr client used to retrieve Secrets</param>
-        public DaprSecretStoreConfigurationProvider(string store, bool normalizeKey, IList<string>? keyDelimiters, IEnumerable<DaprSecretDescriptor> secretDescriptors, DaprClient client)
+        /// <param name="sidecarWaitTimeout">The <see cref="TimeSpan"/> used to configure the timeout waiting for Dapr.</param>
+        public DaprSecretStoreConfigurationProvider(
+            string store,
+            bool normalizeKey,
+            IList<string>? keyDelimiters,
+            IEnumerable<DaprSecretDescriptor> secretDescriptors,
+            DaprClient client,
+            TimeSpan sidecarWaitTimeout)
         {
             ArgumentVerifier.ThrowIfNullOrEmpty(store, nameof(store));
             ArgumentVerifier.ThrowIfNull(secretDescriptors, nameof(secretDescriptors));
@@ -85,35 +105,57 @@ public DaprSecretStoreConfigurationProvider(string store, bool normalizeKey, ILi
             this.keyDelimiters = keyDelimiters;
             this.secretDescriptors = secretDescriptors;
             this.client = client;
+            this.sidecarWaitTimeout = sidecarWaitTimeout;
         }
 
         /// <summary>
         /// Creates a new instance of <see cref="DaprSecretStoreConfigurationProvider"/>.
         /// </summary>
-        /// <param name="store">Dapr Secre Store name.</param>
+        /// <param name="store">Dapr Secret Store name.</param>
         /// <param name="normalizeKey">Indicates whether any key delimiters should be replaced with the delimiter ":".</param>
         /// <param name="metadata">A collection of metadata key-value pairs that will be provided to the secret store. The valid metadata keys and values are determined by the type of secret store used.</param>
         /// <param name="client">Dapr client used to retrieve Secrets</param>
-        public DaprSecretStoreConfigurationProvider(string store, bool normalizeKey, IReadOnlyDictionary<string, string>? metadata, DaprClient client)
+        public DaprSecretStoreConfigurationProvider(
+            string store,
+            bool normalizeKey,
+            IReadOnlyDictionary<string, string>? metadata,
+            DaprClient client) : this(store, normalizeKey, null, metadata, client, DefaultSidecarWaitTimeout)
         {
-            ArgumentVerifier.ThrowIfNullOrEmpty(store, nameof(store));
-            ArgumentVerifier.ThrowIfNull(client, nameof(client));
+        }
 
-            this.store = store;
-            this.normalizeKey = normalizeKey;
-            this.metadata = metadata;
-            this.client = client;
+        /// <summary>
+        /// Creates a new instance of <see cref="DaprSecretStoreConfigurationProvider"/>.
+        /// </summary>
+        /// <param name="store">Dapr Secret Store name.</param>
+        /// <param name="normalizeKey">Indicates whether any key delimiters should be replaced with the delimiter ":".</param>
+        /// <param name="keyDelimiters">A collection of delimiters that will be replaced by ':' in the key of every secret.</param>
+        /// <param name="metadata">A collection of metadata key-value pairs that will be provided to the secret store. The valid metadata keys and values are determined by the type of secret store used.</param>
+        /// <param name="client">Dapr client used to retrieve Secrets</param>
+        public DaprSecretStoreConfigurationProvider(
+            string store,
+            bool normalizeKey,
+            IList<string>? keyDelimiters,
+            IReadOnlyDictionary<string, string>? metadata,
+            DaprClient client) : this(store, normalizeKey, keyDelimiters, metadata, client, DefaultSidecarWaitTimeout)
+        {
         }
 
         /// <summary>
         /// Creates a new instance of <see cref="DaprSecretStoreConfigurationProvider"/>.
         /// </summary>
-        /// <param name="store">Dapr Secre Store name.</param>
+        /// <param name="store">Dapr Secret Store name.</param>
         /// <param name="normalizeKey">Indicates whether any key delimiters should be replaced with the delimiter ":".</param>
         /// <param name="keyDelimiters">A collection of delimiters that will be replaced by ':' in the key of every secret.</param>
         /// <param name="metadata">A collection of metadata key-value pairs that will be provided to the secret store. The valid metadata keys and values are determined by the type of secret store used.</param>
         /// <param name="client">Dapr client used to retrieve Secrets</param>
-        public DaprSecretStoreConfigurationProvider(string store, bool normalizeKey, IList<string>? keyDelimiters, IReadOnlyDictionary<string, string>? metadata, DaprClient client)
+        /// <param name="sidecarWaitTimeout">The <see cref="TimeSpan"/> used to configure the timeout waiting for Dapr.</param>
+        public DaprSecretStoreConfigurationProvider(
+            string store,
+            bool normalizeKey,
+            IList<string>? keyDelimiters,
+            IReadOnlyDictionary<string, string>? metadata,
+            DaprClient client,
+            TimeSpan sidecarWaitTimeout)
         {
             ArgumentVerifier.ThrowIfNullOrEmpty(store, nameof(store));
             ArgumentVerifier.ThrowIfNull(client, nameof(client));
@@ -123,6 +165,7 @@ public DaprSecretStoreConfigurationProvider(string store, bool normalizeKey, ILi
             this.keyDelimiters = keyDelimiters;
             this.metadata = metadata;
             this.client = client;
+            this.sidecarWaitTimeout = sidecarWaitTimeout;
         }
 
         private string NormalizeKey(string key)
@@ -144,6 +187,12 @@ private async Task LoadAsync()
         {
             var data = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase);
 
+            // Wait for the Dapr Sidecar to report healthy before attempting to fetch secrets.
+            using (var tokenSource = new CancellationTokenSource(sidecarWaitTimeout))
+            {
+                await client.WaitForSidecarAsync(tokenSource.Token);
+            }
+
             if (secretDescriptors != null)
             {
                 foreach (var secretDescriptor in secretDescriptors)