diff --git a/.github/workflows/templates-build-push-image.yml b/.github/workflows/templates-build-push-image.yml index 60a8380f8..fc7a2afc6 100644 --- a/.github/workflows/templates-build-push-image.yml +++ b/.github/workflows/templates-build-push-image.yml @@ -50,4 +50,4 @@ jobs: context: ./src/ file: ./src/${{ inputs.project_name }}/Dockerfile.linux tags: ${{ env.image_commit_uri }},${{ env.image_latest_uri }} - push: true + push: true \ No newline at end of file diff --git a/src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs b/src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs index 658c47466..59c3e7abc 100644 --- a/src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs +++ b/src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs @@ -15,6 +15,7 @@ using Promitor.Core.Extensions; using Promitor.Core.Metrics.Interfaces; using Promitor.Core.Metrics.Sinks; +using Promitor.Core.Scraping.Batching; using Promitor.Core.Scraping.Configuration.Model; using Promitor.Core.Scraping.Configuration.Model.Metrics; using Promitor.Core.Scraping.Factories; @@ -134,7 +135,6 @@ public async Task ExecuteAsync(CancellationToken cancellationToken) try { var scrapeDefinitions = await GetAllScrapeDefinitions(cancellationToken); - await ScrapeMetrics(scrapeDefinitions, cancellationToken); } catch (OperationCanceledException) @@ -251,22 +251,59 @@ private void GetResourceScrapeDefinition(IAzureResourceDefinition resourceDefini } private async Task ScrapeMetrics(IEnumerable> scrapeDefinitions, CancellationToken cancellationToken) - { + { var tasks = new List(); + var batchScrapingEnabled = this._azureMonitorIntegrationConfiguration.Value.MetricsBatching?.Enabled ?? false; + if (batchScrapingEnabled) { + Logger.LogInformation("Promitor Scraper with operate in batch scraping mode, with max batch size {BatchSize}", this._azureMonitorIntegrationConfiguration.Value.MetricsBatching.MaxBatchSize); + Logger.LogWarning("Batch scraping is an experimental feature. See Promitor.io for its limitations and cost considerations"); + + var batchScrapeDefinitions = AzureResourceDefinitionBatching.GroupScrapeDefinitions(scrapeDefinitions, this._azureMonitorIntegrationConfiguration.Value.MetricsBatching.MaxBatchSize); + + foreach(var batchScrapeDefinition in batchScrapeDefinitions) { + var azureMetricName = batchScrapeDefinition.ScrapeDefinitionBatchProperties.AzureMetricConfiguration.MetricName; + var resourceType = batchScrapeDefinition.ScrapeDefinitionBatchProperties.ResourceType; + Logger.LogInformation("Executing batch scrape job of size {BatchSize} for Azure Metric {AzureMetricName} for resource type {ResourceType}.", batchScrapeDefinition.ScrapeDefinitions.Count, azureMetricName, resourceType); + await ScheduleLimitedConcurrencyAsyncTask(tasks, () => ScrapeMetricBatched(batchScrapeDefinition), cancellationToken); + } + } else { + foreach (var scrapeDefinition in scrapeDefinitions) + { + cancellationToken.ThrowIfCancellationRequested(); - foreach (var scrapeDefinition in scrapeDefinitions) - { - cancellationToken.ThrowIfCancellationRequested(); - - var metricName = scrapeDefinition.PrometheusMetricDefinition.Name; - var resourceType = scrapeDefinition.Resource.ResourceType; - Logger.LogInformation("Scraping {MetricName} for resource type {ResourceType}.", metricName, resourceType); + var metricName = scrapeDefinition.PrometheusMetricDefinition.Name; + var resourceType = scrapeDefinition.Resource.ResourceType; + Logger.LogInformation("Scraping {MetricName} for resource type {ResourceType}.", metricName, resourceType); - await ScheduleLimitedConcurrencyAsyncTask(tasks, () => ScrapeMetric(scrapeDefinition), cancellationToken); + await ScheduleLimitedConcurrencyAsyncTask(tasks, () => ScrapeMetric(scrapeDefinition), cancellationToken); + } } await Task.WhenAll(tasks); } + private async Task ScrapeMetricBatched(BatchScrapeDefinition batchScrapeDefinition) { + try + { + var resourceSubscriptionId = batchScrapeDefinition.ScrapeDefinitionBatchProperties.SubscriptionId; + var azureMonitorClient = _azureMonitorClientFactory.CreateIfNotExists(_metricsDeclaration.AzureMetadata.Cloud, _metricsDeclaration.AzureMetadata.TenantId, + resourceSubscriptionId, _metricSinkWriter, _azureScrapingSystemMetricsPublisher, _resourceMetricDefinitionMemoryCache, _configuration, + _azureMonitorIntegrationConfiguration, _azureMonitorLoggingConfiguration, _loggerFactory); + var azureEnvironent = _metricsDeclaration.AzureMetadata.Cloud.GetAzureEnvironment(); + + var tokenCredential = AzureAuthenticationFactory.GetTokenCredential(azureEnvironent.ManagementEndpoint, _metricsDeclaration.AzureMetadata.TenantId, + AzureAuthenticationFactory.GetConfiguredAzureAuthentication(_configuration), new Uri(_metricsDeclaration.AzureMetadata.Cloud.GetAzureEnvironment().AuthenticationEndpoint)); + var logAnalyticsClient = new LogAnalyticsClient(_loggerFactory, azureEnvironent, tokenCredential); + + var scraper = _metricScraperFactory.CreateScraper(batchScrapeDefinition.ScrapeDefinitionBatchProperties.ResourceType, _metricSinkWriter, _azureScrapingSystemMetricsPublisher, azureMonitorClient, logAnalyticsClient); + + await scraper.BatchScrapeAsync(batchScrapeDefinition); + } + catch (Exception ex) + { + Logger.LogError(ex, "Failed to scrape metric {MetricName} for resource batch {ResourceName}. Details: {Details}", + batchScrapeDefinition.ScrapeDefinitionBatchProperties.PrometheusMetricDefinition.Name, batchScrapeDefinition.ScrapeDefinitionBatchProperties.ResourceType, ex.ToString()); + } + } private async Task ScrapeMetric(ScrapeDefinition scrapeDefinition) { @@ -287,6 +324,7 @@ private async Task ScrapeMetric(ScrapeDefinition scrap var logAnalyticsClient = new LogAnalyticsClient(_loggerFactory, azureEnvironent, tokenCredential); var scraper = _metricScraperFactory.CreateScraper(scrapeDefinition.Resource.ResourceType, _metricSinkWriter, _azureScrapingSystemMetricsPublisher, azureMonitorClient, logAnalyticsClient); + await scraper.ScrapeAsync(scrapeDefinition); } catch (Exception ex) diff --git a/src/Promitor.Core.Scraping/AzureMonitorScraper.cs b/src/Promitor.Core.Scraping/AzureMonitorScraper.cs index 81b143e9a..c9d2ce9d2 100644 --- a/src/Promitor.Core.Scraping/AzureMonitorScraper.cs +++ b/src/Promitor.Core.Scraping/AzureMonitorScraper.cs @@ -1,10 +1,13 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Collections.Immutable; using System.Linq; using System.Threading.Tasks; using GuardNet; using Microsoft.Extensions.Logging; using Promitor.Core.Contracts; +using Promitor.Core.Extensions; using Promitor.Core.Metrics; using Promitor.Core.Scraping.Configuration.Model; using Promitor.Core.Scraping.Configuration.Model.Metrics; @@ -18,13 +21,19 @@ namespace Promitor.Core.Scraping /// Type of metric definition that is being used public abstract class AzureMonitorScraper : Scraper where TResourceDefinition : class, IAzureResourceDefinition - { + { + /// + /// A cache to store resource definitions. Used to hydrate resource info from resource ID, when processing batch query results + /// + private readonly ConcurrentDictionary> _resourceDefinitions; // using a dictionary for now since IMemoryCache involves layers of injection + /// /// Constructor /// protected AzureMonitorScraper(ScraperConfiguration scraperConfiguration) : base(scraperConfiguration) { + _resourceDefinitions = new ConcurrentDictionary>(); } /// @@ -73,6 +82,69 @@ protected override async Task ScrapeResourceAsync(string subscript return new ScrapeResult(subscriptionId, scrapeDefinition.ResourceGroupName, resourceDefinition.ResourceName, resourceUri, finalMetricValues, metricLabels); } + protected override async Task> BatchScrapeResourceAsync(string subscriptionId, BatchScrapeDefinition batchScrapeDefinition, PromitorMetricAggregationType aggregationType, TimeSpan aggregationInterval) + { + Guard.NotNull(batchScrapeDefinition, nameof(batchScrapeDefinition)); + Guard.NotLessThan(batchScrapeDefinition.ScrapeDefinitions.Count(), 1, nameof(batchScrapeDefinition)); + Guard.NotNull(batchScrapeDefinition.ScrapeDefinitionBatchProperties.AzureMetricConfiguration, nameof(batchScrapeDefinition.ScrapeDefinitionBatchProperties.AzureMetricConfiguration)); + + var metricName = batchScrapeDefinition.ScrapeDefinitionBatchProperties.AzureMetricConfiguration.MetricName; + + // Build list of resource URIs based on definitions in the batch + var resourceUriList = new List(); + foreach (ScrapeDefinition scrapeDefinition in batchScrapeDefinition.ScrapeDefinitions) + { + var resourceUri = $"/{BuildResourceUri(subscriptionId, scrapeDefinition, (TResourceDefinition) scrapeDefinition.Resource)}"; + resourceUriList.Add(resourceUri); + // cache resource info + // the TResourceDefinition resource definition attached to scrape definition can sometimes missing some attributes, need to them in here + var resourceDefinitionToCache = new AzureResourceDefinition + ( + resourceType: scrapeDefinition.Resource.ResourceType, + resourceGroupName: scrapeDefinition.ResourceGroupName, + subscriptionId: scrapeDefinition.SubscriptionId, + resourceName: scrapeDefinition.Resource.ResourceName + ); + _resourceDefinitions.AddOrUpdate(resourceUri, new Tuple(resourceDefinitionToCache, (TResourceDefinition)scrapeDefinition.Resource), (newTuple, oldTuple) => oldTuple); + } + + var metricLimit = batchScrapeDefinition.ScrapeDefinitionBatchProperties.AzureMetricConfiguration.Limit; + var dimensionNames = DetermineMetricDimensions(metricName, (TResourceDefinition) batchScrapeDefinition.ScrapeDefinitions[0].Resource, batchScrapeDefinition.ScrapeDefinitionBatchProperties.AzureMetricConfiguration); // TODO: resource definition doesn't seem to be used, can we remove it from function signature? + + var resourceIdTaggedMeasuredMetrics = new List(); + try + { + // Query Azure Monitor for metrics + resourceIdTaggedMeasuredMetrics = await AzureMonitorClient.BatchQueryMetricAsync(metricName, dimensionNames, aggregationType, aggregationInterval, resourceUriList, null, metricLimit); + } + catch (MetricInformationNotFoundException metricsNotFoundException) + { + Logger.LogWarning("No metric information found for metric {MetricName} with dimensions {MetricDimensions}. Details: {Details}", metricsNotFoundException.Name, metricsNotFoundException.Dimensions, metricsNotFoundException.Details); + + var measuredMetric = dimensionNames.Count > 0 + ? MeasuredMetric.CreateForDimensions(dimensionNames) + : MeasuredMetric.CreateWithoutDimensions(null); + resourceIdTaggedMeasuredMetrics.Add(measuredMetric.WithResourceIdAssociation(null)); + } + + var scrapeResults = new List(); + // group based on resource, then do enrichment per group + var groupedMeasuredMetrics = resourceIdTaggedMeasuredMetrics.GroupBy(measuredMetric => measuredMetric.ResourceId); + foreach (IGrouping resourceMetricsGroup in groupedMeasuredMetrics) + { + var resourceId = resourceMetricsGroup.Key; + if (_resourceDefinitions.TryGetValue(resourceId, out Tuple resourceDefinitionTuple)) + { + var resourceDefinition = resourceDefinitionTuple.Item1; + var metricLabels = DetermineMetricLabels(resourceDefinitionTuple.Item2); + var finalMetricValues = EnrichMeasuredMetrics(resourceDefinitionTuple.Item2, dimensionNames, resourceMetricsGroup.ToImmutableList()); + scrapeResults.Add(new ScrapeResult(subscriptionId, resourceDefinition.ResourceGroupName, resourceDefinition.ResourceName, resourceId, finalMetricValues, metricLabels)); + } + } + + return scrapeResults; + } + private int? DetermineMetricLimit(ScrapeDefinition scrapeDefinition) { return scrapeDefinition.AzureMetricConfiguration.Limit; @@ -89,9 +161,9 @@ protected override async Task ScrapeResourceAsync(string subscript /// List of names of the specified dimensions provided by the scraper. /// Measured metric values that were found /// - protected virtual List EnrichMeasuredMetrics(TResourceDefinition resourceDefinition, List dimensionNames, List metricValues) + protected virtual List EnrichMeasuredMetrics(TResourceDefinition resourceDefinition, List dimensionNames, IReadOnlyList metricValues) { - return metricValues; + return metricValues.ToList(); } /// diff --git a/src/Promitor.Core.Scraping/Batching/AzureResourceDefinitionBatching.cs b/src/Promitor.Core.Scraping/Batching/AzureResourceDefinitionBatching.cs new file mode 100644 index 000000000..c5d5eec6e --- /dev/null +++ b/src/Promitor.Core.Scraping/Batching/AzureResourceDefinitionBatching.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using System.Linq; +using GuardNet; +using Promitor.Core.Contracts; +using Promitor.Core.Scraping.Configuration.Model.Metrics; + +namespace Promitor.Core.Scraping.Batching +{ + public static class AzureResourceDefinitionBatching + { + /// + /// groups scrape definitions based on following conditions: + /// 1. Definitions in a batch must target the same resource type + /// 2. Definitions in a batch must target the same Azure metric with identical dimensions + /// 3. Definitions in a batch must have the same time granularity + /// 4. Batch size cannot exceed configured maximum + /// + /// + public static List> GroupScrapeDefinitions(IEnumerable> allScrapeDefinitions, int maxBatchSize) + { + // ReSharper disable PossibleMultipleEnumeration + Guard.NotNull(allScrapeDefinitions, nameof(allScrapeDefinitions)); + + return allScrapeDefinitions.GroupBy(def => def.BuildScrapingBatchInfo()) + .ToDictionary(group => group.Key, group => group.ToList()) // first pass to build batches that could exceed max + .ToDictionary(group => group.Key, group => SplitScrapeDefinitionBatch(group.Value, maxBatchSize)) // split to right-sized batches + .SelectMany(group => group.Value.Select(batch => new BatchScrapeDefinition(group.Key, batch))) + .ToList(); // flatten + } + + /// + /// splits the "raw" batch according to max batch size configured + /// + private static List>> SplitScrapeDefinitionBatch(List> batchToSplit, int maxBatchSize) + { + // ReSharper disable PossibleMultipleEnumeration + Guard.NotNull(batchToSplit, nameof(batchToSplit)); + + int numNewGroups = ((batchToSplit.Count - 1) / maxBatchSize) + 1; + + return Enumerable.Range(0, numNewGroups) + .Select(i => batchToSplit.Skip(i * maxBatchSize).Take(maxBatchSize).ToList()) + .ToList(); + } + } +} \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/Configuration/Model/AzureMetricConfiguration.cs b/src/Promitor.Core.Scraping/Configuration/Model/AzureMetricConfiguration.cs index f1bac2974..3e9e1514b 100644 --- a/src/Promitor.Core.Scraping/Configuration/Model/AzureMetricConfiguration.cs +++ b/src/Promitor.Core.Scraping/Configuration/Model/AzureMetricConfiguration.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; namespace Promitor.Core.Scraping.Configuration.Model { @@ -45,5 +46,29 @@ public class AzureMetricConfiguration } return Dimensions?.Any(dimension => dimension.Name.Equals(dimensionName, StringComparison.InvariantCultureIgnoreCase)); } + + // A unique string to represent this Azure metric and its configured dimensions + public string ToUniqueStringRepresentation() + { + StringBuilder sb = new StringBuilder(); + sb.Append(MetricName); + if (Dimension != null) + { + sb.Append('_'); + sb.Append(Dimension.Name); + } + else if (Dimensions != null) + { + foreach (var dimension in Dimensions) + { + sb.Append('_'); + sb.Append(dimension.Name); + } + } + sb.Append($"_limit{Limit}"); + + + return sb.ToString(); + } } } \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/Configuration/Model/Metrics/BatchScrapeDefinition.cs b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/BatchScrapeDefinition.cs new file mode 100644 index 000000000..d5d9052f8 --- /dev/null +++ b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/BatchScrapeDefinition.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using GuardNet; +using Promitor.Core.Contracts; + +namespace Promitor.Core.Scraping.Configuration.Model.Metrics +{ + /// + /// Defines a batch of ScrapeDefinitions to be executed in a single request + /// Scrape definitions within a batch should share + /// 1. The same resource type + /// 2. The same Azure metric scrape target with identical dimensions + /// 3. The same time granularity + /// 4. The same filters + /// + public class BatchScrapeDefinition where TResourceDefinition : class, IAzureResourceDefinition + { + /// + /// Creates a new instance of the class. + /// + /// Shared Properties Among ScrapeDefinition's in the batch + /// Scape definitions in the batch + public BatchScrapeDefinition(ScrapeDefinitionBatchProperties scrapeDefinitionBatchProperties, List> groupedScrapeDefinitions) + { + Guard.NotNull(groupedScrapeDefinitions, nameof(groupedScrapeDefinitions)); + Guard.NotLessThan(groupedScrapeDefinitions.Count, 1, nameof(groupedScrapeDefinitions)); + Guard.NotNull(scrapeDefinitionBatchProperties, nameof(scrapeDefinitionBatchProperties)); + + ScrapeDefinitionBatchProperties = scrapeDefinitionBatchProperties; + ScrapeDefinitions = groupedScrapeDefinitions; + } + + /// + /// A batch of scrape job definitions to be executed as a single request + /// + public List> ScrapeDefinitions { get; set; } + + public ScrapeDefinitionBatchProperties ScrapeDefinitionBatchProperties { get; set; } + } +} \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ScrapeDefinition.cs b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ScrapeDefinition.cs index 2d743d222..71acf10ba 100644 --- a/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ScrapeDefinition.cs +++ b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ScrapeDefinition.cs @@ -94,5 +94,16 @@ public ScrapeDefinition( } return AzureMetricConfiguration?.Aggregation?.Interval; } + + public ScrapeDefinitionBatchProperties BuildScrapingBatchInfo() { + return new ScrapeDefinitionBatchProperties( + this.AzureMetricConfiguration, + this.LogAnalyticsConfiguration, + this.PrometheusMetricDefinition, + this.Resource.ResourceType, + this.Scraping, + this.SubscriptionId + ); + } } } diff --git a/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ScrapeDefinitionBatchProperties.cs b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ScrapeDefinitionBatchProperties.cs new file mode 100644 index 000000000..d990d7f93 --- /dev/null +++ b/src/Promitor.Core.Scraping/Configuration/Model/Metrics/ScrapeDefinitionBatchProperties.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using GuardNet; +using Promitor.Core.Contracts; + +namespace Promitor.Core.Scraping.Configuration.Model.Metrics +{ + /// + /// Defines properties of a batch of scrape definitions + /// + public class ScrapeDefinitionBatchProperties : IEquatable + { + /// Configuration about the Azure Monitor metric to scrape + /// Configuration about the LogAnalytics resource to scrape + /// The details of the prometheus metric that will be created. + /// The scraping model. + /// Resource type of the batch + /// Specify a subscription to scrape that defers from the default subscription. + public ScrapeDefinitionBatchProperties( + AzureMetricConfiguration azureMetricConfiguration, + LogAnalyticsConfiguration logAnalyticsConfiguration, + PrometheusMetricDefinition prometheusMetricDefinition, + ResourceType resourceType, + Scraping scraping, + string subscriptionId) + { + Guard.NotNull(azureMetricConfiguration, nameof(azureMetricConfiguration)); + Guard.NotNull(prometheusMetricDefinition, nameof(prometheusMetricDefinition)); + Guard.NotNull(scraping, nameof(scraping)); + Guard.NotNull(subscriptionId, nameof(subscriptionId)); + + AzureMetricConfiguration = azureMetricConfiguration; + LogAnalyticsConfiguration = logAnalyticsConfiguration; + PrometheusMetricDefinition = prometheusMetricDefinition; + Scraping = scraping; + SubscriptionId = subscriptionId; + ResourceType = resourceType; + } + + /// + /// Configuration about the Azure Monitor metric to scrape + /// + public AzureMetricConfiguration AzureMetricConfiguration { get; } + + /// + /// Configuration about the Azure Monitor log analytics resource to scrape + /// + public LogAnalyticsConfiguration LogAnalyticsConfiguration { get; } + + /// + /// The details of the prometheus metric that will be created. + /// + public PrometheusMetricDefinition PrometheusMetricDefinition { get; } + + /// + /// The scraping model. + /// + public Scraping Scraping { get; } + + /// + /// The Azure subscription to get the metric from. + /// + public string SubscriptionId { get; } + + /// + /// The Azure resource type shared by all scrape definitions in the batch + /// + public ResourceType ResourceType { get; } + + public TimeSpan? GetAggregationInterval() + { + if (ResourceType == ResourceType.LogAnalytics) + { + return LogAnalyticsConfiguration?.Aggregation?.Interval; + } + return AzureMetricConfiguration?.Aggregation?.Interval; + } + + public override int GetHashCode() + { + return this.BuildBatchHashKey().GetHashCode(); + } + + /// + /// Builds a namespaced string key to satisfy batch restrictions, in the format of + /// (AzureMetricAndDimensionsAndFilter)_(SubscriptionId)_(ResourceType)_(AggregationInterval>) + /// + public string BuildBatchHashKey() + { + return string.Join("_", new List {AzureMetricConfiguration.ToUniqueStringRepresentation(), SubscriptionId, ResourceType.ToString(), GetAggregationInterval().ToString()}); + } + + /// + /// Equality comparison override in case of hash collision + /// + public bool Equals(ScrapeDefinitionBatchProperties other) + { + if (other is null) { + return false; + } + + return ResourceType == other.ResourceType && AzureMetricConfiguration.ToUniqueStringRepresentation() == other.AzureMetricConfiguration.ToUniqueStringRepresentation() && SubscriptionId == other.SubscriptionId && GetAggregationInterval().Equals(other.GetAggregationInterval()); + } + } +} \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/Interfaces/IScraper.cs b/src/Promitor.Core.Scraping/Interfaces/IScraper.cs index 420957123..87b321c69 100644 --- a/src/Promitor.Core.Scraping/Interfaces/IScraper.cs +++ b/src/Promitor.Core.Scraping/Interfaces/IScraper.cs @@ -7,5 +7,6 @@ namespace Promitor.Core.Scraping.Interfaces public interface IScraper where TResourceDefinition : class, IAzureResourceDefinition { Task ScrapeAsync(ScrapeDefinition scrapeDefinition); + Task BatchScrapeAsync(BatchScrapeDefinition batchScrapeDefinition); } } \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/LogAnalyticsScraper.cs b/src/Promitor.Core.Scraping/LogAnalyticsScraper.cs index eadfd90a7..3f32d30a4 100644 --- a/src/Promitor.Core.Scraping/LogAnalyticsScraper.cs +++ b/src/Promitor.Core.Scraping/LogAnalyticsScraper.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using GuardNet; using Promitor.Core.Contracts; @@ -49,5 +50,13 @@ private Dictionary DetermineMetricLabels(LogAnalyticsResourceDef { return new Dictionary { { "workspace_id", resourceDefinition.WorkspaceId }, {"workspace_name", resourceDefinition.WorkspaceName} }; } + + protected override async Task> BatchScrapeResourceAsync(string subscriptionId, BatchScrapeDefinition batchScrapeDefinition, PromitorMetricAggregationType aggregationType, TimeSpan aggregationInterval) + { + var logScrapingTasks = batchScrapeDefinition.ScrapeDefinitions.Select(definition => ScrapeResourceAsync(subscriptionId, definition, (LogAnalyticsResourceDefinition) definition.Resource, aggregationType, aggregationInterval)).ToList(); + + var resultsList = await Task.WhenAll(logScrapingTasks); + return resultsList.ToList(); + } } } \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/ResourceTypes/AzureMessagingScraper.cs b/src/Promitor.Core.Scraping/ResourceTypes/AzureMessagingScraper.cs index 88bb8acac..02ec91cc4 100644 --- a/src/Promitor.Core.Scraping/ResourceTypes/AzureMessagingScraper.cs +++ b/src/Promitor.Core.Scraping/ResourceTypes/AzureMessagingScraper.cs @@ -16,7 +16,7 @@ protected AzureMessagingScraper(ScraperConfiguration scraperConfiguration) { } - protected override List EnrichMeasuredMetrics(TResourceDefinition resourceDefinition, List dimensionNames, List metricValues) + protected override List EnrichMeasuredMetrics(TResourceDefinition resourceDefinition, List dimensionNames, IReadOnlyList metricValues) { // Change Azure Monitor Dimension name to more representable value foreach (var measuredMetric in metricValues.Where(metricValue => metricValue.Dimensions.Any())) @@ -24,7 +24,7 @@ protected override List EnrichMeasuredMetrics(TResourceDefinitio measuredMetric.Dimensions[0].Name = EntityNameLabel; } - return metricValues; + return metricValues.ToList(); } protected override Dictionary DetermineMetricLabels(TResourceDefinition resourceDefinition) diff --git a/src/Promitor.Core.Scraping/ResourceTypes/DataShareScraper.cs b/src/Promitor.Core.Scraping/ResourceTypes/DataShareScraper.cs index c5b26abbf..efe88ec5c 100644 --- a/src/Promitor.Core.Scraping/ResourceTypes/DataShareScraper.cs +++ b/src/Promitor.Core.Scraping/ResourceTypes/DataShareScraper.cs @@ -51,7 +51,7 @@ protected override List DetermineMetricDimensions(string metricName, Dat return new List { dimensionName }; } - protected override List EnrichMeasuredMetrics(DataShareResourceDefinition resourceDefinition, List dimensionNames, List metricValues) + protected override List EnrichMeasuredMetrics(DataShareResourceDefinition resourceDefinition, List dimensionNames, IReadOnlyList metricValues) { // Change Azure Monitor dimension name to more representable value foreach (var dimension in metricValues.SelectMany(measuredMetric => measuredMetric.Dimensions.Where(dimension => (dimension.Name == "ShareName" || dimension.Name == "ShareSubscriptionName")))) @@ -59,7 +59,7 @@ protected override List EnrichMeasuredMetrics(DataShareResourceD dimension.Name = "share_name"; } - return metricValues; + return metricValues.ToList(); } private static string GetMetricFilterFieldName(string metricName) diff --git a/src/Promitor.Core.Scraping/ResourceTypes/StorageQueueScraper.cs b/src/Promitor.Core.Scraping/ResourceTypes/StorageQueueScraper.cs index 9a0d4c32b..62e33eb69 100644 --- a/src/Promitor.Core.Scraping/ResourceTypes/StorageQueueScraper.cs +++ b/src/Promitor.Core.Scraping/ResourceTypes/StorageQueueScraper.cs @@ -60,5 +60,10 @@ protected override string BuildResourceUri(string subscriptionId, ScrapeDefiniti { return string.Format(ResourceUriTemplate, subscriptionId, scrapeDefinition.ResourceGroupName, resource.AccountName); } + + protected override Task> BatchScrapeResourceAsync(string subscriptionId, BatchScrapeDefinition batchScrapeDefinition, PromitorMetricAggregationType aggregationType, TimeSpan aggregationInterval) + { + throw new NotImplementedException("Batch scaping is not possible for storage queue resources"); + } } } \ No newline at end of file diff --git a/src/Promitor.Core.Scraping/Scraper.cs b/src/Promitor.Core.Scraping/Scraper.cs index 9be7f9137..59ccefc3c 100644 --- a/src/Promitor.Core.Scraping/Scraper.cs +++ b/src/Promitor.Core.Scraping/Scraper.cs @@ -98,14 +98,64 @@ public async Task ScrapeAsync(ScrapeDefinition scrapeD } catch (Exception exception) { - Logger.LogCritical(exception, "Failed to scrape resource for metric '{MetricName}'", scrapeDefinition.PrometheusMetricDefinition.Name); + Logger.LogCritical(exception, "Failed to scrape resource for metric '{MetricName}'. Details: {Details}", scrapeDefinition.PrometheusMetricDefinition.Name, exception.ToString()); await ReportScrapingOutcomeAsync(scrapeDefinition, isSuccessful: false); } } + public async Task BatchScrapeAsync(BatchScrapeDefinition batchScrapeDefinition) + { + // would the large volume of JSON be an issue? Can it be handled by the SDK? + if (batchScrapeDefinition == null) + { + throw new ArgumentNullException(nameof(batchScrapeDefinition)); + } + + var aggregationInterval = batchScrapeDefinition.ScrapeDefinitionBatchProperties.GetAggregationInterval(); + if (aggregationInterval == null) + { + throw new ArgumentNullException(nameof(batchScrapeDefinition)); + } + + try + { + var aggregationType = batchScrapeDefinition.ScrapeDefinitionBatchProperties.AzureMetricConfiguration.Aggregation.Type; + var scrapeDefinitions = batchScrapeDefinition.ScrapeDefinitions; + var scrapedMetricResults = await BatchScrapeResourceAsync( + batchScrapeDefinition.ScrapeDefinitionBatchProperties.SubscriptionId, + batchScrapeDefinition, + aggregationType, + aggregationInterval.Value); + + foreach (int i in Enumerable.Range(0, scrapedMetricResults.Count)) + { + var scrapedMetricResult = scrapedMetricResults[i]; + var scrapeDefinition = scrapeDefinitions[i]; + LogMeasuredMetrics(scrapeDefinition, scrapedMetricResult, aggregationInterval); + + await _metricSinkWriter.ReportMetricAsync(scrapeDefinition.PrometheusMetricDefinition.Name, scrapeDefinition.PrometheusMetricDefinition.Description, scrapedMetricResult); + + await ReportBatchScrapingOutcomeAsync(batchScrapeDefinition, isSuccessful: true, batchScrapeDefinition.ScrapeDefinitions.Count) ; + } + } + catch (ErrorResponseException errorResponseException) + { + HandleErrorResponseException(errorResponseException, batchScrapeDefinition.ScrapeDefinitionBatchProperties.PrometheusMetricDefinition.Name); + + await ReportBatchScrapingOutcomeAsync(batchScrapeDefinition, isSuccessful: false, batchScrapeDefinition.ScrapeDefinitions.Count); + } + catch (Exception exception) + { + Logger.LogCritical(exception, "Failed to scrape resource for metric '{MetricName}'", batchScrapeDefinition.ScrapeDefinitionBatchProperties.AzureMetricConfiguration.MetricName); + + await ReportBatchScrapingOutcomeAsync(batchScrapeDefinition, isSuccessful: false, batchScrapeDefinition.ScrapeDefinitions.Count); + } + } + private const string ScrapeSuccessfulMetricDescription = "Provides an indication that the scraping of the resource was successful"; private const string ScrapeErrorMetricDescription = "Provides an indication that the scraping of the resource has failed"; + private const string BatvhSizeMetricDescription = "Provides an indication that the scraping of the resource has failed"; private async Task ReportScrapingOutcomeAsync(ScrapeDefinition scrapeDefinition, bool isSuccessful) { @@ -130,7 +180,7 @@ private async Task ReportScrapingOutcomeAsync(ScrapeDefinition batchScrapeDefinition, bool isSuccessful, int batchSize) + { + // We reset all values, by default + double successfulMetricValue = 0; + double unsuccessfulMetricValue = 0; + + // Based on the result, we reflect that in the metric + if (isSuccessful) + { + successfulMetricValue = 1; + } + else + { + unsuccessfulMetricValue = 1; + } + + // Enrich with context + var labels = new Dictionary + { + {"metric_name", batchScrapeDefinition.ScrapeDefinitionBatchProperties.PrometheusMetricDefinition.Name}, + {"resource_type", batchScrapeDefinition.ScrapeDefinitionBatchProperties.ResourceType.ToString()}, + {"subscription_id", batchScrapeDefinition.ScrapeDefinitionBatchProperties.SubscriptionId}, + {"is_batch", "1"}, + }; + // Report! + await AzureScrapingSystemMetricsPublisher.WriteGaugeMeasurementAsync(RuntimeMetricNames.ScrapeSuccessful, ScrapeSuccessfulMetricDescription, successfulMetricValue, labels); + await AzureScrapingSystemMetricsPublisher.WriteGaugeMeasurementAsync(RuntimeMetricNames.ScrapeError, ScrapeErrorMetricDescription, unsuccessfulMetricValue, labels); + + if (batchSize > 0) + { + await AzureScrapingSystemMetricsPublisher.WriteHistogramMeasurementAsync(RuntimeMetricNames.BatchSize, BatvhSizeMetricDescription, batchSize, labels); + } + } + private void LogMeasuredMetrics(ScrapeDefinition scrapeDefinition, ScrapeResult scrapedMetricResult, TimeSpan? aggregationInterval) { foreach (var measuredMetric in scrapedMetricResult.MetricValues) @@ -218,6 +302,19 @@ protected abstract Task ScrapeResourceAsync( TResourceDefinition resourceDefinition, PromitorMetricAggregationType aggregationType, TimeSpan aggregationInterval); + + /// + /// Scrapes configured resource batch. Should return telemetry for all scrape definitions as a list + /// + /// Metric subscription Id + /// Contains all scrape definitions in the batch and their shared properties(like resource type) + /// Aggregation for the metric to use + /// Interval that is used to aggregate metrics + protected abstract Task> BatchScrapeResourceAsync( + string subscriptionId, + BatchScrapeDefinition batchScrapeDefinition, + PromitorMetricAggregationType aggregationType, + TimeSpan aggregationInterval); /// /// Builds the URI of the resource to scrape diff --git a/src/Promitor.Core/Extensions/AzureCloudExtensions.cs b/src/Promitor.Core/Extensions/AzureCloudExtensions.cs index 6dc2a385a..0980bd118 100644 --- a/src/Promitor.Core/Extensions/AzureCloudExtensions.cs +++ b/src/Promitor.Core/Extensions/AzureCloudExtensions.cs @@ -31,7 +31,7 @@ public static AzureEnvironment GetAzureEnvironment(this AzureCloud azureCloud) } /// - /// Get Azure environment information under legacy SDK model + /// Get Azure environment information for Azure.Monitor SDK single resource queries /// /// Microsoft Azure cloud /// Azure environment information for specified cloud @@ -49,6 +49,26 @@ public static MetricsQueryAudience DetermineMetricsClientAudience(this AzureClou } } + /// + /// Get Azure environment information for Azure.Monitor SDK batch queries + /// + /// Microsoft Azure cloud + /// Azure environment information for specified cloud + public static MetricsClientAudience DetermineMetricsClientBatchQueryAudience(this AzureCloud azureCloud) { + switch (azureCloud) + { + case AzureCloud.Global: + return MetricsClientAudience.AzurePublicCloud; + case AzureCloud.UsGov: + return MetricsClientAudience.AzureGovernment; + case AzureCloud.China: + return MetricsClientAudience.AzureChina; + default: + throw new ArgumentOutOfRangeException(nameof(azureCloud), "No Azure environment is known for"); // Azure.Monitory.Query package does not support any other sovereign regions + } + } + + public static Uri GetAzureAuthorityHost(this AzureCloud azureCloud) { switch (azureCloud) diff --git a/src/Promitor.Core/Extensions/MeasureMetricExtensions.cs b/src/Promitor.Core/Extensions/MeasureMetricExtensions.cs new file mode 100644 index 000000000..a886ab27e --- /dev/null +++ b/src/Promitor.Core/Extensions/MeasureMetricExtensions.cs @@ -0,0 +1,17 @@ +using Promitor.Core.Metrics; + +namespace Promitor.Core.Extensions +{ + public static class MeasuredMetricExtensions + { + /// A time series value + /// Resource ID to associate the metric with + /// Instance of MeasuredMetric subclass with resourceId attached + public static ResourceAssociatedMeasuredMetric WithResourceIdAssociation(this MeasuredMetric measuredMetric, string resourceId) + { + return measuredMetric.IsDimensional + ? new ResourceAssociatedMeasuredMetric(measuredMetric.Value, measuredMetric.Dimensions, resourceId) + : new ResourceAssociatedMeasuredMetric(measuredMetric.Value, resourceId); + } + } +} \ No newline at end of file diff --git a/src/Promitor.Core/Metrics/AggregatedSystemMetricsPublisher.cs b/src/Promitor.Core/Metrics/AggregatedSystemMetricsPublisher.cs index 597ec1ed9..735a39be2 100644 --- a/src/Promitor.Core/Metrics/AggregatedSystemMetricsPublisher.cs +++ b/src/Promitor.Core/Metrics/AggregatedSystemMetricsPublisher.cs @@ -30,5 +30,23 @@ public async Task WriteGaugeMeasurementAsync (string name, string description, d await metricCollector.WriteGaugeMeasurementAsync(name, description, value, labels, includeTimestamp); } } + + public async Task WriteHistogramMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp) + { + if (_metricSinks == null) + { + return; + } + + foreach (var metricCollector in _metricSinks) + { + if (metricCollector == null) + { + continue; + } + + await metricCollector.WriteHistogramMeasurementAsync(name, description, value, labels, includeTimestamp); + } + } } } diff --git a/src/Promitor.Core/Metrics/Interfaces/IAzureScrapingSystemMetricsPublisher.cs b/src/Promitor.Core/Metrics/Interfaces/IAzureScrapingSystemMetricsPublisher.cs index e723c765f..42110cc2c 100644 --- a/src/Promitor.Core/Metrics/Interfaces/IAzureScrapingSystemMetricsPublisher.cs +++ b/src/Promitor.Core/Metrics/Interfaces/IAzureScrapingSystemMetricsPublisher.cs @@ -13,5 +13,14 @@ public interface IAzureScrapingSystemMetricsPublisher : ISystemMetricsPublisher /// New measured value /// Labels that are applicable for this measurement Task WriteGaugeMeasurementAsync(string name, string description, double value, Dictionary labels); + + /// + /// Records a histogram value + /// + /// Name of the metric + /// Description of the metric + /// New measured value + /// Labels that are applicable for this measurement + Task WriteHistogramMeasurementAsync(string name, string description, double value, Dictionary labels); } } \ No newline at end of file diff --git a/src/Promitor.Core/Metrics/Interfaces/ISystemMetricsPublisher.cs b/src/Promitor.Core/Metrics/Interfaces/ISystemMetricsPublisher.cs index d4e5a07b2..55228c16d 100644 --- a/src/Promitor.Core/Metrics/Interfaces/ISystemMetricsPublisher.cs +++ b/src/Promitor.Core/Metrics/Interfaces/ISystemMetricsPublisher.cs @@ -14,5 +14,15 @@ public interface ISystemMetricsPublisher /// Labels that are applicable for this measurement /// Indication whether or not a timestamp should be reported Task WriteGaugeMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp); + + /// + /// Records a histogram measurement + /// + /// Name of the metric + /// Description of the metric + /// New measured value + /// Labels that are applicable for this measurement + /// Indication whether or not a timestamp should be reported + Task WriteHistogramMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp); } } diff --git a/src/Promitor.Core/Metrics/Interfaces/ISystemMetricsSink.cs b/src/Promitor.Core/Metrics/Interfaces/ISystemMetricsSink.cs index a32bf6b43..2ec45850f 100644 --- a/src/Promitor.Core/Metrics/Interfaces/ISystemMetricsSink.cs +++ b/src/Promitor.Core/Metrics/Interfaces/ISystemMetricsSink.cs @@ -14,5 +14,15 @@ public interface ISystemMetricsSink /// Labels that are applicable for this measurement /// Indication whether or not a timestamp should be reported Task WriteGaugeMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp); + + /// + /// Records a histogram measurement + /// + /// Name of the metric + /// Description of the metric + /// New measured value + /// Labels that are applicable for this measurement + /// Indication whether or not a timestamp should be reported + Task WriteHistogramMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp); } } diff --git a/src/Promitor.Core/Metrics/MeasuredMetric.cs b/src/Promitor.Core/Metrics/MeasuredMetric.cs index 4e4c6d675..6563b138c 100644 --- a/src/Promitor.Core/Metrics/MeasuredMetric.cs +++ b/src/Promitor.Core/Metrics/MeasuredMetric.cs @@ -25,12 +25,12 @@ public class MeasuredMetric /// public bool IsDimensional { get; } - private MeasuredMetric(double? value) + protected MeasuredMetric(double? value) { Value = value; } - private MeasuredMetric(double? value, List dimensions) + protected MeasuredMetric(double? value, List dimensions) { Guard.NotAny(dimensions, nameof(dimensions)); diff --git a/src/Promitor.Core/Metrics/ResourceAssociatedMeasuredMetric.cs b/src/Promitor.Core/Metrics/ResourceAssociatedMeasuredMetric.cs new file mode 100644 index 000000000..c2bf88895 --- /dev/null +++ b/src/Promitor.Core/Metrics/ResourceAssociatedMeasuredMetric.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; + +namespace Promitor.Core.Metrics +{ + /// + /// A subclass of MeasuredMetric model to be used in batch query settings, where metrics across many resources are mixed together in the response. + /// The ResourceId attribute allows grouping/tagging by resource IDs during processing + /// + public class ResourceAssociatedMeasuredMetric : MeasuredMetric + { + /// + /// resourceId associated with this metric + /// + public string ResourceId { get; } + + + public ResourceAssociatedMeasuredMetric(double? value, string resourceId) : base(value) + { + ResourceId = resourceId; + } + + public ResourceAssociatedMeasuredMetric(double? value, List dimensions, string resourceId) : base(value, dimensions) + { + ResourceId = resourceId; + } + } +} \ No newline at end of file diff --git a/src/Promitor.Core/Promitor.Core.csproj b/src/Promitor.Core/Promitor.Core.csproj index 38f35f074..4a4120d4d 100644 --- a/src/Promitor.Core/Promitor.Core.csproj +++ b/src/Promitor.Core/Promitor.Core.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/Promitor.Core/RuntimeMetricNames.cs b/src/Promitor.Core/RuntimeMetricNames.cs index 1d7b44e5b..75bebddfd 100644 --- a/src/Promitor.Core/RuntimeMetricNames.cs +++ b/src/Promitor.Core/RuntimeMetricNames.cs @@ -8,5 +8,6 @@ public static class RuntimeMetricNames public static string ResourceGraphThrottled => "promitor_ratelimit_resource_graph_throttled"; public static string ScrapeSuccessful => "promitor_scrape_success"; public static string ScrapeError => "promitor_scrape_error"; + public static string BatchSize => "promitor_batch_size"; } } \ No newline at end of file diff --git a/src/Promitor.Integrations.AzureMonitor/AzureMonitorQueryClient.cs b/src/Promitor.Integrations.AzureMonitor/AzureMonitorQueryClient.cs index 17c6754f8..68253287a 100644 --- a/src/Promitor.Integrations.AzureMonitor/AzureMonitorQueryClient.cs +++ b/src/Promitor.Integrations.AzureMonitor/AzureMonitorQueryClient.cs @@ -6,7 +6,6 @@ using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Promitor.Core; using Promitor.Core.Metrics; using Promitor.Core.Metrics.Interfaces; using Promitor.Core.Metrics.Sinks; @@ -23,6 +22,7 @@ using Promitor.Core.Extensions; using Azure.Core.Diagnostics; using System.Diagnostics.Tracing; +using Promitor.Integrations.AzureMonitor.Extensions; namespace Promitor.Integrations.AzureMonitor { @@ -30,7 +30,8 @@ public class AzureMonitorQueryClient : IAzureMonitorClient { private readonly IOptions _azureMonitorIntegrationConfiguration; private readonly TimeSpan _metricDefinitionCacheDuration = TimeSpan.FromHours(1); - private readonly MetricsQueryClient _metricsQueryClient; + private readonly MetricsQueryClient _metricsQueryClient; // for single resource queries + private readonly MetricsClient _metricsBatchQueryClient; // for batch queries private readonly IMemoryCache _resourceMetricDefinitionMemoryCache; private readonly ILogger _logger; @@ -60,6 +61,10 @@ public AzureMonitorQueryClient(AzureCloud azureCloud, string tenantId, string su _azureMonitorIntegrationConfiguration = azureMonitorIntegrationConfiguration; _logger = loggerFactory.CreateLogger(); _metricsQueryClient = CreateAzureMonitorMetricsClient(azureCloud, tenantId, subscriptionId, azureAuthenticationInfo, metricSinkWriter, azureScrapingSystemMetricsPublisher, azureMonitorLoggingConfiguration); + if (_azureMonitorIntegrationConfiguration.Value.MetricsBatching.Enabled) + { + _metricsBatchQueryClient = CreateAzureMonitorMetricsBatchClient(azureCloud, tenantId, azureAuthenticationInfo, azureMonitorIntegrationConfiguration, azureMonitorLoggingConfiguration); + } } /// @@ -81,13 +86,13 @@ public async Task> QueryMetricAsync(string metricName, List // Get all metrics var startQueryingTime = DateTime.UtcNow; - var metricNamespaces = await GetMetricNamespacesAsync(resourceId); + var metricNamespaces = await _metricsQueryClient.GetAndCacheMetricNamespacesAsync(resourceId, _resourceMetricDefinitionMemoryCache, _metricDefinitionCacheDuration); var metricNamespace = metricNamespaces.SingleOrDefault(); if (metricNamespace == null) { throw new MetricNotFoundException(metricName); } - var metricsDefinitions = await GetMetricDefinitionsAsync(resourceId, metricNamespace); + var metricsDefinitions = await _metricsQueryClient.GetAndCacheMetricDefinitionsAsync(resourceId, metricNamespace, _resourceMetricDefinitionMemoryCache, _metricDefinitionCacheDuration); var metricDefinition = metricsDefinitions.SingleOrDefault(definition => definition.Name.ToUpper() == metricName.ToUpper()); if (metricDefinition == null) { @@ -97,8 +102,50 @@ public async Task> QueryMetricAsync(string metricName, List var closestAggregationInterval = DetermineAggregationInterval(metricName, aggregationInterval, metricDefinition.MetricAvailabilities); // Get the most recent metric - var metricResult = await GetRelevantMetric(resourceId, metricName, MetricAggregationTypeConverter.AsMetricAggregationType(aggregationType), closestAggregationInterval, metricFilter, metricDimensions, metricLimit, startQueryingTime); + var metricResult = await _metricsQueryClient.GetRelevantMetricSingleResource(resourceId, metricName, MetricAggregationTypeConverter.AsMetricAggregationType(aggregationType), closestAggregationInterval, metricFilter, metricDimensions, metricLimit, startQueryingTime, _azureMonitorIntegrationConfiguration); + return ProcessMetricResult(metricResult, metricName, startQueryingTime, closestAggregationInterval, aggregationType, metricDimensions); + } + + public async Task> BatchQueryMetricAsync(string metricName, List metricDimensions, PromitorMetricAggregationType aggregationType, TimeSpan aggregationInterval, + ListresourceIds, string metricFilter = null, int? metricLimit = null) + { + Guard.NotNullOrWhitespace(metricName, nameof(metricName)); + Guard.NotLessThan(resourceIds.Count(), 1, nameof(resourceIds)); + Guard.NotNull(_metricsBatchQueryClient, nameof(_metricsBatchQueryClient)); + + // Get all metrics + var startQueryingTime = DateTime.UtcNow; + var metricNamespaces = await _metricsQueryClient.GetAndCacheMetricNamespacesAsync(resourceIds.First(), _resourceMetricDefinitionMemoryCache, _metricDefinitionCacheDuration); + var metricNamespace = metricNamespaces.SingleOrDefault(); + if (metricNamespace == null) + { + throw new MetricNotFoundException(metricName); + } + var metricsDefinitions = await _metricsQueryClient.GetAndCacheMetricDefinitionsAsync(resourceIds.First(), metricNamespace, _resourceMetricDefinitionMemoryCache, _metricDefinitionCacheDuration); + var metricDefinition = metricsDefinitions.SingleOrDefault(definition => definition.Name.ToUpper() == metricName.ToUpper()); + if (metricDefinition == null) + { + throw new MetricNotFoundException(metricName); + } + + var closestAggregationInterval = DetermineAggregationInterval(metricName, aggregationInterval, metricDefinition.MetricAvailabilities); + + // Get the most recent metric + var metricResultsList = await _metricsBatchQueryClient.GetRelevantMetricForResources(resourceIds, metricName, metricNamespace, MetricAggregationTypeConverter.AsMetricAggregationType(aggregationType), closestAggregationInterval, metricFilter, metricDimensions, metricLimit, startQueryingTime, _azureMonitorIntegrationConfiguration, _logger); + + //TODO: This is potentially a lot of results to process in a single thread. Think of ways to utilize additional parallelism + return metricResultsList + .SelectMany(metricResult => ProcessMetricResult(metricResult, metricName, startQueryingTime, closestAggregationInterval, aggregationType, metricDimensions) + .Select(measuredMetric => measuredMetric.WithResourceIdAssociation(metricResult.ParseResourceIdFromResultId()))) + .ToList(); + } + + /// + /// Process metrics query response as time series values using the Promitor data model(MeasuredMetric) + /// + private List ProcessMetricResult(MetricResult metricResult, string metricName, DateTime startQueryingTime, TimeSpan closestAggregationInterval, PromitorMetricAggregationType aggregationType, List metricDimensions) + { var seriesForMetric = metricResult.TimeSeries; if (seriesForMetric.Count < 1) { @@ -125,53 +172,14 @@ public async Task> QueryMetricAsync(string metricName, List } catch (MissingDimensionException e) { - _logger.LogWarning("{MetricName} has return a time series with empty value for {Dimension} and the measurements will be dropped", metricName, e.DimensionName); + _logger.LogWarning("{MetricName} has returned a time series with empty value for {Dimension} and the measurements will be dropped", metricName, e.DimensionName); _logger.LogDebug("The violating time series has content {Details}", JsonConvert.SerializeObject(e.TimeSeries)); } } return measuredMetrics; - } - - private async Task> GetMetricDefinitionsAsync(string resourceId, string metricNamespace) - { - // Get cached metric definitions - if (_resourceMetricDefinitionMemoryCache.TryGetValue(resourceId, out IReadOnlyList metricDefinitions)) - { - return metricDefinitions; - } - var metricsDefinitions = new List(); - await foreach (var definition in _metricsQueryClient.GetMetricDefinitionsAsync(resourceId, metricNamespace)) - { - metricsDefinitions.Add(definition); - } - - // Get from API and cache it - _resourceMetricDefinitionMemoryCache.Set(resourceId, metricsDefinitions, _metricDefinitionCacheDuration); - - return metricsDefinitions; - } - - private async Task> GetMetricNamespacesAsync(string resourceId) - { - // Get cached metric namespaces - var namespaceKey = $"{resourceId}_namespace"; - if (_resourceMetricDefinitionMemoryCache.TryGetValue(namespaceKey, out List metricNamespaces)) - { - return metricNamespaces; - } - var foundMetricNamespaces = new List(); - await foreach (var metricNamespace in _metricsQueryClient.GetMetricNamespacesAsync(resourceId)) - { - foundMetricNamespaces.Add(metricNamespace.FullyQualifiedName); - } - - // Get from API and cache it - _resourceMetricDefinitionMemoryCache.Set(namespaceKey, foundMetricNamespaces, _metricDefinitionCacheDuration); - - return foundMetricNamespaces; - } - + } + private TimeSpan DetermineAggregationInterval(string metricName, TimeSpan requestedAggregationInterval, IReadOnlyList availableMetricPeriods) { // Get perfect match @@ -211,48 +219,6 @@ private static TimeSpan GetClosestAggregationInterval(TimeSpan requestedAggregat return closestAggregationInterval; } - private async Task GetRelevantMetric(string resourceId, string metricName, MetricAggregationType metricAggregation, TimeSpan metricInterval, - string metricFilter, List metricDimensions, int? metricLimit, DateTime recordDateTime) - { - MetricsQueryOptions queryOptions; - var querySizeLimit = metricLimit ?? Defaults.MetricDefaults.Limit; - var historyStartingFromInHours = _azureMonitorIntegrationConfiguration.Value.History.StartingFromInHours; - var filter = BuildFilter(metricDimensions, metricFilter); - - if (!string.IsNullOrEmpty(filter)) - { - queryOptions = new MetricsQueryOptions { - Aggregations = { - metricAggregation - }, - Granularity = metricInterval, - Filter = filter, - Size = querySizeLimit, - TimeRange= new QueryTimeRange(new DateTimeOffset(recordDateTime.AddHours(-historyStartingFromInHours)), new DateTimeOffset(recordDateTime)) - }; - } - else - { - queryOptions = new MetricsQueryOptions { - Aggregations= { - metricAggregation - }, - Granularity = metricInterval, - Size = querySizeLimit, - TimeRange= new QueryTimeRange(new DateTimeOffset(recordDateTime.AddHours(-historyStartingFromInHours)), new DateTimeOffset(recordDateTime)) - }; - } - - var metricsQueryResponse = await _metricsQueryClient.QueryResourceAsync(resourceId, [metricName], queryOptions); - var relevantMetric = metricsQueryResponse.Value.Metrics.SingleOrDefault(var => var.Name.ToUpper() == metricName.ToUpper()); - if (relevantMetric == null) - { - throw new MetricNotFoundException(metricName); - } - - return relevantMetric; - } - private MetricValue GetMostRecentMetricValue(string metricName, MetricTimeSeriesElement timeSeries, DateTimeOffset recordDateTime) { var relevantMetricValue = timeSeries.Values.Where(metricValue => metricValue.TimeStamp < recordDateTime) @@ -286,35 +252,6 @@ private MetricValue GetMostRecentMetricValue(string metricName, MetricTimeSeries throw new Exception($"Unable to determine the metrics value for aggregator '{metricAggregation}'"); } } - - private static string BuildFilter(List metricDimensions, string metricFilter) - { - var filterDictionary = new Dictionary(); - metricDimensions.ForEach(metricDimension => filterDictionary.Add(metricDimension, "'*'")); - - if (string.IsNullOrWhiteSpace(metricFilter) == false) { - var filterConditions = metricFilter.Split(" and ").ToList(); - foreach (string condition in filterConditions) - { - string[] parts = condition.Split(" eq ", StringSplitOptions.None); - if (filterDictionary.ContainsKey(parts[0])) - { - filterDictionary[parts[0]] = parts[1]; - } - else - { - filterDictionary.Add(parts[0].Trim(), parts[1]); - } - } - } - - if (filterDictionary.Count > 0) - { - return string.Join(" and ", filterDictionary.Select(kvp => $"{kvp.Key} eq {kvp.Value}")); - } - return null; - } - /// /// Creates authenticated client to query for metrics /// @@ -336,5 +273,45 @@ private MetricsQueryClient CreateAzureMonitorMetricsClient(AzureCloud azureCloud } return new MetricsQueryClient(tokenCredential, metricsQueryClientOptions); } + + /// + /// Creates authenticated client for metrics batch queries + /// + private MetricsClient CreateAzureMonitorMetricsBatchClient(AzureCloud azureCloud, string tenantId, AzureAuthenticationInfo azureAuthenticationInfo, IOptions azureMonitorIntegrationConfiguration, IOptions azureMonitorLoggingConfiguration) { + var azureRegion = azureMonitorIntegrationConfiguration.Value.MetricsBatching.AzureRegion; + var metricsClientOptions = new MetricsClientOptions{ + Audience = azureCloud.DetermineMetricsClientBatchQueryAudience(), + Retry = + { + Mode = RetryMode.Exponential, + MaxRetries = 3, + Delay = TimeSpan.FromSeconds(1), + MaxDelay = TimeSpan.FromSeconds(30), + } + }; // retry policy as suggested in the documentation: https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/migrate-to-batch-api?tabs=individual-response#529-throttling-errors + var tokenCredential = AzureAuthenticationFactory.GetTokenCredential(nameof(azureCloud), tenantId, azureAuthenticationInfo, azureCloud.GetAzureAuthorityHost()); + metricsClientOptions.AddPolicy(new ModifyOutgoingAzureMonitorRequestsPolicy(_logger), HttpPipelinePosition.BeforeTransport); + var azureMonitorLogging = azureMonitorLoggingConfiguration.Value; + if (azureMonitorLogging.IsEnabled) + { + using AzureEventSourceListener traceListener = AzureEventSourceListener.CreateTraceLogger(EventLevel.Informational); + metricsClientOptions.Diagnostics.IsLoggingEnabled = true; + } + _logger.LogWarning("Using batch scraping API URL: {URL}", InsertRegionIntoUrl(azureRegion, azureCloud.DetermineMetricsClientBatchQueryAudience().ToString())); + return new MetricsClient(new Uri(InsertRegionIntoUrl(azureRegion, azureCloud.DetermineMetricsClientBatchQueryAudience().ToString())), tokenCredential, metricsClientOptions); + } + + public static string InsertRegionIntoUrl(string region, string baseUrl) + { + // Find the position where ".metrics" starts in the URL + int metricsIndex = baseUrl.IndexOf("metrics", StringComparison.Ordinal); + + // Split the base URL into two parts: before and after the ".metrics" + string beforeMetrics = baseUrl.Substring(0, metricsIndex); + string afterMetrics = baseUrl.Substring(metricsIndex); + + // Concatenate the region between the two parts + return $"{beforeMetrics}{region}.{afterMetrics}"; + } } } \ No newline at end of file diff --git a/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorIntegrationConfiguration.cs b/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorIntegrationConfiguration.cs index 577f15979..d053cd18d 100644 --- a/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorIntegrationConfiguration.cs +++ b/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorIntegrationConfiguration.cs @@ -4,5 +4,6 @@ public class AzureMonitorIntegrationConfiguration { public AzureMonitorHistoryConfiguration History { get; set; } = new(); public bool UseAzureMonitorSdk { get; set; } = true; + public AzureMonitorMetricBatchScrapeConfig MetricsBatching { get; set; } = new(); } } \ No newline at end of file diff --git a/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorMetricBatchScrapeConfig.cs b/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorMetricBatchScrapeConfig.cs new file mode 100644 index 000000000..bf7f36bba --- /dev/null +++ b/src/Promitor.Integrations.AzureMonitor/Configuration/AzureMonitorMetricBatchScrapeConfig.cs @@ -0,0 +1,12 @@ +using System.Diagnostics.CodeAnalysis; + +namespace Promitor.Integrations.AzureMonitor.Configuration +{ + public class AzureMonitorMetricBatchScrapeConfig + { + [SuppressMessage("ReSharper", "RedundantDefaultMemberInitializer", Justification = "Explicit initialization to false for better readability")] + public bool Enabled { get; set; } = false; + public int MaxBatchSize { get; set; } + public string AzureRegion { get; set; } // Batch scrape endpoints are deployed by region + } +} diff --git a/src/Promitor.Integrations.AzureMonitor/Extensions/AzureMonitorMetadataTasks.cs b/src/Promitor.Integrations.AzureMonitor/Extensions/AzureMonitorMetadataTasks.cs new file mode 100644 index 000000000..8687103f5 --- /dev/null +++ b/src/Promitor.Integrations.AzureMonitor/Extensions/AzureMonitorMetadataTasks.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Azure.Monitor.Query; +using Azure.Monitor.Query.Models; +using Microsoft.Extensions.Caching.Memory; + +namespace Promitor.Integrations.AzureMonitor.Extensions +{ + public static class AzureMonitorMetadataTasks + { + public static async Task> GetAndCacheMetricDefinitionsAsync(this MetricsQueryClient metricsQueryClient, string resourceId, string metricNamespace, IMemoryCache resourceMetricDefinitionMemoryCache, TimeSpan cacheDuration) + { + // Get cached metric definitions + if (resourceMetricDefinitionMemoryCache.TryGetValue(resourceId, out IReadOnlyList metricDefinitions)) + { + return metricDefinitions; + } + var metricsDefinitions = new List(); + await foreach (var definition in metricsQueryClient.GetMetricDefinitionsAsync(resourceId, metricNamespace)) + { + metricsDefinitions.Add(definition); + } + + // Get from API and cache it + resourceMetricDefinitionMemoryCache.Set(resourceId, metricsDefinitions, cacheDuration); + + return metricsDefinitions; + } + + public static async Task> GetAndCacheMetricNamespacesAsync(this MetricsQueryClient metricsQueryClient, string resourceId, IMemoryCache resourceMetricDefinitionMemoryCache, TimeSpan cacheDuration) + { + // Get cached metric namespaces + var namespaceKey = $"{resourceId}_namespace"; + if (resourceMetricDefinitionMemoryCache.TryGetValue(namespaceKey, out List metricNamespaces)) + { + return metricNamespaces; + } + var foundMetricNamespaces = new List(); + await foreach (var metricNamespace in metricsQueryClient.GetMetricNamespacesAsync(resourceId)) + { + foundMetricNamespaces.Add(metricNamespace.FullyQualifiedName); + } + + // Get from API and cache it + resourceMetricDefinitionMemoryCache.Set(namespaceKey, foundMetricNamespaces, cacheDuration); + + return foundMetricNamespaces; + } + } +} \ No newline at end of file diff --git a/src/Promitor.Integrations.AzureMonitor/Extensions/AzureMonitorQueryTasks.cs b/src/Promitor.Integrations.AzureMonitor/Extensions/AzureMonitorQueryTasks.cs new file mode 100644 index 000000000..0240335e3 --- /dev/null +++ b/src/Promitor.Integrations.AzureMonitor/Extensions/AzureMonitorQueryTasks.cs @@ -0,0 +1,129 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Monitor.Query; +using Azure.Monitor.Query.Models; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Promitor.Core; +using Promitor.Integrations.AzureMonitor.Configuration; +using Promitor.Integrations.AzureMonitor.Exceptions; + +namespace Promitor.Integrations.AzureMonitor.Extensions +{ + public static class AzureMonitorQueryTasks + { + public static async Task GetRelevantMetricSingleResource(this MetricsQueryClient metricsQueryClient, string resourceId, string metricName, MetricAggregationType metricAggregation, TimeSpan metricInterval, + string metricFilter, List metricDimensions, int? metricLimit, DateTime recordDateTime, IOptions azureMonitorIntegrationConfiguration) + { + MetricsQueryOptions queryOptions; + var querySizeLimit = metricLimit ?? Defaults.MetricDefaults.Limit; + var historyStartingFromInHours = azureMonitorIntegrationConfiguration.Value.History.StartingFromInHours; + var filter = BuildFilter(metricDimensions, metricFilter); + + if (!string.IsNullOrEmpty(filter)) + { + queryOptions = new MetricsQueryOptions { + Aggregations = { + metricAggregation + }, + Granularity = metricInterval, + Filter = filter, + Size = querySizeLimit, + TimeRange= new QueryTimeRange(new DateTimeOffset(recordDateTime.AddHours(-historyStartingFromInHours)), new DateTimeOffset(recordDateTime)) + }; + } + else + { + queryOptions = new MetricsQueryOptions { + Aggregations= { + metricAggregation + }, + Granularity = metricInterval, + Size = querySizeLimit, + TimeRange= new QueryTimeRange(new DateTimeOffset(recordDateTime.AddHours(-historyStartingFromInHours)), new DateTimeOffset(recordDateTime)) + }; + } + + var metricsQueryResponse = await metricsQueryClient.QueryResourceAsync(resourceId, [metricName], queryOptions); + return GetRelevantMetricResultOrThrow(metricsQueryResponse.Value, metricName); + } + + public static async Task> GetRelevantMetricForResources(this MetricsClient metricsClient, List resourceIds, string metricName, string metricNamespace, MetricAggregationType metricAggregation, TimeSpan metricInterval, + string metricFilter, List metricDimensions, int? metricLimit, DateTime recordDateTime, IOptions azureMonitorIntegrationConfiguration, ILogger logger) + { + MetricsQueryResourcesOptions queryOptions; + var querySizeLimit = metricLimit ?? Defaults.MetricDefaults.Limit; + //var historyStartingFromInHours = azureMonitorIntegrationConfiguration.Value.History.StartingFromInHours; + var historyStartingFromInHours = 2; + var filter = BuildFilter(metricDimensions, metricFilter); + List resourceIdentifiers = resourceIds.Select(id => new ResourceIdentifier(id)).ToList(); + + if (!string.IsNullOrEmpty(filter)) + { + queryOptions = new MetricsQueryResourcesOptions { + Aggregations = { metricAggregation.ToString().ToLower() }, + Granularity = metricInterval, + Filter = filter, + Size = querySizeLimit, + TimeRange= new QueryTimeRange(new DateTimeOffset(recordDateTime.AddHours(-historyStartingFromInHours)), new DateTimeOffset(recordDateTime)) + }; + } + else + { + queryOptions = new MetricsQueryResourcesOptions { + Aggregations = { metricAggregation.ToString().ToLower() }, + Granularity = metricInterval, + TimeRange= new QueryTimeRange(new DateTimeOffset(recordDateTime.AddHours(-historyStartingFromInHours)), new DateTimeOffset(recordDateTime)) + }; + } + + var metricsBatchQueryResponse = await metricsClient.QueryResourcesAsync(resourceIdentifiers, [metricName], metricNamespace, queryOptions); + var metricsQueryResults = metricsBatchQueryResponse.Value; + return metricsQueryResults.Values + .Select(result => GetRelevantMetricResultOrThrow(result, metricName)) + .ToList(); + } + + private static string BuildFilter(List metricDimensions, string metricFilter) + { + var filterDictionary = new Dictionary(); + metricDimensions.ForEach(metricDimension => filterDictionary.Add(metricDimension, "'*'")); + + if (string.IsNullOrWhiteSpace(metricFilter) == false) { + var filterConditions = metricFilter.Split(" and ").ToList(); + foreach (string condition in filterConditions) + { + string[] parts = condition.Split(" eq ", StringSplitOptions.None); + if (filterDictionary.ContainsKey(parts[0])) + { + filterDictionary[parts[0]] = parts[1]; + } + else + { + filterDictionary.Add(parts[0].Trim(), parts[1]); + } + } + } + + if (filterDictionary.Count > 0) + { + return string.Join(" and ", filterDictionary.Select(kvp => $"{kvp.Key} eq {kvp.Value}")); + } + return null; + } + + private static MetricResult GetRelevantMetricResultOrThrow(MetricsQueryResult metricsQueryResult, string metricName) + { + var relevantMetric = metricsQueryResult.Metrics.SingleOrDefault(var => var.Name.ToUpper() == metricName.ToUpper()); + if (relevantMetric == null) + { + throw new MetricNotFoundException(metricName); + } + + return relevantMetric; + } + } +} \ No newline at end of file diff --git a/src/Promitor.Integrations.AzureMonitor/Extensions/MetricResultExtension.cs b/src/Promitor.Integrations.AzureMonitor/Extensions/MetricResultExtension.cs new file mode 100644 index 000000000..6db3e1c37 --- /dev/null +++ b/src/Promitor.Integrations.AzureMonitor/Extensions/MetricResultExtension.cs @@ -0,0 +1,31 @@ +using Azure.Monitor.Query.Models; + +namespace Promitor.Integrations.AzureMonitor.Extensions +{ + public static class MetricResultExtension + { + /// + /// hacky to to get resource ID since it's not available directly through the SDK model, because the MetricResult model does not have the ResourceID attribute that comes with Response JSON + /// + public static string ParseResourceIdFromResultId(this MetricResult metricResult) + { + return ExtractResourceId(metricResult.Id); + } + + private static string ExtractResourceId(string fullId) + { + // Find the index of the second occurrence of "/providers/" + int firstIndex = fullId.IndexOf("/providers/", System.StringComparison.Ordinal); + int secondIndex = fullId.IndexOf("/providers/", firstIndex + 1, System.StringComparison.Ordinal); + + // If the second "/providers/" is found, slice the string up to that point + if (secondIndex != -1) + { + return fullId[..secondIndex]; + } + + // If not found, return the full string + return fullId; + } + } +} \ No newline at end of file diff --git a/src/Promitor.Integrations.AzureMonitor/HttpPipelinePolicies/ModifyOutgoingAzureMonitorRequestsPolicy.cs b/src/Promitor.Integrations.AzureMonitor/HttpPipelinePolicies/ModifyOutgoingAzureMonitorRequestsPolicy.cs new file mode 100644 index 000000000..4e89a4f64 --- /dev/null +++ b/src/Promitor.Integrations.AzureMonitor/HttpPipelinePolicies/ModifyOutgoingAzureMonitorRequestsPolicy.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.Pipeline; +using GuardNet; +using Microsoft.Extensions.Logging; + +namespace Promitor.Integrations.AzureMonitor.HttpPipelinePolicies{ + /// + /// Work around to make sure range queries work properly. + /// + public class ModifyOutgoingAzureMonitorRequestsPolicy : HttpPipelinePolicy + { + private readonly ILogger _logger; + public ModifyOutgoingAzureMonitorRequestsPolicy(ILogger logger) + { + Guard.NotNull(logger, nameof(logger)); + _logger = logger; + } + + public override async ValueTask ProcessAsync(HttpMessage message, ReadOnlyMemory pipeline) + { + ModifyDateTimeParam(["starttime", "endtime"], message); + await ProcessNextAsync(message, pipeline); + } + + public override void Process(HttpMessage message, ReadOnlyMemory pipeline) + { + throw new NotSupportedException("Synchronous HTTP request path is not supported"); + } + + // ReSharper disable once AssignNullToNotNullAttribute + private void ModifyDateTimeParam(List paramNames, HttpMessage message) + { + // Modify the request URL by updating or adding a query parameter + var uriBuilder = new UriBuilder(message.Request.Uri.ToString()); + var query = System.Web.HttpUtility.ParseQueryString(uriBuilder.Query); + bool queryModified = false; + + foreach (var param in paramNames) + { + if (DateTimeOffset.TryParseExact(query[param], ["MM/dd/yyyy HH:mm:ss zzz", "M/d/yyyy h:mm:ss tt zzz"], CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTimeOffset dateTime)) + { + // Transform to ISO 8601 format (e.g., "2024-09-09T20:46:14") + query[param] = dateTime.ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ"); + queryModified = true; + } + } + if (queryModified) + { + message.Request.Uri.Query = query.ToString(); + } + else + { + _logger.LogWarning("Failed to modify parameters {Parms}", string.Join("and ", paramNames)); + } + } + } +} + diff --git a/src/Promitor.Integrations.AzureMonitor/IAzureMonitorClient.cs b/src/Promitor.Integrations.AzureMonitor/IAzureMonitorClient.cs index 743eef465..e1a742251 100644 --- a/src/Promitor.Integrations.AzureMonitor/IAzureMonitorClient.cs +++ b/src/Promitor.Integrations.AzureMonitor/IAzureMonitorClient.cs @@ -9,5 +9,8 @@ public interface IAzureMonitorClient { public Task> QueryMetricAsync(string metricName, List metricDimensions, PromitorMetricAggregationType aggregationType, TimeSpan aggregationInterval, string resourceId, string metricFilter = null, int? metricLimit = null); + + public Task> BatchQueryMetricAsync(string metricName, List metricDimensions, PromitorMetricAggregationType aggregationType, TimeSpan aggregationInterval, + ListresourceIds, string metricFilter = null, int? metricLimit = null); } } \ No newline at end of file diff --git a/src/Promitor.Integrations.AzureMonitor/LegacyAzureMonitorClient.cs b/src/Promitor.Integrations.AzureMonitor/LegacyAzureMonitorClient.cs index 0a541a60a..f1da8b47d 100644 --- a/src/Promitor.Integrations.AzureMonitor/LegacyAzureMonitorClient.cs +++ b/src/Promitor.Integrations.AzureMonitor/LegacyAzureMonitorClient.cs @@ -65,6 +65,12 @@ public LegacyAzureMonitorClient(AzureEnvironment azureCloud, string tenantId, st _authenticatedAzureSubscription = CreateLegacyAzureClient(azureCloud, tenantId, subscriptionId, azureAuthenticationInfo, loggerFactory, metricSinkWriter, azureScrapingSystemMetricsPublisher, azureMonitorLoggingConfiguration); } + public Task> BatchQueryMetricAsync(string metricName, List metricDimensions, PromitorMetricAggregationType aggregationType, TimeSpan aggregationInterval, + ListresourceIds, string metricFilter = null, int? metricLimit = null) + { + throw new NotSupportedException("Legacy SDK does not support batch queries. Consider migrating to the new Azure.Monitor SDK instead"); + } + /// /// Queries Azure Monitor to get the latest value for a specific metric /// diff --git a/src/Promitor.Integrations.AzureMonitor/Promitor.Integrations.AzureMonitor.csproj b/src/Promitor.Integrations.AzureMonitor/Promitor.Integrations.AzureMonitor.csproj index a6327449e..23d58e9d1 100644 --- a/src/Promitor.Integrations.AzureMonitor/Promitor.Integrations.AzureMonitor.csproj +++ b/src/Promitor.Integrations.AzureMonitor/Promitor.Integrations.AzureMonitor.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/Promitor.Integrations.Sinks.OpenTelemetry/Collectors/OpenTelemetrySystemMetricsSink.cs b/src/Promitor.Integrations.Sinks.OpenTelemetry/Collectors/OpenTelemetrySystemMetricsSink.cs index b446aa786..fc57a5116 100644 --- a/src/Promitor.Integrations.Sinks.OpenTelemetry/Collectors/OpenTelemetrySystemMetricsSink.cs +++ b/src/Promitor.Integrations.Sinks.OpenTelemetry/Collectors/OpenTelemetrySystemMetricsSink.cs @@ -20,5 +20,10 @@ public async Task WriteGaugeMeasurementAsync(string name, string description, do { await _metricSink.ReportMetricAsync(name, description, value, labels); } + + public Task WriteHistogramMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp) + { + return Task.CompletedTask; + } } } diff --git a/src/Promitor.Integrations.Sinks.Prometheus/Collectors/AzureScrapingSystemMetricsPublisher.cs b/src/Promitor.Integrations.Sinks.Prometheus/Collectors/AzureScrapingSystemMetricsPublisher.cs index c5e46b75c..5970a2beb 100644 --- a/src/Promitor.Integrations.Sinks.Prometheus/Collectors/AzureScrapingSystemMetricsPublisher.cs +++ b/src/Promitor.Integrations.Sinks.Prometheus/Collectors/AzureScrapingSystemMetricsPublisher.cs @@ -48,5 +48,21 @@ public async Task WriteGaugeMeasurementAsync(string name, string description, do { await _systemMetricsPublisher.WriteGaugeMeasurementAsync(name, description, value, labels, includeTimestamp); } + + public async Task WriteHistogramMeasurementAsync(string name, string description, double value, Dictionary labels) + { + var enableMetricTimestamps = _prometheusConfiguration.CurrentValue.EnableMetricTimestamps; + + var metricsDeclaration = _metricsDeclarationProvider.Get(applyDefaults: true); + labels.TryAdd("tenant_id", metricsDeclaration.AzureMetadata.TenantId); + + var orderedLabels = labels.OrderByDescending(kvp => kvp.Key).ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + + await _systemMetricsPublisher.WriteHistogramMeasurementAsync(name, description, value, orderedLabels, enableMetricTimestamps); + } + public async Task WriteHistogramMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp) + { + await _systemMetricsPublisher.WriteHistogramMeasurementAsync(name, description, value, labels, includeTimestamp); + } } } diff --git a/src/Promitor.Integrations.Sinks.Prometheus/Collectors/PrometheusSystemMetricsSink.cs b/src/Promitor.Integrations.Sinks.Prometheus/Collectors/PrometheusSystemMetricsSink.cs index c307fd074..239e3330f 100644 --- a/src/Promitor.Integrations.Sinks.Prometheus/Collectors/PrometheusSystemMetricsSink.cs +++ b/src/Promitor.Integrations.Sinks.Prometheus/Collectors/PrometheusSystemMetricsSink.cs @@ -28,6 +28,8 @@ public PrometheusSystemMetricsSink(IMetricFactory metricFactory) /// Indication whether or not a timestamp should be reported public Task WriteGaugeMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp) { + Guard.NotNull(labels, nameof(labels)); + // Order labels alphabetically var orderedLabels = labels.OrderByDescending(kvp => kvp.Key).ToDictionary(kvp => kvp.Key, kvp => kvp.Value); @@ -36,5 +38,23 @@ public Task WriteGaugeMeasurementAsync(string name, string description, double v return Task.CompletedTask; } + + /// + /// Records measurement for a histogram instrument + /// + /// Name of the metric + /// Description of the metric + /// New measured value + /// Labels that are applicable for this measurement + /// Indication whether or not a timestamp should be reported + public Task WriteHistogramMeasurementAsync(string name, string description, double value, Dictionary labels, bool includeTimestamp) + { + Guard.NotNull(labels, nameof(labels)); + var orderedLabels = labels.OrderByDescending(kvp => kvp.Key).ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + + var histogram = _metricFactory.CreateHistogram(name, help: description, includeTimestamp: includeTimestamp, labelNames: orderedLabels.Keys.ToArray(), buckets: [1, 2, 4, 8, 16, 32, 64]); + histogram.WithLabels(orderedLabels.Values.ToArray()).Observe(value); + return Task.CompletedTask; + } } } diff --git a/src/Promitor.Tests.Unit/Configuration/RuntimeConfigurationUnitTest.cs b/src/Promitor.Tests.Unit/Configuration/RuntimeConfigurationUnitTest.cs index 0fb5a064e..88c63ee8b 100644 --- a/src/Promitor.Tests.Unit/Configuration/RuntimeConfigurationUnitTest.cs +++ b/src/Promitor.Tests.Unit/Configuration/RuntimeConfigurationUnitTest.cs @@ -132,6 +132,43 @@ public async Task RuntimeConfiguration_OverrideNewSdkFlagForAzureMonitorIntegrat Assert.False(runtimeConfiguration.AzureMonitor.Integration.UseAzureMonitorSdk); } + [Fact] + public async Task RuntimeConfiguration_HasNoBatchingConfigurationForAzureMonitorIntegration_DefaultsToDisabled() + { + // Arrange + var configuration = await RuntimeConfigurationGenerator.WithServerConfiguration() + .WithAzureMonitorIntegration(useAzureMonitorSdk: null) + .GenerateAsync(); + + // Act + var runtimeConfiguration = configuration.Get(); + + // Assert + Assert.NotNull(runtimeConfiguration); + Assert.NotNull(runtimeConfiguration.AzureMonitor); + Assert.NotNull(runtimeConfiguration.AzureMonitor.Integration); + Assert.False(runtimeConfiguration.AzureMonitor.Integration.MetricsBatching.Enabled); + } + + [Fact] + public async Task RuntimeConfiguration_BatchingConfigurationForAzureMonitorIntegration_UsesConfigured() + { + // Arrange + var configuration = await RuntimeConfigurationGenerator.WithServerConfiguration() + .WithAzureMonitorIntegration(batchSize: 50) + .GenerateAsync(); + + // Act + var runtimeConfiguration = configuration.Get(); + + // Assert + Assert.NotNull(runtimeConfiguration); + Assert.NotNull(runtimeConfiguration.AzureMonitor); + Assert.NotNull(runtimeConfiguration.AzureMonitor.Integration); + Assert.True(runtimeConfiguration.AzureMonitor.Integration.MetricsBatching.Enabled); + Assert.Equal(50, runtimeConfiguration.AzureMonitor.Integration.MetricsBatching.MaxBatchSize); + } + [Theory] [InlineData(true)] diff --git a/src/Promitor.Tests.Unit/Core/Extensions/MeasureMetricExtensionsTests.cs b/src/Promitor.Tests.Unit/Core/Extensions/MeasureMetricExtensionsTests.cs new file mode 100644 index 000000000..65d19d61b --- /dev/null +++ b/src/Promitor.Tests.Unit/Core/Extensions/MeasureMetricExtensionsTests.cs @@ -0,0 +1,20 @@ +using System.ComponentModel; +using Promitor.Core.Extensions; +using Promitor.Core.Metrics; +using Xunit; + +namespace Promitor.Tests.Unit.Core.Extensions +{ + [Category("Unit")] + public class MeasureMetricExtensionsTests + { + [Fact] + public void AssociateWithResourceId() + { + var measuredMetricUnassociated = MeasuredMetric.CreateWithoutDimensions(1); + var resourceId = "/subscriptions/abc/providers/def/test"; + var measuredMetricAssociated = measuredMetricUnassociated.WithResourceIdAssociation(resourceId); + Assert.Equal(resourceId, measuredMetricAssociated.ResourceId); + } + } +} \ No newline at end of file diff --git a/src/Promitor.Tests.Unit/Core/Metrics/ScrapeDefinitionBatchPropertiesTest.cs b/src/Promitor.Tests.Unit/Core/Metrics/ScrapeDefinitionBatchPropertiesTest.cs new file mode 100644 index 000000000..c95cc4479 --- /dev/null +++ b/src/Promitor.Tests.Unit/Core/Metrics/ScrapeDefinitionBatchPropertiesTest.cs @@ -0,0 +1,205 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using AutoMapper; +using Promitor.Core.Metrics; +using Promitor.Core.Scraping.Configuration.Model; +using Promitor.Core.Scraping.Configuration.Model.Metrics; +using Promitor.Core.Scraping.Configuration.Serialization.v1.Mapping; +using Promitor.Core.Scraping.Configuration.Serialization.v1.Model; +using Xunit; + +namespace Promitor.Tests.Unit.Core.Metrics +{ + [Category("Unit")] + public class ScrapeDefinitionBatchPropertiesTest + { + private readonly IMapper _mapper; // to model instantiation happen + private readonly static string azureMetricNameBase = "promitor_batch_test_metric"; + private readonly static PrometheusMetricDefinition prometheusMetricDefinition = + new("promitor_batch_test", "test", new Dictionary()); + private readonly static string subscriptionId = "subscription"; + private readonly static AzureMetricConfigurationV1 azureMetricConfigurationBase = new AzureMetricConfigurationV1 + { + MetricName = azureMetricNameBase, + Aggregation = new MetricAggregationV1 + { + Type = PromitorMetricAggregationType.Average + }, + }; + private readonly static LogAnalyticsConfigurationV1 logAnalyticsConfigurationBase = new LogAnalyticsConfigurationV1 + { + Query = "A eq B", + Aggregation = new AggregationV1 + { + Interval = TimeSpan.FromMinutes(60) + }, + }; + private readonly static ScrapingV1 scrapingBase = new ScrapingV1 + { + Schedule = "5 4 3 2 1" + }; + + public ScrapeDefinitionBatchPropertiesTest() + { + var config = new MapperConfiguration(c => c.AddProfile()); + _mapper = config.CreateMapper(); + } + + [Fact] + public void BuildBatchHashKeySameResultNoDimensions() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + + var scraping = _mapper.Map(scrapingBase); + var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + + var hashCode1 = batchProperties.GetHashCode(); + var hashCode2 = batchProperties2.GetHashCode(); + Assert.Equal(hashCode1, hashCode2); + } + + [Fact] + public void BuildBatchHashKeySameResultIdenticalDimensions() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + azureMetricConfiguration.Dimensions = [new MetricDimension{Name = "Dimension1"}, new MetricDimension{Name = "Dimension2"}]; + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + + var scraping = _mapper.Map(scrapingBase); + + var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + + var hashCode1 = batchProperties.GetHashCode(); + var hashCode2 = batchProperties2.GetHashCode(); + Assert.Equal(hashCode1, hashCode2); + } + + [Fact] + public void BuildBatchHashKeyDifferentResultDifferentDimensions() + { + var azureMetricConfiguration1 = _mapper.Map(azureMetricConfigurationBase); + azureMetricConfiguration1.Dimensions = [new MetricDimension{Name = "Dimension1"}, new MetricDimension{Name = "Dimension2"}]; + var azureMetricConfiguration2 = _mapper.Map(azureMetricConfigurationBase); + azureMetricConfiguration2.Dimensions = [new MetricDimension{Name = "DiffDimension1"}, new MetricDimension{Name = "DiffDimension2"}]; + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + + var scraping = _mapper.Map(scrapingBase); + + var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration1, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration2, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + + var hashCode1 = batchProperties.GetHashCode(); + var hashCode2 = batchProperties2.GetHashCode(); + Assert.NotEqual(hashCode1, hashCode2); + } + + [Fact] + public void BuildBatchHashKeyDifferentResultDifferentMetricName() + { + var azureMetricConfiguration1 = _mapper.Map(azureMetricConfigurationBase); + azureMetricConfiguration1.Dimensions = [new MetricDimension{Name = "Dimension1"}, new MetricDimension{Name = "Dimension2"}]; + var azureMetricConfiguration2 = _mapper.Map(azureMetricConfigurationBase); + azureMetricConfiguration2.Dimensions = [new MetricDimension{Name = "Dimension1"}, new MetricDimension{Name = "Dimension2"}]; + azureMetricConfiguration2.MetricName = "diffName"; + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + + var scraping = _mapper.Map(scrapingBase); + + var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration1, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration2, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + + var hashCode1 = batchProperties.GetHashCode(); + var hashCode2 = batchProperties2.GetHashCode(); + Assert.NotEqual(hashCode1, hashCode2); + } + + [Fact] + public void BuildBatchHashKeyDifferentResultDifferentSubscription() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var scraping = _mapper.Map(scrapingBase); + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + + + var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: "subscription2"); + + var hashCode1 = batchProperties.GetHashCode(); + var hashCode2 = batchProperties2.GetHashCode(); + Assert.NotEqual(hashCode1, hashCode2); + } + + [Fact] + public void BuildBatchHashKeyDifferentResultDifferentResourceType() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var scraping = _mapper.Map(scrapingBase); + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + + var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId); + var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.LoadBalancer, scraping: scraping, subscriptionId: "subscription2"); + + var hashCode1 = batchProperties.GetHashCode(); + var hashCode2 = batchProperties2.GetHashCode(); + Assert.NotEqual(hashCode1, hashCode2); + } + + [Fact] + public void BuildBatchHashKeyDifferentResultDifferentSchedule() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var scraping1 = _mapper.Map(scrapingBase); + var scraping2 = _mapper.Map(scrapingBase); + scraping2.Schedule = "6 4 3 2 1"; + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + + + var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping1, subscriptionId: subscriptionId); + var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping2, subscriptionId: "subscription2"); + + var hashCode1 = batchProperties.GetHashCode(); + var hashCode2 = batchProperties2.GetHashCode(); + Assert.NotEqual(hashCode1, hashCode2); + } + + [Fact] + public void BuildBatchHashKeyTest() + { + AzureMetricConfigurationV1 azureMetricConfigurationTest1 = new AzureMetricConfigurationV1 + { + MetricName = "availabilityResults/availabilityPercentage", + Aggregation = new MetricAggregationV1 + { + Type = PromitorMetricAggregationType.Average + }, + }; + AzureMetricConfigurationV1 azureMetricConfigurationTest2 = new AzureMetricConfigurationV1 + { + MetricName = "availabilityResults/availabilityPercentage", + Dimensions = [new MetricDimensionV1{Name = "availabilityResult/name"}], + Aggregation = new MetricAggregationV1 + { + Type = PromitorMetricAggregationType.Average + }, + }; + var azureMetricConfiguration1 = _mapper.Map(azureMetricConfigurationTest1); + var azureMetricConfiguration2 = _mapper.Map(azureMetricConfigurationTest2); + + var scraping1 = _mapper.Map(scrapingBase); + var scraping2 = _mapper.Map(scrapingBase); + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + + + var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration1, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.ApplicationInsights, scraping: scraping1, subscriptionId: subscriptionId); + var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration2, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.ApplicationInsights, scraping: scraping2, subscriptionId: subscriptionId); + + var hashCode1 = batchProperties.GetHashCode(); + var hashCode2 = batchProperties2.GetHashCode(); + Assert.NotEqual(hashCode1, hashCode2); + } + } +} \ No newline at end of file diff --git a/src/Promitor.Tests.Unit/Core/Scraping/Batching/AzureResourceDefinitionBatchingTests.cs b/src/Promitor.Tests.Unit/Core/Scraping/Batching/AzureResourceDefinitionBatchingTests.cs new file mode 100644 index 000000000..a23989993 --- /dev/null +++ b/src/Promitor.Tests.Unit/Core/Scraping/Batching/AzureResourceDefinitionBatchingTests.cs @@ -0,0 +1,201 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using AutoMapper; +using Promitor.Core.Contracts; +using Promitor.Core.Metrics; +using Promitor.Core.Scraping.Batching; +using Promitor.Core.Scraping.Configuration.Model; +using Promitor.Core.Scraping.Configuration.Model.Metrics; +using Promitor.Core.Scraping.Configuration.Serialization.v1.Mapping; +using Promitor.Core.Scraping.Configuration.Serialization.v1.Model; +using Xunit; + +namespace Promitor.Tests.Unit.Core.Scraping.Batching +{ + [Category("Unit")] + public class AzureResourceDefinitionBatchingTests + { + private readonly IMapper _mapper; // to model instantiation happen + private readonly static string azureMetricNameBase = "promitor_batch_test_metric"; + private readonly static PrometheusMetricDefinition prometheusMetricDefinitionTest = + new("promitor_batch_test", "test", new Dictionary()); + private readonly static string subscriptionIdTest = "subscription"; + private readonly static AzureMetricConfigurationV1 azureMetricConfigurationBase = new AzureMetricConfigurationV1 + { + MetricName = azureMetricNameBase, + Aggregation = new MetricAggregationV1 + { + Type = PromitorMetricAggregationType.Average + }, + }; + private readonly static LogAnalyticsConfigurationV1 logAnalyticsConfigurationBase = new LogAnalyticsConfigurationV1 + { + Query = "A eq B", + Aggregation = new AggregationV1 + { + Interval = TimeSpan.FromMinutes(60) + }, + }; + private readonly static ScrapingV1 scrapingBase = new ScrapingV1 + { + Schedule = "5 4 3 2 1" + }; + private readonly static string resourceGroupNameTest = "batch_test_group"; + private readonly static int batchSize = 50; + + public AzureResourceDefinitionBatchingTests() + { + var config = new MapperConfiguration(c => c.AddProfile()); + _mapper = config.CreateMapper(); + } + + [Fact] + public void IdenticalBatchPropertiesShouldBatchTogether() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var scraping = _mapper.Map(scrapingBase); + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + var scrapeDefinitions = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 10 + ); + var groupedScrapeDefinitions = AzureResourceDefinitionBatching.GroupScrapeDefinitions(scrapeDefinitions, maxBatchSize: batchSize); + // expect one batch of 10 + Assert.Single(groupedScrapeDefinitions); + Assert.Equal(10, groupedScrapeDefinitions[0].ScrapeDefinitions.Count); + } + + [Fact] + public void BatchShouldSplitAccordingToConfiguredBatchSize() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + var testBatchSize = 10; + + var scraping = _mapper.Map(scrapingBase); + var scrapeDefinitions = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 25 + ); + var groupedScrapeDefinitions = AzureResourceDefinitionBatching.GroupScrapeDefinitions(scrapeDefinitions, maxBatchSize: testBatchSize); + // expect three batches adding up to total size + Assert.Equal(3, groupedScrapeDefinitions.Count); + Assert.Equal(25, CountTotalScrapeDefinitions(groupedScrapeDefinitions)); + } + + [Fact] + public void DifferentBatchPropertiesShouldBatchSeparately() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var scraping = _mapper.Map(scrapingBase); + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + var scrapeDefinitions = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 10 + ); + var differentScrapeDefinitions = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.BlobStorage, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 10 + ); + var groupedScrapeDefinitions = AzureResourceDefinitionBatching.GroupScrapeDefinitions([.. scrapeDefinitions, .. differentScrapeDefinitions], maxBatchSize: batchSize); + // expect two batch of 10 each + Assert.Equal(2, groupedScrapeDefinitions.Count); + Assert.Equal(10, groupedScrapeDefinitions[0].ScrapeDefinitions.Count); + Assert.Equal(10, groupedScrapeDefinitions[1].ScrapeDefinitions.Count); + } + + [Fact] + public void DifferentAggregationIntervalsShouldBatchSeparately() + { + var azureMetricConfiguration5MInterval = _mapper.Map(azureMetricConfigurationBase); + azureMetricConfiguration5MInterval.Aggregation.Interval = TimeSpan.FromMinutes(5); + var azureMetricConfiguration2MInterval = _mapper.Map(azureMetricConfigurationBase); + azureMetricConfiguration5MInterval.Aggregation.Interval = TimeSpan.FromMinutes(2); + var logAnalyticsConfiguration = _mapper.Map(logAnalyticsConfigurationBase); + var scraping = _mapper.Map(scrapingBase); + var scrapeDefinitions5M = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration5MInterval, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 10 + ); + var differentScrapeDefinitions2M = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration2MInterval, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.BlobStorage, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 10 + ); + var groupedScrapeDefinitions = AzureResourceDefinitionBatching.GroupScrapeDefinitions([.. scrapeDefinitions5M, .. differentScrapeDefinitions2M], maxBatchSize: batchSize); + // expect two batch of 10 each + Assert.Equal(2, groupedScrapeDefinitions.Count); + Assert.Equal(10, groupedScrapeDefinitions[0].ScrapeDefinitions.Count); + Assert.Equal(10, groupedScrapeDefinitions[1].ScrapeDefinitions.Count); + } + + + [Fact] + public void MixedBatchShouldSplitAccordingToConfiguredBatchSize() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var scraping = _mapper.Map(scrapingBase); + var logAnalyticsConfiguration = new LogAnalyticsConfiguration(); + var scrapeDefinitions = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 130 + ); + var differentScrapeDefinitions = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.BlobStorage, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 120 + ); + var groupedScrapeDefinitions = AzureResourceDefinitionBatching.GroupScrapeDefinitions([.. scrapeDefinitions, .. differentScrapeDefinitions], maxBatchSize: batchSize); + // expect two batch of 10 each + Assert.Equal(6, groupedScrapeDefinitions.Count); + Assert.Equal(250, CountTotalScrapeDefinitions(groupedScrapeDefinitions)); + } + + [Fact] + public void BatchConstructionShouldBeAgnosticToResourceGroup() + { + var azureMetricConfiguration = _mapper.Map(azureMetricConfigurationBase); + var scraping = _mapper.Map(scrapingBase); + var logAnalyticsConfiguration = new LogAnalyticsConfiguration(); + var scrapeDefinitions = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 10 + ); + var differentScrapeDefinitions = BuildScrapeDefinitionBatch( + azureMetricConfiguration: azureMetricConfiguration, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, + resourceType: ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: "group2", 10 + ); + var groupedScrapeDefinitions = AzureResourceDefinitionBatching.GroupScrapeDefinitions([.. scrapeDefinitions, .. differentScrapeDefinitions], maxBatchSize: batchSize); + // expect two batch of 10 each + Assert.Single(groupedScrapeDefinitions); + Assert.Equal(20, groupedScrapeDefinitions[0].ScrapeDefinitions.Count); + } + + private static List> BuildScrapeDefinitionBatch( + AzureMetricConfiguration azureMetricConfiguration, + LogAnalyticsConfiguration logAnalyticsConfiguration, + PrometheusMetricDefinition prometheusMetricDefinition, + Promitor.Core.Scraping.Configuration.Model.Scraping scraping, + ResourceType resourceType, + string subscriptionId, + string resourceGroupName, + int size) + { + // builds a batch of scrape definitions of specified size, each sharing properties passed in through function parameters + var batch = new List>(); + for (var resoureceIdCounter = 0; resoureceIdCounter < size; resoureceIdCounter++) + { + var resourceName = "resource" + resoureceIdCounter.ToString(); + var resourceDefinition = new AzureResourceDefinition(resourceType, subscriptionId, resourceGroupName, resourceName: resourceName, uniqueName: resourceName); + batch.Add(new ScrapeDefinition(azureMetricConfiguration, logAnalyticsConfiguration, prometheusMetricDefinition, scraping, resourceDefinition, subscriptionId, resourceGroupName)); + } + return batch; + } + + private static int CountTotalScrapeDefinitions(List> groupedScrapeDefinitions) + { + var count = 0; + groupedScrapeDefinitions.ForEach(batch => count += batch.ScrapeDefinitions.Count); + return count; + } + } +} \ No newline at end of file diff --git a/src/Promitor.Tests.Unit/Generators/Config/RuntimeConfigurationGenerator.cs b/src/Promitor.Tests.Unit/Generators/Config/RuntimeConfigurationGenerator.cs index 0c4751a2b..245e24045 100644 --- a/src/Promitor.Tests.Unit/Generators/Config/RuntimeConfigurationGenerator.cs +++ b/src/Promitor.Tests.Unit/Generators/Config/RuntimeConfigurationGenerator.cs @@ -235,7 +235,7 @@ public RuntimeConfigurationGenerator WithAzureMonitorLogging(bool isEnabled = tr return this; } - public RuntimeConfigurationGenerator WithAzureMonitorIntegration(int? startingFromInHours = 100, bool? useAzureMonitorSdk = true) + public RuntimeConfigurationGenerator WithAzureMonitorIntegration(int? startingFromInHours = 100, bool? useAzureMonitorSdk = true, int? batchSize = null) { _runtimeConfiguration.AzureMonitor ??= new AzureMonitorConfiguration(); _runtimeConfiguration.AzureMonitor.Integration ??= new AzureMonitorIntegrationConfiguration(); @@ -252,6 +252,11 @@ public RuntimeConfigurationGenerator WithAzureMonitorIntegration(int? startingFr _runtimeConfiguration.AzureMonitor.Integration.UseAzureMonitorSdk = useAzureMonitorSdk.Value; } + if (batchSize != null) + { + _runtimeConfiguration.AzureMonitor.Integration.MetricsBatching = new AzureMonitorMetricBatchScrapeConfig {Enabled = true, MaxBatchSize = batchSize.Value}; + } + return this; } @@ -348,6 +353,9 @@ public async Task GenerateAsync() configurationBuilder.AppendLine($" useAzureMonitorSdk: {_runtimeConfiguration?.AzureMonitor.Integration.UseAzureMonitorSdk}"); configurationBuilder.AppendLine(" history:"); configurationBuilder.AppendLine($" startingFromInHours: {_runtimeConfiguration?.AzureMonitor.Integration.History.StartingFromInHours}"); + configurationBuilder.AppendLine(" metricsBatching:"); + configurationBuilder.AppendLine($" enabled: {_runtimeConfiguration?.AzureMonitor.Integration.MetricsBatching.Enabled}"); + configurationBuilder.AppendLine($" maxBatchSize: {_runtimeConfiguration?.AzureMonitor.Integration.MetricsBatching.MaxBatchSize}"); } if (_runtimeConfiguration?.AzureMonitor.Logging != null)