Skip to content

Commit 2afddd7

Browse files
danielodievichdanielodievich
committed
Added list of metrics and what uses metric
Fixed #20 List of metrics Speeded up environment loading with parallelization
1 parent 0776089 commit 2afddd7

29 files changed

+1488
-413
lines changed

‎Connection/ObserveConnection.cs

Lines changed: 121 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -291,20 +291,32 @@ ... on AuditedObject {
291291
inputRole
292292
__typename
293293
}
294-
metrics {
295-
name
296-
nameWithPath
297-
type
298-
unit
299-
description
300-
rollup
301-
aggregate
302-
interval
303-
suggestedBucketSize
304-
userDefined
305-
state
294+
transform {
295+
current {
296+
query {
297+
outputStage
298+
stages {
299+
id
300+
params
301+
pipeline
302+
layout
303+
input {
304+
inputName
305+
inputRole
306+
datasetId
307+
datasetPath
308+
stageId
309+
__typename
310+
}
311+
__typename
312+
}
313+
layout
314+
__typename
315+
}
316+
__typename
317+
}
306318
__typename
307-
}
319+
}
308320
latencyDesired
309321
effectiveSettings {
310322
dataset {
@@ -797,20 +809,6 @@ ... on AuditedObject {
797809
}
798810
__typename
799811
}
800-
metrics {
801-
name
802-
nameWithPath
803-
type
804-
unit
805-
description
806-
rollup
807-
aggregate
808-
interval
809-
suggestedBucketSize
810-
userDefined
811-
state
812-
__typename
813-
}
814812
latencyDesired
815813
effectiveSettings {
816814
dataset {
@@ -2845,7 +2843,91 @@ public static string executeQueryAndGetData(AuthenticatedUser currentUser, strin
28452843

28462844
#endregion
28472845

2848-
# region Retrieval methods
2846+
#region Metrics Metadata
2847+
2848+
public static string metricsSearch_all(AuthenticatedUser currentUser)
2849+
{
2850+
string graphQLQuery = @"query MetricSearch($workspaces: [ObjectId!], $inDatasets: [ObjectId!], $linkToDatasets: [ObjectId!], $correlationTagMatches: [String!], $match: String!, $heuristicsOptions: MetricHeuristicsOptions) {
2851+
metricSearch(
2852+
workspaces: $workspaces
2853+
inDatasets: $inDatasets
2854+
linkToDatasets: $linkToDatasets
2855+
correlationTagMatches: $correlationTagMatches
2856+
match: $match
2857+
heuristicsOptions: $heuristicsOptions
2858+
) {
2859+
matches {
2860+
datasetId
2861+
metric {
2862+
name
2863+
nameWithPath
2864+
type
2865+
unit
2866+
description
2867+
rollup
2868+
aggregate
2869+
interval
2870+
suggestedBucketSize
2871+
userDefined
2872+
state
2873+
heuristics {
2874+
cardinality
2875+
interval
2876+
intervalStddev
2877+
numOfPoints
2878+
validLinkLabels
2879+
tags {
2880+
path
2881+
column
2882+
__typename
2883+
}
2884+
lastReported
2885+
__typename
2886+
}
2887+
__typename
2888+
}
2889+
__typename
2890+
}
2891+
numSearched
2892+
__typename
2893+
}
2894+
}";
2895+
2896+
JObject queryObject = new JObject();
2897+
queryObject.Add("query", graphQLQuery);
2898+
JObject heuristicsOptionsObject = new JObject();
2899+
heuristicsOptionsObject.Add("globalLimit", "100000");
2900+
heuristicsOptionsObject.Add("perDatasetLimit", "100000");
2901+
JObject variablesObject = new JObject();
2902+
variablesObject.Add("heuristicsOptions", heuristicsOptionsObject);
2903+
variablesObject.Add("match", "");
2904+
queryObject.Add("variables", variablesObject);
2905+
2906+
string queryBody = JSONHelper.getCompactSerializedValueOfObject(queryObject);
2907+
2908+
Tuple<string, List<string>, HttpStatusCode> results = apiPOST(
2909+
currentUser.CustomerEnvironmentUrl,
2910+
"v1/meta",
2911+
"application/json",
2912+
queryBody,
2913+
"application/json",
2914+
currentUser.CustomerName,
2915+
currentUser.AuthToken);
2916+
2917+
if (results.Item3 == HttpStatusCode.OK)
2918+
{
2919+
return results.Item1;
2920+
}
2921+
else
2922+
{
2923+
string queryBeginning = graphQLQuery.Split('\n')[0];
2924+
throw new WebException(String.Format("Call to {0}v1/meta for user {1} with '{2}' returned {3} {4}", currentUser.CustomerEnvironmentUrl, currentUser.UserName, queryBeginning, results.Item3, results.Item1));
2925+
}
2926+
}
2927+
2928+
#endregion
2929+
2930+
#region Retrieval methods
28492931

28502932
/// Returns Tuple<string, List<string>, HttpStatusCode>
28512933
/// ^^^^^^ results of the page
@@ -2920,7 +3002,6 @@ private static Tuple<string, List<string>, HttpStatusCode> apiGET(
29203002
}
29213003
else
29223004
{
2923-
29243005
logLevel = NLog.LogLevel.Error;
29253006
}
29263007

@@ -2980,6 +3061,13 @@ private static Tuple<string, List<string>, HttpStatusCode> apiPOST(
29803061
{
29813062
Stopwatch stopWatch = new Stopwatch();
29823063
stopWatch.Start();
3064+
3065+
string requestBodyFragment = requestBody;
3066+
if (requestBody.Length > 64)
3067+
{
3068+
requestBodyFragment = String.Format("|{0}", requestBody.Substring(0, 64).Replace("\n", ""));
3069+
}
3070+
29833071
try
29843072
{
29853073
HttpClientHandler httpClientHandler = new HttpClientHandler();
@@ -3050,8 +3138,7 @@ private static Tuple<string, List<string>, HttpStatusCode> apiPOST(
30503138
logLevel = NLog.LogLevel.Warn;
30513139
}
30523140
else
3053-
{
3054-
3141+
{
30553142
logLevel = NLog.LogLevel.Error;
30563143
}
30573144

@@ -3090,17 +3177,17 @@ private static Tuple<string, List<string>, HttpStatusCode> apiPOST(
30903177
}
30913178
catch (Exception ex)
30923179
{
3093-
logger.Error("POST {0}{1} threw {2} ({3})", baseUri, restAPIUrl, ex.Message, ex.Source);
3180+
logger.Error("POST {0}{1}{4} threw {2} ({3})", baseUri, restAPIUrl, ex.Message, ex.Source, requestBodyFragment);
30943181
logger.Error(ex);
30953182

3096-
loggerConsole.Error("POST {0}{1} threw {2} ({3})", baseUri, restAPIUrl, ex.Message, ex.Source);
3183+
loggerConsole.Error("POST {0}{1}{4} threw {2} ({3})", baseUri, restAPIUrl, ex.Message, ex.Source, requestBodyFragment);
30973184

30983185
return new Tuple<string, List<string>, HttpStatusCode>(String.Empty, new List<string>(0), HttpStatusCode.InternalServerError);
30993186
}
31003187
finally
31013188
{
31023189
stopWatch.Stop();
3103-
logger.Info("POST {0}{1} took {2:c} ({3} ms)", baseUri, restAPIUrl, stopWatch.Elapsed.ToString("c"), stopWatch.ElapsedMilliseconds);
3190+
logger.Info("POST {0}{1}{4} took {2:c} ({3} ms)", baseUri, restAPIUrl, stopWatch.Elapsed.ToString("c"), stopWatch.ElapsedMilliseconds, requestBodyFragment);
31043191
}
31053192
}
31063193

‎Controllers/CommonControllerMethods.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,11 @@ public void enrichTrace(ObsWorksheet thisObject)
158158
Activity.Current.AddTag("currentObject", thisObject.ToString());
159159
}
160160

161+
public void enrichTrace(ObsMetric thisObject)
162+
{
163+
Activity.Current.AddTag("currentObject", thisObject.ToString());
164+
}
165+
161166
public void enrichTrace(Observe.EntityExplorer.Models.SearchResultsViewModel viewModel)
162167
{
163168
Activity.Current.AddTag("searchQuery", viewModel.SearchQuery);

‎Controllers/DetailsController.cs

Lines changed: 97 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
using System.Diagnostics;
1+
using System;
2+
using System.Diagnostics;
3+
using System.Text;
24
using Microsoft.AspNetCore.Mvc;
35
using Microsoft.Extensions.Caching.Memory;
46
using NLog;
@@ -341,6 +343,100 @@ public IActionResult Dashboard(
341343
}
342344
}
343345

346+
public IActionResult Metric(
347+
string userid,
348+
string id)
349+
{
350+
Stopwatch stopWatch = new Stopwatch();
351+
stopWatch.Start();
352+
353+
try
354+
{
355+
AuthenticatedUser currentUser = this.CommonControllerMethods.GetUser(userid);
356+
if (currentUser == null)
357+
{
358+
return RedirectToAction("Connect", "Connection");
359+
}
360+
ObserveEnvironment observeEnvironment = this.CommonControllerMethods.GetObserveEnvironment(currentUser);
361+
if (observeEnvironment == null)
362+
{
363+
throw new Exception("Unable to retrieve the Observe Environment from cache or server");
364+
}
365+
DetailsMetricViewModel viewModel = new DetailsMetricViewModel(currentUser, observeEnvironment);
366+
367+
CommonControllerMethods.enrichTrace(currentUser);
368+
CommonControllerMethods.enrichTrace(observeEnvironment);
369+
370+
id = Encoding.UTF8.GetString(Convert.FromBase64String(id));
371+
372+
switch (HttpContext.Request.Method)
373+
{
374+
case "GET":
375+
ObsMetric thisMetric = null;
376+
if (observeEnvironment.AllMetricsDict.TryGetValue(id, out thisMetric) == false)
377+
{
378+
throw new KeyNotFoundException(String.Format("Unable to retrieve the Observe Metric {0} from Observe Environment", id));
379+
}
380+
viewModel.CurrentMetric = thisMetric;
381+
382+
// Populate all dataset stages that haven't been populated yet - we lazy load them in advance
383+
observeEnvironment.PopulateAllDatasetStages(currentUser);
384+
385+
viewModel.SearchResults = new List<ObsStage>(256);
386+
387+
foreach (ObsDataset entity in observeEnvironment.AllDatasetsDict.Values)
388+
{
389+
viewModel.SearchResults.AddRange(entity.Stages.Where(s => s.Metrics.Contains(thisMetric) == true).ToList());
390+
}
391+
392+
foreach (ObsDashboard entity in observeEnvironment.AllDashboardsDict.Values)
393+
{
394+
viewModel.SearchResults.AddRange(entity.Stages.Where(s => s.Metrics.Contains(thisMetric) == true).ToList());
395+
}
396+
397+
foreach (ObsMonitor entity in observeEnvironment.AllMonitorsDict.Values)
398+
{
399+
viewModel.SearchResults.AddRange(entity.Stages.Where(s => s.Metrics.Contains(thisMetric) == true).ToList());
400+
}
401+
402+
foreach (ObsMonitor2 entity in observeEnvironment.AllMonitors2Dict.Values)
403+
{
404+
viewModel.SearchResults.AddRange(entity.Stages.Where(s => s.Metrics.Contains(thisMetric) == true).ToList());
405+
}
406+
407+
foreach (ObsWorksheet entity in observeEnvironment.AllWorksheetsDict.Values)
408+
{
409+
viewModel.SearchResults.AddRange(entity.Stages.Where(s => s.Metrics.Contains(thisMetric) == true).ToList());
410+
}
411+
412+
break;
413+
414+
case "POST":
415+
break;
416+
}
417+
418+
CommonControllerMethods.enrichTrace(viewModel.CurrentMetric);
419+
420+
return View(viewModel);
421+
}
422+
catch (Exception ex)
423+
{
424+
logger.Error(ex, ex.Message);
425+
loggerConsole.Error(ex, ex.Message);
426+
427+
ViewData["ErrorMessage"] = ex.Message;
428+
429+
return View(new DetailsMetricViewModel(null, null));
430+
}
431+
finally
432+
{
433+
stopWatch.Stop();
434+
435+
logger.Trace("{0}:{1}/{2}: total duration {3:c} ({4} ms)", HttpContext.Request.Method, this.ControllerContext.RouteData.Values["controller"], this.ControllerContext.RouteData.Values["action"], stopWatch.Elapsed, stopWatch.ElapsedMilliseconds);
436+
loggerConsole.Trace("{0}:{1}/{2}: total duration {3:c} ({4} ms)", HttpContext.Request.Method, this.ControllerContext.RouteData.Values["controller"], this.ControllerContext.RouteData.Values["action"], stopWatch.Elapsed, stopWatch.ElapsedMilliseconds);
437+
}
438+
}
439+
344440
public IActionResult Worksheet(
345441
string userid,
346442
string id)

‎Controllers/SelectController.cs

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,48 @@ public IActionResult Worksheet(
238238
loggerConsole.Trace("{0}:{1}/{2}: total duration {3:c} ({4} ms)", HttpContext.Request.Method, this.ControllerContext.RouteData.Values["controller"], this.ControllerContext.RouteData.Values["action"], stopWatch.Elapsed, stopWatch.ElapsedMilliseconds);
239239
} }
240240

241+
public IActionResult Metric(
242+
string userid)
243+
{
244+
Stopwatch stopWatch = new Stopwatch();
245+
stopWatch.Start();
246+
247+
try
248+
{
249+
AuthenticatedUser currentUser = this.CommonControllerMethods.GetUser(userid);
250+
if (currentUser == null)
251+
{
252+
return RedirectToAction("Connect", "Connection");
253+
}
254+
ObserveEnvironment observeEnvironment = this.CommonControllerMethods.GetObserveEnvironment(currentUser);
255+
if (observeEnvironment == null)
256+
{
257+
ViewData["ErrorMessage"] = "Unable to retrieve the Observe Environment from cache or server";
258+
}
259+
260+
CommonControllerMethods.enrichTrace(currentUser);
261+
CommonControllerMethods.enrichTrace(observeEnvironment);
262+
263+
return View(new BaseViewModel(currentUser, observeEnvironment));
264+
}
265+
catch (Exception ex)
266+
{
267+
logger.Error(ex, ex.Message);
268+
loggerConsole.Error(ex, ex.Message);
269+
270+
ViewData["ErrorMessage"] = ex.Message;
271+
272+
return View(new BaseViewModel(null, null));
273+
}
274+
finally
275+
{
276+
stopWatch.Stop();
277+
278+
logger.Trace("{0}:{1}/{2}: total duration {3:c} ({4} ms)", HttpContext.Request.Method, this.ControllerContext.RouteData.Values["controller"], this.ControllerContext.RouteData.Values["action"], stopWatch.Elapsed, stopWatch.ElapsedMilliseconds);
279+
loggerConsole.Trace("{0}:{1}/{2}: total duration {3:c} ({4} ms)", HttpContext.Request.Method, this.ControllerContext.RouteData.Values["controller"], this.ControllerContext.RouteData.Values["action"], stopWatch.Elapsed, stopWatch.ElapsedMilliseconds);
280+
}
281+
}
282+
241283
public IActionResult Relationship(
242284
string userid)
243285
{

0 commit comments

Comments
 (0)