Skip to content

Commit fdc1f87

Browse files
gfsgfs
authored
Parallelization Stability Improvements (#216)
* Fix more parallel edge cases * Only read through tags list once. * threadsafe metrictagcounter.count * Revert counter only tag change * Update all related counters * Clean up * Partial revert change to AnalyzeHtmlWriter ConcurrentStack was overkill here. * Remove unused dependency.
1 parent e976a5c commit fdc1f87

File tree

5 files changed

+59
-69
lines changed

5 files changed

+59
-69
lines changed

‎AppInspector.CLI/Writers/AnalyzeHtmlWriter.cs

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -507,19 +507,17 @@ private List<TagInfo> GetTagInfoListBySeverity()
507507
/// <summary>
508508
/// Opportunity for any final data prep before report gen
509509
/// </summary>
510-
public List<TagCounterUI> ConvertTagCounters(List<MetricTagCounter> metricTagCounters)
510+
public List<TagCounterUI> ConvertTagCounters(IEnumerable<MetricTagCounter> metricTagCounters)
511511
{
512512
List<TagCounterUI> result = new List<TagCounterUI>();
513513
//TagCountersUI is liquid compatible while TagCounters is not to support json serialization; the split prevents exception
514514
//not fixable via json iteration disabling
515-
foreach (MetricTagCounter counter in metricTagCounters)
515+
516+
result.AddRange(metricTagCounters.Select(counter => new TagCounterUI()
516517
{
517-
result.Add(new TagCounterUI
518-
{
519-
Tag = counter.Tag,
520-
Count = counter.Count
521-
});
522-
}
518+
Tag = counter.Tag,
519+
Count = counter.Count
520+
}));
523521

524522
return result;
525523
}

‎AppInspector/MetaData.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ public class MetaData
190190
/// List of detected tag counters i.e. metrics
191191
/// </summary>
192192
[JsonProperty(PropertyName = "tagCounters")]
193-
public List<MetricTagCounter> TagCounters { get; set; }
193+
public ConcurrentStack<MetricTagCounter> TagCounters { get; set; }
194194

195195
/// <summary>
196196
/// List of detailed MatchRecords from scan
@@ -225,7 +225,7 @@ public MetaData(string applicationName, string sourcePath)
225225
};
226226

227227
Languages = new ConcurrentDictionary<string, int>();
228-
TagCounters = new List<MetricTagCounter>();
228+
TagCounters = new ConcurrentStack<MetricTagCounter>();
229229
}
230230

231231
internal void IncrementFilesAnalyzed(int amount = 1)

‎AppInspector/MetaDataHelper.cs

Lines changed: 42 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -49,54 +49,43 @@ public void AddMatchRecord(MatchRecord matchRecord)
4949
}
5050
}
5151

52-
// single standard properties we capture from any supported file type; others just captured as general tag matches...
53-
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Author")))
54-
{
55-
Metadata.Authors = ExtractValue(matchRecord.Sample);
56-
}
57-
58-
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Publisher")))
59-
{
60-
Metadata.Authors = ExtractValue(matchRecord.Sample);
61-
}
62-
63-
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Description")))
64-
{
65-
Metadata.Description = ExtractValue(matchRecord.Sample);
66-
}
67-
68-
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Name")))
69-
{
70-
Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
71-
}
72-
73-
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Version")))
74-
{
75-
Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
76-
}
77-
78-
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Target.Processor")))
79-
{
80-
_ = Metadata.CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
81-
}
82-
83-
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Output.Type")))
84-
{
85-
_ = Metadata.Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
86-
}
87-
88-
if (matchRecord.Tags.Any(v => v.Contains("Platform.OS")))
89-
{
90-
_ = Metadata.OSTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
91-
}
92-
93-
if (matchRecord.Tags.Any(v => v.Contains("Metric.")))
52+
//Update metric counters for default or user specified tags; don't add as match detail
53+
foreach (var tag in matchRecord.Tags)
9454
{
95-
Metadata.TagCounters.Add(new MetricTagCounter()
55+
switch (tag)
9656
{
97-
Tag = matchRecord.Tags[0],
98-
Count = 0
99-
});
57+
case "Metadata.Application.Author":
58+
case "Metadata.Application.Publisher":
59+
Metadata.Authors = ExtractValue(matchRecord.Sample);
60+
break;
61+
case "Metadata.Application.Description":
62+
Metadata.Description = ExtractValue(matchRecord.Sample);
63+
break;
64+
case "Metadata.Application.Name":
65+
Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
66+
break;
67+
case "Metadata.Application.Version":
68+
Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
69+
break;
70+
case "Metadata.Application.Target.Processor":
71+
_ = Metadata.CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
72+
break;
73+
case "Metadata.Application.Output.Type":
74+
_ = Metadata.Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
75+
break;
76+
case "Platform.OS":
77+
_ = Metadata.OSTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
78+
break;
79+
default:
80+
if (tag.Contains("Metric."))
81+
{
82+
Metadata.TagCounters.Push(new MetricTagCounter()
83+
{
84+
Tag = tag
85+
});
86+
}
87+
break;
88+
}
10089
}
10190

10291
//safeguard sample output now that we've matched properties for blocking browser xss
@@ -109,20 +98,16 @@ public void AddMatchRecord(MatchRecord matchRecord)
10998
_ = Metadata.AppTypes.TryAdd(solutionType,0);
11099
}
111100

112-
//Update metric counters for default or user specified tags; don't add as match detail
113-
bool counterOnlyTag = false;
114-
foreach (MetricTagCounter counter in Metadata.TagCounters)
101+
bool CounterOnlyTagSet = false;
102+
var selected = Metadata.TagCounters.Where(x => matchRecord.Tags.Any(y => y.Contains(x.Tag)));
103+
foreach (var select in selected)
115104
{
116-
if (matchRecord.Tags.Any(v => v.Contains(counter.Tag)))
117-
{
118-
counterOnlyTag = true;
119-
counter.Count++;
120-
break;
121-
}
105+
CounterOnlyTagSet = true;
106+
select.IncrementCount();
122107
}
123108

124-
//omit adding if only a counter metric tag
125-
if (!counterOnlyTag)
109+
//omit adding if ther a counter metric tag
110+
if (!CounterOnlyTagSet)
126111
{
127112
//update list of unique tags as we go
128113
foreach (string tag in matchRecord.Tags)

‎AppInspector/MetricTagCounter.cs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
// Licensed under the MIT License.
33

44
using Newtonsoft.Json;
5+
using System.Threading;
56

67
namespace Microsoft.ApplicationInspector.Commands
78
{
@@ -14,9 +15,16 @@ public class MetricTagCounter
1415
public string Tag { get; set; }
1516

1617
[JsonProperty(PropertyName = "count")]
17-
public int Count { get; set; }
18+
public int Count { get { return _count; } }
1819

1920
[JsonProperty(PropertyName = "includeAsMatch")]
2021
public bool IncludeAsMatch => false;
22+
23+
private int _count = 0;
24+
25+
internal void IncrementCount(int amount = 1)
26+
{
27+
Interlocked.Add(ref _count, amount);
28+
}
2129
}
2230
}

‎MultiExtractor/Extractor.cs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,6 @@ private void CheckResourceGovernor(long additionalBytes = 0)
188188
/// <returns>Extracted files</returns>
189189
public IEnumerable<FileEntry> ExtractFile(string filename, bool parallel = false)
190190
{
191-
Console.Write(parallel);
192191
if (!File.Exists(filename))
193192
{
194193
Logger.Warn("ExtractFile called, but {0} does not exist.", filename);

0 commit comments

Comments
 (0)