Skip to content

Commit 5059422

Browse files
dbalikhingfs
dbalikhin
and
gfs
authored
Use System.Text.Json instead of Newton.Json (#525)
* Use System.Text.Json instead of Newtonsoft.Json. Approximately 2x faster result output serialization with significantly reduced memory usage (for JSON format). * Adds simple benchmark for json writer. --------- Co-authored-by: Gabe Stocco <[email protected]>
1 parent d39172e commit 5059422

40 files changed

+414
-250
lines changed

‎Benchmarks/AnalyzeBenchmark.cs renamed to ‎AppInspector.Benchmarks/AnalyzeBenchmark.cs

File renamed without changes.

‎Benchmarks/Benchmarks.csproj renamed to ‎AppInspector.Benchmarks/AppInspector.Benchmarks.csproj

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
<TargetFrameworks>net6.0;net7.0</TargetFrameworks>
66
<Nullable>enable</Nullable>
77
<LangVersion>10.0</LangVersion>
8+
<RootNamespace>ApplicationInspector.Benchmarks</RootNamespace>
9+
<Company>Microsoft Corporation</Company>
810
</PropertyGroup>
911

1012
<ItemGroup>
@@ -13,6 +15,7 @@
1315

1416
</ItemGroup>
1517
<ItemGroup>
18+
<ProjectReference Include="..\AppInspector.CLI\AppInspector.CLI.csproj" />
1619
<ProjectReference Include="..\AppInspector\AppInspector.Commands.csproj" />
1720
<ProjectReference Include="..\AppInspector.Tests\AppInspector.Tests.csproj" />
1821
</ItemGroup>

‎Benchmarks/DistinctBenchmarks.cs renamed to ‎AppInspector.Benchmarks/DistinctBenchmarks.cs

File renamed without changes.

‎Benchmarks/Program.cs renamed to ‎AppInspector.Benchmarks/Program.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@ public static void Main(string[] args)
99
{
1010
// new DebugInProcessConfig()
1111
//BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new DebugInProcessConfig());
12-
var summary = BenchmarkRunner.Run<AnalyzeBenchmark>();
12+
var summary = BenchmarkRunner.Run<WriterBench>();
1313
}
1414
}

‎AppInspector.Benchmarks/WriterBench.cs

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
using System;
2+
using System.IO;
3+
using System.Reflection;
4+
using BenchmarkDotNet.Attributes;
5+
using BenchmarkDotNet.Jobs;
6+
using DotLiquid;
7+
using Microsoft.ApplicationInspector.CLI;
8+
using Microsoft.ApplicationInspector.Commands;
9+
using Microsoft.ApplicationInspector.RulesEngine;
10+
11+
namespace ApplicationInspector.Benchmarks;
12+
[MemoryDiagnoser]
13+
[SimpleJob(RuntimeMoniker.Net70)]
14+
public class WriterBench
15+
{
16+
[Params(1000, 10000)]
17+
public int N;
18+
19+
// Holds the result object which will be serialized
20+
private AnalyzeResult _result;
21+
22+
[GlobalSetup]
23+
public void GlobalSetup()
24+
{
25+
var _exerpt = "Hello World";
26+
var helper = new MetaDataHelper("..");
27+
var matchRecord = new MatchRecord("rule-id", "rule-name")
28+
{
29+
Boundary = new Boundary() { Index = 0, Length = 1 },
30+
EndLocationColumn = 0,
31+
EndLocationLine = 1,
32+
Excerpt = _exerpt,
33+
FileName = "TestFile",
34+
LanguageInfo = new LanguageInfo(),
35+
Tags = new []{"TestTag"}
36+
};
37+
for (int i = 0; i < N; i++)
38+
{
39+
helper.AddMatchRecord(matchRecord);
40+
}
41+
helper.PrepareReport();
42+
43+
_result = new AnalyzeResult() { Metadata = helper.Metadata, ResultCode = 0 };
44+
}
45+
46+
[Benchmark(Baseline = true)]
47+
public void ExportRecordsToJson()
48+
{
49+
var tmpPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
50+
CLIAnalyzeCmdOptions analyzeOpts = new CLIAnalyzeCmdOptions()
51+
{
52+
OutputFileFormat = "json",
53+
OutputFilePath = tmpPath
54+
};
55+
var writerFactory = new WriterFactory();
56+
var writer = writerFactory.GetWriter(analyzeOpts);
57+
writer.WriteResults(_result,analyzeOpts);
58+
File.Delete(tmpPath);
59+
}
60+
61+
public static string GetExecutingDirectoryName()
62+
{
63+
if (Assembly.GetEntryAssembly()?.GetName().CodeBase is string codeBaseLoc)
64+
{
65+
var location = new Uri(codeBaseLoc);
66+
return new FileInfo(location.AbsolutePath).Directory?.FullName ?? string.Empty;
67+
}
68+
69+
return string.Empty;
70+
}
71+
}

‎AppInspector.CLI/TagInfo.cs

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33

44
using System;
55
using System.Collections.Generic;
6+
using System.Text.Json.Serialization;
67
using System.Text.RegularExpressions;
78
using DotLiquid;
89
using Microsoft.ApplicationInspector.RulesEngine;
9-
using Newtonsoft.Json;
1010

1111
namespace Microsoft.ApplicationInspector.Commands;
1212

@@ -21,17 +21,18 @@ public enum tagInfoType
2121
allTags
2222
}
2323

24-
[JsonProperty(PropertyName = "type")] public tagInfoType Type;
24+
[JsonPropertyName("type")]
25+
public tagInfoType Type;
2526

2627
public TagCategory()
2728
{
2829
Groups = new List<TagGroup>();
2930
}
3031

31-
[JsonProperty(PropertyName = "categoryName")]
32+
[JsonPropertyName("categoryName")]
3233
public string? Name { get; set; }
3334

34-
[JsonProperty(PropertyName = "groups")]
35+
[JsonPropertyName("groups")]
3536
public List<TagGroup>? Groups { get; set; }
3637
}
3738

@@ -45,14 +46,14 @@ public TagGroup()
4546
Patterns = new List<TagSearchPattern>();
4647
}
4748

48-
[JsonProperty(PropertyName = "title")] public string? Title { get; set; }
49+
[JsonPropertyName("title")] public string? Title { get; set; }
4950

5051
[JsonIgnore] public string? IconURL { get; set; }
5152

52-
[JsonProperty(PropertyName = "dataRef")]
53+
[JsonPropertyName("dataRef")]
5354
public string? DataRef { get; set; }
5455

55-
[JsonProperty(PropertyName = "patterns")]
56+
[JsonPropertyName("patterns")]
5657
public List<TagSearchPattern>? Patterns { get; set; }
5758
}
5859

@@ -61,7 +62,7 @@ public class TagSearchPattern : Drop
6162
private Regex? _expression;
6263
private string _searchPattern = "";
6364

64-
[JsonProperty(PropertyName = "searchPattern")]
65+
[JsonPropertyName("searchPattern")]
6566
public string SearchPattern
6667
{
6768
get => _searchPattern;
@@ -85,19 +86,19 @@ public Regex Expression
8586
}
8687
}
8788

88-
[JsonProperty(PropertyName = "displayName")]
89+
[JsonPropertyName("displayName")]
8990
public string? DisplayName { get; set; }
9091

91-
[JsonProperty(PropertyName = "detectedIcon")]
92+
[JsonPropertyName("detectedIcon")]
9293
public string? DetectedIcon { get; set; } = "fas fa-cat"; //default
9394

94-
[JsonProperty(PropertyName = "notDetectedIcon")]
95+
[JsonPropertyName("notDetectedIcon")]
9596
public string? NotDetectedIcon { get; set; }
9697

97-
[JsonProperty(PropertyName = "detected")]
98+
[JsonPropertyName("detected")]
9899
public bool Detected { get; set; }
99100

100-
[JsonProperty(PropertyName = "details")]
101+
[JsonPropertyName("details")]
101102
public string Details
102103
{
103104
get
@@ -107,7 +108,7 @@ public string Details
107108
}
108109
}
109110

110-
[JsonProperty(PropertyName = "confidence")]
111+
[JsonPropertyName("confidence")]
111112
public string Confidence { get; set; } = "Medium";
112113

113114
public static bool ShouldSerializeExpression()
@@ -123,14 +124,14 @@ public class TagInfo : Drop
123124
{
124125
private string _confidence = "Medium";
125126

126-
[JsonProperty(PropertyName = "tag")] public string? Tag { get; set; }
127+
[JsonPropertyName("tag")] public string? Tag { get; set; }
127128

128-
[JsonProperty(PropertyName = "displayName")]
129+
[JsonPropertyName("displayName")]
129130
public string? ShortTag { get; set; }
130131

131132
[JsonIgnore] public string? StatusIcon { get; set; }
132133

133-
[JsonProperty(PropertyName = "confidence")]
134+
[JsonPropertyName("confidence")]
134135
public string Confidence
135136
{
136137
get => _confidence;
@@ -143,14 +144,14 @@ public string Confidence
143144
}
144145
}
145146

146-
[JsonProperty(PropertyName = "severity")]
147+
[JsonPropertyName("severity")]
147148
public string Severity { get; set; } = "Moderate";
148149

149-
[JsonProperty(PropertyName = "detected")]
150+
[JsonPropertyName("detected")]
150151
public bool Detected { get; set; }
151152
}
152153

153154
public class TagException
154155
{
155-
[JsonProperty(PropertyName = "tag")] public string? Tag { get; set; }
156+
[JsonPropertyName("tag")] public string? Tag { get; set; }
156157
}

‎AppInspector.CLI/Writers/AnalyzeHtmlWriter.cs

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
using Microsoft.ApplicationInspector.RulesEngine;
1717
using Microsoft.Extensions.Logging;
1818
using Microsoft.Extensions.Logging.Abstractions;
19-
using Newtonsoft.Json;
19+
using System.Text.Json.Serialization;
20+
using System.Text.Json;
2021

2122
namespace Microsoft.ApplicationInspector.CLI;
2223

@@ -27,7 +28,7 @@ public class AnalyzeHtmlWriter : CommandResultsWriter
2728

2829
private MetaData? _appMetaData;
2930

30-
public AnalyzeHtmlWriter(TextWriter textWriter, ILoggerFactory? loggerFactory = null) : base(textWriter)
31+
public AnalyzeHtmlWriter(StreamWriter textWriter, ILoggerFactory? loggerFactory = null) : base(textWriter)
3132
{
3233
_logger = loggerFactory?.CreateLogger<AnalyzeHtmlWriter>() ?? NullLogger<AnalyzeHtmlWriter>.Instance;
3334
KeyedTagInfoLists = new Dictionary<string, List<TagInfo>>();
@@ -91,7 +92,7 @@ private void WriteHtmlResult()
9192
string? jsonData;
9293
try
9394
{
94-
jsonData = JsonConvert.SerializeObject(data);
95+
jsonData = JsonSerializer.Serialize(data);
9596
}
9697
catch (Exception e)
9798
{
@@ -232,9 +233,23 @@ public void PopulateTagGroups()
232233
//read default/user preferences on what tags to report presence on and groupings
233234
if (File.Exists(Utils.GetPath(Utils.AppPath.tagGroupPref)))
234235
{
235-
TagGroupPreferences =
236-
JsonConvert.DeserializeObject<List<TagCategory>>(
237-
File.ReadAllText(Utils.GetPath(Utils.AppPath.tagGroupPref)));
236+
try
237+
{
238+
var options = new JsonSerializerOptions
239+
{
240+
ReadCommentHandling = JsonCommentHandling.Skip, // Allow strings to start with '/', e.g., "// Copyright (C) Microsoft. All rights reserved"
241+
};
242+
TagGroupPreferences =
243+
JsonSerializer.Deserialize<List<TagCategory>>(
244+
File.ReadAllText(Utils.GetPath(Utils.AppPath.tagGroupPref)), options);
245+
}
246+
catch (Exception e)
247+
{
248+
_logger.LogError(
249+
"Failed to populate tag groups. Failed to serialize JSON representation of results in memory. {Type} : {Message}",
250+
e.GetType().Name, e.Message);
251+
throw;
252+
}
238253
}
239254
else
240255
{
@@ -627,13 +642,15 @@ public List<TagCounterUI> ConvertTagCounters(IEnumerable<MetricTagCounter> metri
627642
/// </summary>
628643
public class TagCounterUI : Drop
629644
{
630-
[JsonProperty(PropertyName = "tag")] public string? Tag { get; set; }
645+
[JsonPropertyName("tag")]
646+
public string? Tag { get; set; }
631647

632-
[JsonProperty(PropertyName = "displayName")]
648+
[JsonPropertyName("displayName")]
633649
public string? ShortTag { get; set; }
634650

635-
[JsonProperty(PropertyName = "count")] public int Count { get; set; }
651+
[JsonPropertyName("count")]
652+
public int Count { get; set; }
636653

637-
[JsonProperty(PropertyName = "includeAsMatch")]
654+
[JsonPropertyName("includeAsMatch")]
638655
public bool IncludeAsMatch => false;
639656
}

‎AppInspector.CLI/Writers/AnalyzeJsonWriter.cs

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
using Microsoft.ApplicationInspector.Commands;
66
using Microsoft.Extensions.Logging;
77
using Microsoft.Extensions.Logging.Abstractions;
8-
using Newtonsoft.Json;
8+
using System.Text.Json.Serialization;
9+
using System.Text.Json;
10+
using System;
911

1012
namespace Microsoft.ApplicationInspector.CLI;
1113

@@ -20,20 +22,33 @@ public class AnalyzeJsonWriter : CommandResultsWriter
2022
{
2123
private readonly ILogger<AnalyzeJsonWriter> _logger;
2224

23-
public AnalyzeJsonWriter(TextWriter textWriter, ILoggerFactory? loggerFactory = null) : base(textWriter)
25+
public AnalyzeJsonWriter(StreamWriter textWriter, ILoggerFactory? loggerFactory = null) : base(textWriter)
2426
{
2527
_logger = loggerFactory?.CreateLogger<AnalyzeJsonWriter>() ?? NullLogger<AnalyzeJsonWriter>.Instance;
2628
}
2729

2830
public override void WriteResults(Result result, CLICommandOptions commandOptions, bool autoClose = true)
2931
{
3032
var analyzeResult = (AnalyzeResult)result;
33+
if (StreamWriter == null)
34+
{
35+
throw new ArgumentNullException(nameof(StreamWriter));
36+
}
3137

32-
JsonSerializer jsonSerializer = new();
33-
jsonSerializer.Formatting = Formatting.Indented;
34-
if (TextWriter != null)
38+
try
39+
{
40+
var options = new JsonSerializerOptions
41+
{
42+
WriteIndented = true,
43+
};
44+
JsonSerializer.Serialize(StreamWriter.BaseStream, analyzeResult, options);
45+
}
46+
catch (Exception e)
3547
{
36-
jsonSerializer.Serialize(TextWriter, analyzeResult);
48+
_logger.LogError(
49+
"Failed to serialize JSON representation of results in memory. {Type} : {Message}",
50+
e.GetType().Name, e.Message);
51+
throw;
3752
}
3853

3954
if (autoClose)
@@ -47,6 +62,7 @@ public override void WriteResults(Result result, CLICommandOptions commandOption
4762
/// </summary>
4863
private class TagsFile
4964
{
50-
[JsonProperty(PropertyName = "tags")] public string[]? Tags { get; set; }
65+
[JsonPropertyName("tags")]
66+
public string[]? Tags { get; set; }
5167
}
5268
}

0 commit comments

Comments
 (0)