Skip to content

Commit 7c0c3a7

Browse files
gfsgfs
authored
Add "EnableNonBacktrackingRegex" option to oss-characteristic tool (#453)
* Add "EnableNonBacktrackingRegex" option to oss-characteristic tool Also requires bumping .net version for related projects to net 8 to pick up the non-backtracking behavior that requires 7+. * Update projects to net 6 and 8 * Update pipelines for net 8 * Update dependencies * Options for Backtrack and Single Thread Adds options for using the default backtracking engine and to use single threaded app inspector to oss-characteristics and oss-detect-backdoor.
1 parent 16f88ed commit 7c0c3a7

File tree

23 files changed

+168
-78
lines changed

23 files changed

+168
-78
lines changed

‎Pipelines/templates/dotnet-build-publish-all-platforms-job.yml

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,11 @@ parameters:
1010
# Version of Dotnet SDK to use
1111
- name: dotnetVersion
1212
type: string
13-
default: '6.0.x'
13+
default: '8.0.x'
14+
# Version of Dotnet to publish
15+
- name: dotnetPublishVersion
16+
type: string
17+
default: 'net8.0'
1418
# Include preview versions of Dotnet SDK
1519
- name: includePreviewVersions
1620
type: boolean
@@ -67,28 +71,28 @@ jobs:
6771
displayName: Publish Linux x64
6872
inputs:
6973
command: 'publish'
70-
arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/linux/${{ parameters.projectName }}_linux_$(ReleaseVersion) --sc -r linux-x64'
74+
arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/linux/${{ parameters.projectName }}_linux_$(ReleaseVersion) --sc -r linux-x64 -f ${{ parameters.dotnetPublishVersion }}'
7175
publishWebProjects: false
7276
zipAfterPublish: false
7377
- task: DotNetCoreCLI@2
7478
displayName: Publish MacOS x64
7579
inputs:
7680
command: 'publish'
77-
arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/macos/${{ parameters.projectName }}_macos_$(ReleaseVersion) --sc -r osx-x64'
81+
arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/macos/${{ parameters.projectName }}_macos_$(ReleaseVersion) --sc -r osx-x64 -f ${{ parameters.dotnetPublishVersion }}'
7882
publishWebProjects: false
7983
zipAfterPublish: false
8084
- task: DotNetCoreCLI@2
8185
displayName: Publish Win x64
8286
inputs:
8387
command: 'publish'
84-
arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/win/${{ parameters.projectName }}_win_$(ReleaseVersion) --sc -r win-x64'
88+
arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/win/${{ parameters.projectName }}_win_$(ReleaseVersion) --sc -r win-x64 -f ${{ parameters.dotnetPublishVersion }}'
8589
publishWebProjects: false
8690
zipAfterPublish: false
8791
- task: DotNetCoreCLI@2
8892
displayName: Build .NET Core App
8993
inputs:
9094
command: 'publish'
91-
arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/netcoreapp/${{ parameters.projectName }}_netcoreapp_$(ReleaseVersion)'
95+
arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/netcoreapp/${{ parameters.projectName }}_netcoreapp_$(ReleaseVersion) -f ${{ parameters.dotnetPublishVersion }}'
9296
publishWebProjects: false
9397
zipAfterPublish: false
9498
- task: ArchiveFiles@2

‎Pipelines/templates/dotnet-test-job.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ parameters:
66
# Version of Dotnet SDK to use
77
- name: dotnetVersion
88
type: string
9-
default: '6.0.x'
9+
default: '8.0.x'
1010
# Should Dotnet SDK install preview versions?
1111
- name: includePreviewVersions
1212
type: boolean

‎Pipelines/templates/nuget-build-job.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ parameters:
66
# Version of Dotnet SDK to use
77
- name: dotnetVersion
88
type: string
9-
default: '6.0.x'
9+
default: '8.0.x'
1010
# Should Dotnet SDK install preview versions?
1111
- name: includePreviewVersions
1212
type: boolean

‎src/Shared.CLI/Shared.CLI.csproj

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>net6.0</TargetFramework>
4+
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
55
<RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
66
<Description>OSS Gadget - Shared CLI Functionality</Description>
77
<RepositoryType>GitHub</RepositoryType>
@@ -39,19 +39,19 @@
3939
</ItemGroup>
4040

4141
<ItemGroup>
42-
<PackageReference Include="AngleSharp" Version="1.0.0-alpha-844" />
42+
<PackageReference Include="AngleSharp" Version="1.0.7" />
4343
<PackageReference Include="CommandLineParser" Version="2.9.1" />
4444
<PackageReference Include="Crayon" Version="2.0.69" />
45-
<PackageReference Include="F23.StringSimilarity" Version="5.0.0" />
46-
<PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.0.1" />
47-
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
48-
<PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.18" />
49-
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="6.0.1" />
50-
<PackageReference Include="NLog" Version="5.0.4" />
51-
<PackageReference Include="NLog.Schema" Version="5.0.4" />
52-
<PackageReference Include="NuGet.Versioning" Version="6.6.1" />
53-
<PackageReference Include="Octokit" Version="4.0.1" />
54-
<PackageReference Include="Sarif.Sdk" Version="3.1.0" />
45+
<PackageReference Include="F23.StringSimilarity" Version="6.0.0" />
46+
<PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.1.0" />
47+
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
48+
<PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.2.23" />
49+
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
50+
<PackageReference Include="NLog" Version="5.2.8" />
51+
<PackageReference Include="NLog.Schema" Version="5.2.8" />
52+
<PackageReference Include="NuGet.Versioning" Version="6.8.0" />
53+
<PackageReference Include="Octokit" Version="9.1.0" />
54+
<PackageReference Include="Sarif.Sdk" Version="4.4.0" />
5555
<PackageReference Include="SemanticVersioning" Version="2.0.2" />
5656
<PackageReference Include="System.Console" Version="4.3.1" />
5757
</ItemGroup>
@@ -64,4 +64,8 @@
6464
<ItemGroup>
6565
<ProjectReference Include="..\Shared\Shared.Lib.csproj" />
6666
</ItemGroup>
67+
68+
<ItemGroup>
69+
<PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
70+
</ItemGroup>
6771
</Project>

‎src/Shared/Shared.Lib.csproj

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
4-
<TargetFramework>net6.0</TargetFramework>
4+
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
55
<RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
66
<Description>OSS Gadget - Shared Library Functionality</Description>
77
<RepositoryType>GitHub</RepositoryType>
@@ -29,25 +29,25 @@
2929
</PropertyGroup>
3030

3131
<ItemGroup>
32-
<PackageReference Include="AngleSharp" Version="1.0.0-alpha-844" />
32+
<PackageReference Include="AngleSharp" Version="1.0.7" />
3333
<PackageReference Include="CommandLineParser" Version="2.9.1" />
3434
<PackageReference Include="Crayon" Version="2.0.69" />
35-
<PackageReference Include="F23.StringSimilarity" Version="5.0.0" />
36-
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
37-
<PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.18" />
38-
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="6.0.1" />
39-
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.1" />
40-
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0" />
41-
<PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
42-
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="6.0.10" />
43-
<PackageReference Include="NLog" Version="5.0.4" />
44-
<PackageReference Include="NLog.Schema" Version="5.0.4" />
45-
<PackageReference Include="NuGet.Packaging" Version="6.6.1" />
46-
<PackageReference Include="NuGet.Protocol" Version="6.6.1" />
47-
<PackageReference Include="Octokit" Version="4.0.1" />
35+
<PackageReference Include="F23.StringSimilarity" Version="6.0.0" />
36+
<PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
37+
<PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.2.23" />
38+
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
39+
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
40+
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
41+
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
42+
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.0" />
43+
<PackageReference Include="NLog" Version="5.2.8" />
44+
<PackageReference Include="NLog.Schema" Version="5.2.8" />
45+
<PackageReference Include="NuGet.Packaging" Version="6.8.0" />
46+
<PackageReference Include="NuGet.Protocol" Version="6.8.0" />
47+
<PackageReference Include="Octokit" Version="9.1.0" />
4848
<PackageReference Include="packageurl-dotnet" Version="1.3.0" />
4949
<PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
50-
<PackageReference Include="Sarif.Sdk" Version="3.1.0" />
50+
<PackageReference Include="Sarif.Sdk" Version="4.4.0" />
5151
<PackageReference Include="SemanticVersioning" Version="2.0.2" />
5252
<PackageReference Include="System.Console" Version="4.3.1" />
5353
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
@@ -64,4 +64,8 @@
6464
<None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
6565
</ItemGroup>
6666

67+
<ItemGroup>
68+
<PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
69+
</ItemGroup>
70+
6771
</Project>

‎src/oss-characteristics/CharacteristicTool.cs

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ public static IEnumerable<Example> Examples
6666
HelpText = "exclude files or paths which match provided glob patterns.")]
6767
public string FilePathExclusions { get; set; } = "";
6868

69+
[Option('b', "backtracking", Required = false, HelpText = "Use backtracking regex engine by default.")]
70+
public bool EnableBacktracking { get; set; } = false;
71+
72+
[Option('s', "single-threaded", Required = false, HelpText = "Use single-threaded analysis")]
73+
public bool SingleThread { get; set; } = false;
74+
6975
public bool AllowTagsInBuildFiles { get; set; } = true;
7076

7177
public bool AllowDupTags { get; set; } = false;
@@ -107,8 +113,9 @@ public CharacteristicTool() : this(new ProjectManagerFactory())
107113
ConfidenceFilters = new [] { Confidence.High | Confidence.Medium | Confidence.Low },
108114
ScanUnknownTypes = true,
109115
AllowAllTagsInBuildFiles = options.AllowTagsInBuildFiles,
110-
SingleThread = false,
111-
FilePathExclusions = options.FilePathExclusions?.Split(',') ?? Array.Empty<string>()
116+
SingleThread = options.SingleThread,
117+
FilePathExclusions = options.FilePathExclusions?.Split(',') ?? Array.Empty<string>(),
118+
EnableNonBacktrackingRegex = !options.EnableBacktracking
112119
};
113120

114121
try

‎src/oss-characteristics/oss-characteristic.csproj

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net6.0</TargetFramework>
5+
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
66
<RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
77
<Description>OSS Gadget - Characteristic Identifier</Description>
88
<RepositoryType>GitHub</RepositoryType>
@@ -29,7 +29,7 @@
2929
</PropertyGroup>
3030

3131
<ItemGroup>
32-
<PackageReference Include="Microsoft.CST.ApplicationInspector.Commands" Version="1.6.24" />
32+
<PackageReference Include="Microsoft.CST.ApplicationInspector.Commands" Version="1.9.17" />
3333
</ItemGroup>
3434

3535
<ItemGroup>
@@ -41,4 +41,8 @@
4141
<None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
4242
<None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
4343
</ItemGroup>
44+
45+
<ItemGroup>
46+
<PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
47+
</ItemGroup>
4448
</Project>

‎src/oss-defog/oss-defog.csproj

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net6.0</TargetFramework>
5+
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
66
<RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
77
<Description>OSS Gadget - Obfuscated String Detector</Description>
88
<RepositoryType>GitHub</RepositoryType>
@@ -46,4 +46,8 @@
4646
<None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
4747
<None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
4848
</ItemGroup>
49+
50+
<ItemGroup>
51+
<PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
52+
</ItemGroup>
4953
</Project>

‎src/oss-detect-backdoor/DetectBackdoorTool.cs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,12 @@ public static IEnumerable<Example> Examples
4949
[Option('c', "use-cache", Required = false, Default = false,
5050
HelpText = "do not download the package if it is already present in the destination directory.")]
5151
public bool UseCache { get; set; }
52+
53+
[Option('b', "backtracking", Required = false, HelpText = "Use backtracking engine by default.")]
54+
public bool EnableBacktracking { get; set; } = false;
55+
56+
[Option('s', "single-threaded", Required = false, HelpText = "Use single-threaded analysis")]
57+
public bool SingleThread { get; set; } = false;
5258
}
5359

5460
public DetectBackdoorTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory)
@@ -177,7 +183,9 @@ void WriteMatch(MatchRecord match, int index, int matchCount)
177183
AllowTagsInBuildFiles = true,
178184
FilePathExclusions = ".md,LICENSE,.txt",
179185
AllowDupTags = true,
180-
SarifLevel = CodeAnalysis.Sarif.FailureLevel.Warning
186+
SarifLevel = CodeAnalysis.Sarif.FailureLevel.Warning,
187+
EnableBacktracking = options.EnableBacktracking,
188+
SingleThread = options.SingleThread
181189
};
182190

183191
return await characteristicTool.RunAsync(cOptions);

‎src/oss-detect-backdoor/oss-detect-backdoor.csproj

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
<PropertyGroup>
44
<OutputType>Exe</OutputType>
5-
<TargetFramework>net6.0</TargetFramework>
5+
<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
66
<RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
77
<Description>OSS Gadget - Backdoor Identifier</Description>
88
<RepositoryType>GitHub</RepositoryType>
@@ -83,4 +83,8 @@
8383
<None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
8484
<None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
8585
</ItemGroup>
86+
87+
<ItemGroup>
88+
<PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
89+
</ItemGroup>
8690
</Project>

0 commit comments

Comments
 (0)