feat: make inclusion to the test case list based on matching tags (#317)

Use tags to narrow down test cases to those labeled with strings matching
the regex passed as a config parameter.

Author: huberts90

README.md

@@ -133,6 +133,7 @@ Flags:
   -r, --rate-limit duration                    Limit the request rate to the server to 1 request per specified duration. 0 is the default, and disables rate limiting.
       --read-timeout duration                  timeout for receiving responses during test execution (default 10s)
       --show-failures-only                     shows only the results of failed tests
+  -T, --include-tags string                    include tests tagged with labels matching this Go regular expression (e.g. to include all tests being tagged with "cookie", use "^cookie$").
   -t, --time                                   show time spent per test
       --wait-delay duration                    Time to wait between retries for all wait operations. (default 1s)
       --wait-for-connection-timeout duration   Http connection timeout, The timeout includes connection time, any redirects, and reading the response body. (default 3s)

cmd/run.go

@@ -33,6 +33,7 @@ func NewRunCommand() *cobra.Command {
 
 	runCmd.Flags().StringP("exclude", "e", "", "exclude tests matching this Go regular expression (e.g. to exclude all tests beginning with \"91\", use \"^91.*\"). \nIf you want more permanent exclusion, check the 'exclude' option in the config file.")
 	runCmd.Flags().StringP("include", "i", "", "include only tests matching this Go regular expression (e.g. to include only tests beginning with \"91\", use \"^91.*\"). \\nIf you want more permanent inclusion, check the 'include' option in the config file.\"")
+	runCmd.Flags().StringP("include-tags", "T", "", "include tests tagged with labels matching this Go regular expression (e.g. to include all tests being tagged with \"cookie\", use \"^cookie$\").")
 	runCmd.Flags().StringP("dir", "d", ".", "recursively find yaml tests in this directory")
 	runCmd.Flags().StringP("output", "o", "normal", "output type for ftw tests. \"normal\" is the default.")
 	runCmd.Flags().StringP("file", "f", "", "output file path for ftw tests. Prints to standard output by default.")
@@ -65,6 +66,7 @@ func runE(cmd *cobra.Command, _ []string) error {
 	cmd.SilenceUsage = true
 	exclude, _ := cmd.Flags().GetString("exclude")
 	include, _ := cmd.Flags().GetString("include")
+	includeTags, _ := cmd.Flags().GetString("include-tags")
 	dir, _ := cmd.Flags().GetString("dir")
 	outputFilename, _ := cmd.Flags().GetString("file")
 	logFilePath, _ := cmd.Flags().GetString("log-file")
@@ -113,11 +115,21 @@ func runE(cmd *cobra.Command, _ []string) error {
 
 	var includeRE *regexp.Regexp
 	if include != "" {
-		includeRE = regexp.MustCompile(include)
+		if includeRE, err = regexp.Compile(include); err != nil {
+			return fmt.Errorf("invalid --include regular expression: %w", err)
+		}
 	}
 	var excludeRE *regexp.Regexp
 	if exclude != "" {
-		excludeRE = regexp.MustCompile(exclude)
+		if excludeRE, err = regexp.Compile(exclude); err != nil {
+			return fmt.Errorf("invalid --exclude regular expression: %w", err)
+		}
+	}
+	var includeTagsRE *regexp.Regexp
+	if includeTags != "" {
+		if includeTagsRE, err = regexp.Compile(includeTags); err != nil {
+			return fmt.Errorf("invalid --include-tags regular expression: %w", err)
+		}
 	}
 
 	// Add wait4x checkers
@@ -167,6 +179,7 @@ func runE(cmd *cobra.Command, _ []string) error {
 	currentRun, err := runner.Run(cfg, tests, runner.RunnerConfig{
 		Include:        includeRE,
 		Exclude:        excludeRE,
+		IncludeTags:    includeTagsRE,
 		ShowTime:       showTime,
 		ShowOnlyFailed: showOnlyFailed,
 		ConnectTimeout: connectTimeout,

config/config.go

@@ -28,6 +28,7 @@ func NewDefaultConfig() *FTWConfiguration {
 		MaxMarkerLogLines:   DefaultMaxMarkerLogLines,
 		IncludeTests:        make(map[*FTWRegexp]string),
 		ExcludeTests:        make(map[*FTWRegexp]string),
+		IncludeTags:         make(map[*FTWRegexp]string),
 	}
 	return cfg
 }

config/config_test.go

@@ -20,6 +20,8 @@ include:
     '^9.*': 'Include all tests starting with 9'
 exclude:
     '^920400-2$': 'Exclude this test'
+include_tags:
+    '^cookie$': 'Run test tagged with this label'
 testoverride:
   input:
     dest_addr: 'httpbingo.org'

config/types.go

@@ -48,6 +48,8 @@ type FTWConfiguration struct {
 	IncludeTests map[*FTWRegexp]string `koanf:"include"`
 	// ExcludeTests is a list of tests to exclude (same as --exclude)
 	ExcludeTests map[*FTWRegexp]string `koanf:"exclude"`
+	// IncludeTags is a list of tags matching tests to run (same as --tag)
+	IncludeTags map[*FTWRegexp]string `koanf:"include_tags"`
 }
 
 type PlatformOverrides struct {

runner/run.go

@@ -6,7 +6,6 @@ package runner
 import (
 	"errors"
 	"fmt"
-	"regexp"
 	"time"
 
 	schema "github.com/coreruleset/ftw-tests-schema/v2/types"
@@ -52,6 +51,7 @@ func Run(cfg *config.FTWConfiguration, tests []*test.FTWTest, c RunnerConfig, ou
 		RunnerConfig:   &c,
 		Include:        c.Include,
 		Exclude:        c.Exclude,
+		IncludeTags:    c.IncludeTags,
 		ShowTime:       c.ShowTime,
 		Output:         out,
 		ShowOnlyFailed: c.ShowOnlyFailed,
@@ -84,7 +84,7 @@ func RunTest(runContext *TestRunContext, ftwTest *test.FTWTest) error {
 
 	for _, testCase := range ftwTest.Tests {
 		// if we received a particular test ID, skip until we find it
-		if needToSkipTest(runContext.Include, runContext.Exclude, &testCase) {
+		if needToSkipTest(runContext, &testCase) {
 			runContext.Stats.addResultToStats(Skipped, &testCase)
 			continue
 		}
@@ -257,7 +257,11 @@ func markAndFlush(runContext *TestRunContext, dest *ftwhttp.Destination, stageID
 	return nil, fmt.Errorf("can't find log marker. Am I reading the correct log? Log file: %s", runContext.Config.LogFile)
 }
 
-func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *schema.Test) bool {
+func needToSkipTest(runContext *TestRunContext, testCase *schema.Test) bool {
+	include := runContext.Include
+	exclude := runContext.Exclude
+	includeTags := runContext.IncludeTags
+
 	// never skip enabled explicit inclusions
 	if include != nil {
 		if include.MatchString(testCase.IdString()) {
@@ -266,24 +270,31 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *sc
 		}
 	}
 
-	result := false
+	// if the test's tags do not match the passed ones
+	// it needs to be skipped
+	if includeTags != nil {
+		if !utils.MatchSlice(includeTags, testCase.Tags) {
+			return true
+		}
+	}
+
 	// if we need to exclude tests, and the ID matches,
 	// it needs to be skipped
 	if exclude != nil {
 		if exclude.MatchString(testCase.IdString()) {
-			result = true
+			return true
 		}
 	}
 
 	// if we need to include tests, but the ID does not match
 	// it needs to be skipped
 	if include != nil {
 		if !include.MatchString(testCase.IdString()) {
-			result = true
+			return true
 		}
 	}
 
-	return result
+	return false
 }
 
 func checkTestSanity(stage *schema.Stage) error {

runner/run_test.go

@@ -313,6 +313,24 @@ func (s *runTestSuite) TestRunTests_Run() {
 		s.Equal(res.Stats.TotalFailed(), 0, "failed to exclude test")
 	})
 
+	s.Run("count tests tagged with `tag-10`", func() {
+		res, err := Run(s.cfg, s.ftwTests, RunnerConfig{
+			IncludeTags: regexp.MustCompile("^tag-10$"),
+		}, s.out)
+		s.Require().NoError(err)
+		s.Len(res.Stats.Success, 1, "failed to incorporate tagged test")
+		s.Equal(res.Stats.TotalFailed(), 0, "failed to incorporate tagged test")
+	})
+
+	s.Run("count tests tagged with `tag-8` and `tag-10`", func() {
+		res, err := Run(s.cfg, s.ftwTests, RunnerConfig{
+			IncludeTags: regexp.MustCompile("^tag-8$|^tag-10$"),
+		}, s.out)
+		s.Require().NoError(err)
+		s.Len(res.Stats.Success, 2, "failed to incorporate tagged test")
+		s.Equal(res.Stats.TotalFailed(), 0, "failed to incorporate tagged test")
+	})
+
 	s.Run("test exceptions 1", func() {
 		res, err := Run(s.cfg, s.ftwTests, RunnerConfig{
 			Include: regexp.MustCompile("-1.*"),

runner/testdata/TestRunTests_Run.yaml

@@ -17,6 +17,9 @@ tests:
           expect_error: False
           status: 200
   - test_id: 8
+    tags:
+      - tag-8
+      - local
     description: "this test is number 8"
     stages:
       - input:
@@ -29,6 +32,9 @@ tests:
         output:
           status: 200
   - test_id: 10
+    tags:
+      - tag-10
+      - other
     stages:
       - input:
           dest_addr: "{{ .TestAddr }}"

runner/types.go

@@ -19,6 +19,8 @@ type RunnerConfig struct {
 	Include *regexp.Regexp
 	// Exclude is a regular expression to filter tests to exclude. If nil, no tests are excluded.
 	Exclude *regexp.Regexp
+	// IncludeTags is a regular expression to filter tests to count the ones tagged with the mathing label. If nil, no impact on test runner.
+	IncludeTags *regexp.Regexp
 	// ShowTime determines whether to show the time taken to run each test.
 	ShowTime bool
 	// ShowOnlyFailed will only output information related to failed tests
@@ -43,6 +45,7 @@ type TestRunContext struct {
 	RunnerConfig   *RunnerConfig
 	Include        *regexp.Regexp
 	Exclude        *regexp.Regexp
+	IncludeTags    *regexp.Regexp
 	ShowTime       bool
 	ShowOnlyFailed bool
 	Output         *output.Output

utils/slice.go

@@ -0,0 +1,18 @@
+// Copyright 2023 OWASP ModSecurity Core Rule Set Project
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	"regexp"
+)
+
+func MatchSlice(regex *regexp.Regexp, hayStack []string) bool {
+	for _, str := range hayStack {
+		if regex.MatchString(str) {
+			return true
+		}
+	}
+
+	return false
+}