Implement isolated (#300)

* feat: add support for 'isolated'

* fix: typo

Author: theseion

check/logs.go

@@ -54,5 +54,13 @@ func (c *FTWCheck) assertLogContains() bool {
 		}
 	}
 
+	if c.expected.Isolated {
+		ruleIds := c.log.TriggeredRules()
+		result = len(ruleIds) == 1
+		if !result {
+			log.Debug().Msgf("Found more than one triggered rule for isolated test: %v", ruleIds)
+		}
+	}
+
 	return result
 }

check/logs_test.go

@@ -151,3 +151,12 @@ func (s *checkLogsTestSuite) TestAssertLogNoExpectIds_Subset() {
 	s.check.expected.Log.NoExpectIds = []uint{123, 920300}
 	s.False(s.check.AssertLogs(), "Expected to find '920300'")
 }
+
+func (s *checkLogsTestSuite) TestAssertLogIsolated() {
+	s.check.expected.Log.ExpectIds = []uint{920300}
+	s.False(s.check.expected.Isolated)
+	s.True(s.check.AssertLogs(), "Expected to find 920300")
+
+	s.check.expected.Isolated = true
+	s.False(s.check.AssertLogs(), "Expected to find multiple IDs")
+}

go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/Masterminds/sprig v2.22.0+incompatible
-	github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240509130026-48cda85f1d5f
+	github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240513043400-8a1a71caae7a
 	github.com/go-logr/zerologr v1.2.3
 	github.com/goccy/go-yaml v1.9.2
 	github.com/google/uuid v1.6.0

go.sum

@@ -9,8 +9,8 @@ github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6e
 github.com/antchfx/xpath v1.2.3 h1:CCZWOzv5bAqjVv0offZ2LVgVYFbeldKQVuLNbViZdes=
 github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240509130026-48cda85f1d5f h1:lncLdzrrw40U6m5C26lQ7T/EYzzWIlQonnb3JYSaKPE=
-github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240509130026-48cda85f1d5f/go.mod h1:stf2FA6YhkPEMq30FzKvs97Qn5P9F7LHqWQbNGvGhNk=
+github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240513043400-8a1a71caae7a h1:YbO+KcDY2UN1mQ0pZpGWpjlfAnXBu6PvrA+MangtxTE=
+github.com/coreruleset/ftw-tests-schema v1.1.1-0.20240513043400-8a1a71caae7a/go.mod h1:stf2FA6YhkPEMq30FzKvs97Qn5P9F7LHqWQbNGvGhNk=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

runner/run.go

@@ -23,8 +23,6 @@ import (
 	"github.com/coreruleset/go-ftw/waflog"
 )
 
-var errBadTestInput = errors.New("ftw/run: bad test input: choose between data, encoded_request, or raw_request")
-
 // Run runs your tests with the specified Config.
 func Run(cfg *config.FTWConfiguration, tests []*test.FTWTest, c RunnerConfig, out *output.Output) (*TestRunContext, error) {
 	out.Println("%s", out.Message("** Running go-ftw!"))
@@ -147,8 +145,8 @@ func RunStage(runContext *TestRunContext, ftwCheck *check.FTWCheck, testCase sch
 	}
 
 	// Check sanity first
-	if checkTestSanity(testInput) {
-		return errBadTestInput
+	if err := checkTestSanity(&stage); err != nil {
+		return err
 	}
 
 	// Do not even run test if result is overridden. Just use the override and display the overridden result.
@@ -289,12 +287,23 @@ func needToSkipTest(include *regexp.Regexp, exclude *regexp.Regexp, testCase *sc
 	return result
 }
 
-func checkTestSanity(testInput test.Input) bool {
-	return (utils.IsNotEmpty(testInput.Data) && testInput.EncodedRequest != "") ||
-		//nolint:staticcheck
-		(utils.IsNotEmpty(testInput.Data) && testInput.RAWRequest != "") ||
-		//nolint:staticcheck
-		(testInput.EncodedRequest != "" && testInput.RAWRequest != "")
+func checkTestSanity(stage *schema.Stage) error {
+	if utils.IsNotEmpty(stage.Input.Data) && stage.Input.EncodedRequest != "" {
+		return errors.New("'data' and 'encoded_request' must not be set simultaneously")
+	}
+	//nolint:staticcheck
+	if utils.IsNotEmpty(stage.Input.Data) && stage.Input.RAWRequest != "" {
+		return errors.New("'data' and 'raw_request' must not be set simultaneously")
+	}
+	//nolint:staticcheck
+	if stage.Input.EncodedRequest != "" && stage.Input.RAWRequest != "" {
+		return errors.New("'encoded_request' and 'raw_request' must not be set simultaneously")
+	}
+	if len(stage.Output.Log.ExpectIds) != 1 && stage.Output.Isolated {
+		return errors.New("'isolated' is only valid if 'expected_ids' has exactly one entry")
+	}
+
+	return nil
 }
 
 func displayResult(testCase *schema.Test, rc *TestRunContext, result TestResult, roundTripTime time.Duration, stageTime time.Duration) {

runner/run_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/stretchr/testify/suite"
 
+	"github.com/coreruleset/ftw-tests-schema/types"
+	"github.com/coreruleset/go-ftw/check"
 	"github.com/coreruleset/go-ftw/config"
 	"github.com/coreruleset/go-ftw/ftwhttp"
 	"github.com/coreruleset/go-ftw/output"
@@ -530,3 +532,24 @@ func (s *runTestSuite) TestFailFast() {
 	s.Equal(1, res.Stats.TotalFailed(), "Oops, test run failed!")
 	s.Equal(2, res.Stats.Run)
 }
+
+func (s *runTestSuite) TestIsolatedSanity() {
+	rc := &TestRunContext{
+		Config: s.cfg,
+	}
+	stage := types.Stage{
+		Input: types.Input{},
+		Output: types.Output{
+			Isolated: true,
+			Log: types.Log{
+				ExpectIds: []uint{},
+			},
+		},
+	}
+	err := RunStage(rc, &check.FTWCheck{}, types.Test{}, stage)
+	s.ErrorContains(err, "'isolated' is only valid if 'expected_ids' has exactly one entry")
+
+	stage.Output.Log.ExpectIds = []uint{1, 2}
+	err = RunStage(rc, &check.FTWCheck{}, types.Test{}, stage)
+	s.ErrorContains(err, "'isolated' is only valid if 'expected_ids' has exactly one entry")
+}