feat: buildkite output format

cmd/main.go

@@ -161,6 +161,8 @@ Formats:
     testname                 print a line for each test and package
     testdox                  print a sentence for each test using gotestdox
     github-actions           testname format with github actions log grouping
+    buildkite                testname format with buildkite log grouping excluding passed test output
+    buildkite-verbose        testname format with buildkite log grouping including passed test output
     standard-quiet           standard go test format
     standard-verbose         standard go test -v format
 
@@ -336,7 +338,10 @@ func run(opts *options) error {
 }
 
 func finishRun(opts *options, exec *testjson.Execution, exitErr error) error {
-	testjson.PrintSummary(opts.stdout, exec, opts.hideSummary.value)
+	testjson.PrintSummaryWithOpts(opts.stdout, exec, testjson.SummaryOptions{
+		Summary: opts.hideSummary.value,
+		Format:  opts.format,
+	})
 
 	if err := writeJUnitFile(opts, exec); err != nil {
 		return fmt.Errorf("failed to write junit file: %w", err)

cmd/rerunfails.go

@@ -58,7 +58,10 @@ func rerunFailed(ctx context.Context, opts *options, scanConfig testjson.ScanCon
 
 	rec := newFailureRecorderFromExecution(scanConfig.Execution)
 	for attempts := 0; rec.count() > 0 && attempts < opts.rerunFailsMaxAttempts; attempts++ {
-		testjson.PrintSummary(opts.stdout, scanConfig.Execution, testjson.SummarizeNone)
+		testjson.PrintSummaryWithOpts(opts.stdout, scanConfig.Execution, testjson.SummaryOptions{
+			Summary: testjson.SummarizeNone,
+			Format:  opts.format,
+		})
 		opts.stdout.Write([]byte("\n")) //nolint:errcheck
 
 		nextRec := newFailureRecorder(scanConfig.Handler)

cmd/testdata/gotestsum-help-text

@@ -41,6 +41,8 @@ Formats:
     testname                 print a line for each test and package
     testdox                  print a sentence for each test using gotestdox
     github-actions           testname format with github actions log grouping
+    buildkite                testname format with buildkite log grouping excluding passed test output
+    buildkite-verbose        testname format with buildkite log grouping including passed test output
     standard-quiet           standard go test format
     standard-verbose         standard go test -v format
 

testjson/format.go

@@ -449,6 +449,10 @@ func NewEventFormatter(out io.Writer, format string, formatOpts FormatOptions) E
 		return pkgNameWithFailuresFormat(out, formatOpts)
 	case "github-actions", "github-action":
 		return githubActionsFormat(out)
+	case "buildkite":
+		return buildkiteFormat(out, false)
+	case "buildkite-verbose":
+		return buildkiteFormat(out, true)
 	default:
 		return nil
 	}
@@ -513,3 +517,91 @@ func githubActionsFormat(out io.Writer) EventFormatter {
 		return buf.Flush()
 	})
 }
+
+func buildkiteFormat(out io.Writer, verbose bool) EventFormatter {
+	const (
+		bkMarkerDeemphasized = "~~~"
+		bkMarkerCollapsed    = "---"
+		bkMarkerExpanded     = "+++"
+	)
+	markers := [3]string{bkMarkerDeemphasized, bkMarkerCollapsed, bkMarkerExpanded}
+
+	buf := bufio.NewWriter(out)
+
+	type name struct {
+		Package string
+		Test    string
+	}
+	output := map[name][]string{}
+	packageOutput := map[string][]string{}
+
+	printOutput := func(output []string, didNotPass bool) {
+		if verbose || didNotPass {
+			// prevent erroneous groups from test output by prefixing lines with markers by an empty string
+			for _, item := range output {
+				for _, bkMarker := range markers {
+					if strings.HasPrefix(item, bkMarker) {
+						buf.WriteString(" ")
+					}
+				}
+				buf.WriteString(item)
+			}
+		}
+	}
+
+	return eventFormatterFunc(func(event TestEvent, exec *Execution) error {
+		key := name{Package: event.Package, Test: event.Test}
+
+		// test case event
+		if event.Test != "" {
+			if event.Action == ActionOutput {
+				if !isFramingLine(event.Output, event.Test) {
+					output[key] = append(output[key], event.Output)
+				}
+				return nil
+			}
+
+			if event.Action.IsTerminal() {
+				marker := bkMarkerDeemphasized
+				switch event.Action {
+				case ActionFail:
+					marker = bkMarkerCollapsed
+				}
+				buf.WriteString(marker)
+				buf.WriteString(" ")
+				testNameFormatTestEvent(buf, event)
+				printOutput(output[key], event.Action != ActionPass)
+				delete(output, key)
+				return buf.Flush()
+			}
+
+			return nil
+		}
+		// package event
+
+		if event.Action == ActionOutput {
+			packageOutput[event.Package] = append(packageOutput[event.Package], event.Output)
+			return nil
+		}
+		if !event.Action.IsTerminal() {
+			return nil
+		}
+
+		marker := bkMarkerDeemphasized
+		switch event.Action {
+		case ActionFail:
+			marker = bkMarkerCollapsed
+		}
+
+		result := colorEvent(event)(strings.ToUpper(string(event.Action)))
+		pkg := exec.Package(event.Package)
+		if event.Action == ActionSkip || (event.Action == ActionPass && pkg.Total == 0) {
+			event.Action = ActionSkip // always color these as skip actions
+			result = colorEvent(event)("EMPTY")
+		}
+
+		fmt.Fprintf(buf, "%s %s Package %s", marker, result, packageLine(event, exec.Package(event.Package)))
+		printOutput(packageOutput[event.Package], event.Action != ActionPass)
+		return buf.Flush()
+	})
+}

testjson/format_test.go

@@ -172,6 +172,16 @@ func TestFormats_DefaultGoTestJson(t *testing.T) {
 			format:      githubActionsFormat,
 			expectedOut: "format/github-actions.out",
 		},
+		{
+			name:        "buildkite-verbose",
+			format:      func(out io.Writer) EventFormatter { return buildkiteFormat(out, true) },
+			expectedOut: "format/buildkite-verbose.out",
+		},
+		{
+			name:        "buildkite",
+			format:      func(out io.Writer) EventFormatter { return buildkiteFormat(out, false) },
+			expectedOut: "format/buildkite.out",
+		},
 	}
 
 	for _, tc := range testCases {

testjson/summary.go

@@ -64,19 +64,36 @@ func NewSummary(value string) (Summary, bool) {
 	return s, ok
 }
 
+type SummaryPrinter func(out io.Writer, execution *Execution, opts Summary)
+
+type SummaryOptions struct {
+	Summary Summary
+	Format  string
+}
+
+// PrintSummaryWithOpts prints the summary of a test Execution with specific options.
+func PrintSummaryWithOpts(out io.Writer, execution *Execution, opts SummaryOptions) {
+	switch opts.Format {
+	case "buildkite", "buildkite-verbose":
+		// put the summary into an expanded section
+		fmt.Fprintln(out, "+++ Summary")
+	}
+	PrintSummary(out, execution, opts.Summary)
+}
+
 // PrintSummary of a test Execution. Prints a section for each summary type
 // followed by a DONE line to out.
-func PrintSummary(out io.Writer, execution *Execution, opts Summary) {
-	execSummary := newExecSummary(execution, opts)
-	if opts.Includes(SummarizeSkipped) {
+func PrintSummary(out io.Writer, execution *Execution, summary Summary) {
+	execSummary := newExecSummary(execution, summary)
+	if summary.Includes(SummarizeSkipped) {
 		writeTestCaseSummary(out, execSummary, formatSkipped())
 	}
-	if opts.Includes(SummarizeFailed) {
+	if summary.Includes(SummarizeFailed) {
 		writeTestCaseSummary(out, execSummary, formatFailed())
 	}
 
 	errors := execution.Errors()
-	if opts.Includes(SummarizeErrors) {
+	if summary.Includes(SummarizeErrors) {
 		writeErrorSummary(out, errors)
 	}
 

testjson/summary_test.go

@@ -2,6 +2,7 @@ package testjson
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"strings"
 	"testing"
@@ -319,6 +320,32 @@ func TestPrintSummary(t *testing.T) {
 	}
 }
 
+func TestSummary_Formats_DefaultGoTestJson(t *testing.T) {
+	run := func(t *testing.T, format string) {
+		exec, err := ScanTestOutput(scanConfigFromGolden("input/go-test-json.out")(t))
+		assert.NilError(t, err)
+
+		var out bytes.Buffer
+		PrintSummaryWithOpts(&out, exec, SummaryOptions{
+			Summary: SummarizeAll,
+			Format:  format,
+		})
+		golden.Assert(t, out.String(), fmt.Sprintf("format-summary/%s.out", format))
+	}
+
+	formats := []string{
+		"none",
+		"buildkite",
+		"buildkite-verbose",
+	}
+
+	for _, format := range formats {
+		t.Run(format, func(t *testing.T) {
+			run(t, format)
+		})
+	}
+}
+
 func scanConfigFromGolden(filename string) func(t *testing.T) ScanConfig {
 	return func(t *testing.T) ScanConfig {
 		return ScanConfig{Stdout: bytes.NewReader(golden.Get(t, filename))}

testjson/testdata/format-summary/buildkite-verbose.out

@@ -0,0 +1,64 @@
++++ Summary
+
+=== Skipped
+=== SKIP: testjson/internal/good TestSkipped (0.00s)
+    good_test.go:23: 
+
+=== SKIP: testjson/internal/good TestSkippedWitLog (0.00s)
+    good_test.go:27: the skip message
+
+=== SKIP: testjson/internal/withfails TestSkipped (0.00s)
+    fails_test.go:26: 
+
+=== SKIP: testjson/internal/withfails TestSkippedWitLog (0.00s)
+    fails_test.go:30: the skip message
+
+=== SKIP: testjson/internal/withfails TestTimeout (0.00s)
+    timeout_test.go:13: skipping slow test
+
+=== Failed
+=== FAIL: testjson/internal/badmain  (0.00s)
+sometimes main can exit 2
+FAIL	gotest.tools/gotestsum/testjson/internal/badmain	0.001s
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures/a (0.00s)
+    fails_test.go:50: failed sub a
+    --- FAIL: TestNestedParallelFailures/a (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures/d (0.00s)
+    fails_test.go:50: failed sub d
+    --- FAIL: TestNestedParallelFailures/d (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures/c (0.00s)
+    fails_test.go:50: failed sub c
+    --- FAIL: TestNestedParallelFailures/c (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures/b (0.00s)
+    fails_test.go:50: failed sub b
+    --- FAIL: TestNestedParallelFailures/b (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestParallelTheFirst (0.01s)
+    fails_test.go:29: failed the first
+
+=== FAIL: testjson/internal/parallelfails TestParallelTheThird (0.00s)
+    fails_test.go:41: failed the third
+
+=== FAIL: testjson/internal/parallelfails TestParallelTheSecond (0.01s)
+    fails_test.go:35: failed the second
+
+=== FAIL: testjson/internal/withfails TestFailed (0.00s)
+    fails_test.go:34: this failed
+
+=== FAIL: testjson/internal/withfails TestFailedWithStderr (0.00s)
+this is stderr
+    fails_test.go:43: also failed
+
+=== FAIL: testjson/internal/withfails TestNestedWithFailure/c (0.00s)
+    fails_test.go:65: failed
+    --- FAIL: TestNestedWithFailure/c (0.00s)
+
+=== FAIL: testjson/internal/withfails TestNestedWithFailure (0.00s)
+
+DONE 59 tests, 5 skipped, 13 failures in 0.157s

testjson/testdata/format-summary/buildkite.out

@@ -0,0 +1,64 @@
++++ Summary
+
+=== Skipped
+=== SKIP: testjson/internal/good TestSkipped (0.00s)
+    good_test.go:23: 
+
+=== SKIP: testjson/internal/good TestSkippedWitLog (0.00s)
+    good_test.go:27: the skip message
+
+=== SKIP: testjson/internal/withfails TestSkipped (0.00s)
+    fails_test.go:26: 
+
+=== SKIP: testjson/internal/withfails TestSkippedWitLog (0.00s)
+    fails_test.go:30: the skip message
+
+=== SKIP: testjson/internal/withfails TestTimeout (0.00s)
+    timeout_test.go:13: skipping slow test
+
+=== Failed
+=== FAIL: testjson/internal/badmain  (0.00s)
+sometimes main can exit 2
+FAIL	gotest.tools/gotestsum/testjson/internal/badmain	0.001s
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures/a (0.00s)
+    fails_test.go:50: failed sub a
+    --- FAIL: TestNestedParallelFailures/a (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures/d (0.00s)
+    fails_test.go:50: failed sub d
+    --- FAIL: TestNestedParallelFailures/d (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures/c (0.00s)
+    fails_test.go:50: failed sub c
+    --- FAIL: TestNestedParallelFailures/c (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures/b (0.00s)
+    fails_test.go:50: failed sub b
+    --- FAIL: TestNestedParallelFailures/b (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestNestedParallelFailures (0.00s)
+
+=== FAIL: testjson/internal/parallelfails TestParallelTheFirst (0.01s)
+    fails_test.go:29: failed the first
+
+=== FAIL: testjson/internal/parallelfails TestParallelTheThird (0.00s)
+    fails_test.go:41: failed the third
+
+=== FAIL: testjson/internal/parallelfails TestParallelTheSecond (0.01s)
+    fails_test.go:35: failed the second
+
+=== FAIL: testjson/internal/withfails TestFailed (0.00s)
+    fails_test.go:34: this failed
+
+=== FAIL: testjson/internal/withfails TestFailedWithStderr (0.00s)
+this is stderr
+    fails_test.go:43: also failed
+
+=== FAIL: testjson/internal/withfails TestNestedWithFailure/c (0.00s)
+    fails_test.go:65: failed
+    --- FAIL: TestNestedWithFailure/c (0.00s)
+
+=== FAIL: testjson/internal/withfails TestNestedWithFailure (0.00s)
+
+DONE 59 tests, 5 skipped, 13 failures in 0.157s