Storing test summaries (#49)

* feat(summary): add test summary file to artifacts

* fix(summary): fix integration test

* fix(summary): fix integration test implicit dependencies

* fix(summary): add checkout command

* fix(summary): add debug logs

* fix(integration): add global job config

* fix(integration): correct fixtures for test results summary

* test(summary): unit test summary merge method

* feat(publish): add job level summaries

* chore: cleanup TODOs

* fix: update fixture duration data

Co-authored-by: Mikołaj Kutryj <[email protected]>

Author: ralopes

.semaphore/integration.yml

@@ -4,17 +4,18 @@ agent:
   machine:
     type: e1-standard-2
     os_image: ubuntu2004
+global_job_config:
+  prologue:
+    commands:
+      - artifact pull workflow bin/test-results -d /tmp/test-results
+      - sudo mv /tmp/test-results /usr/local/bin/test-results
+      - sudo chmod +x /usr/local/bin/test-results
+      - test-results --version
+      - checkout
 blocks:
   - name: Integration tests
     dependencies: []
     task:
-      prologue:
-        commands:
-          - artifact pull workflow bin/test-results -d /tmp/test-results
-          - sudo mv /tmp/test-results /usr/local/bin/test-results
-          - sudo chmod +x /usr/local/bin/test-results
-          - test-results --version
-          - checkout
       jobs:
         - name: "Parsers"
           matrix:
@@ -30,19 +31,29 @@ blocks:
             - test-results publish priv/merging
             - artifact pull job test-results/junit.json -d /tmp/junit.json
             - diff priv/merging/out.json /tmp/junit.json
+  - name: Integration tests - generate pipeline report
+    dependencies:
+      - Integration tests
+    task:
+      jobs:
+        - name: "generate pipeline report"
+          commands:
+            - test-results gen-pipeline-report
 
   - name: Integration tests - workflow level
     dependencies:
-      - Integration tests
+      - Integration tests - generate pipeline report
     task:
       jobs:
         - name: "Generate pipeline report"
           commands:
-            - checkout
-            - test-results gen-pipeline-report
             - artifact pull workflow test-results/${SEMAPHORE_PIPELINE_ID}.json -d /tmp/junit.json
             - artifact push job /tmp/junit.json -d test-results/junit.json
             - diff /tmp/junit.json priv/workflow/out.json
+        - name: "Generate pipeline summary"
+          commands:
+            - artifact pull workflow test-results/${SEMAPHORE_PIPELINE_ID}-summary.json -d /tmp/summary.json
+            - diff /tmp/summary.json priv/workflow/summary-out.json
 
 promotions:
   - name: Release

cmd/compile.go

@@ -32,7 +32,7 @@ var compileCmd = &cobra.Command{
 	Short: "parses xml files to well defined json schema",
 	Long: `Parses xml file to well defined json schema
 
-	It traverses through directory sturcture specificed by <xml-file-path> and compiles
+	It traverses through directory structure specified by <xml-file-path> and compiles
 	every .xml file.
 	`,
 	Args: cobra.MinimumNArgs(2),

cmd/gen-pipeline-report.go

@@ -18,6 +18,7 @@ limitations under the License.
 
 import (
 	"encoding/json"
+	"github.com/semaphoreci/test-results/pkg/parser"
 	"io/ioutil"
 	"os"
 	"path"
@@ -44,7 +45,6 @@ var genPipelineReportCmd = &cobra.Command{
 		}
 
 		var dir string
-		removeDir := true
 
 		pipelineID, found := os.LookupEnv("SEMAPHORE_PIPELINE_ID")
 		if !found {
@@ -58,25 +58,21 @@ var genPipelineReportCmd = &cobra.Command{
 				logger.Error("Creating temporary directory failed %v", err)
 				return err
 			}
+			defer os.Remove(dir)
 
 			dir, err = cli.PullArtifacts("workflow", path.Join("test-results", pipelineID), dir, cmd)
 			if err != nil {
 				return err
 			}
 		} else {
 			dir = args[0]
-			removeDir = false
 		}
 
 		result, err := cli.MergeFiles(dir, cmd)
 		if err != nil {
 			return err
 		}
 
-		if removeDir {
-			defer os.Remove(dir)
-		}
-
 		jsonData, err := json.Marshal(result)
 		if err != nil {
 			logger.Error("Marshaling results failed with: %v", err)
@@ -87,16 +83,42 @@ var genPipelineReportCmd = &cobra.Command{
 		if err != nil {
 			return err
 		}
+		defer os.Remove(fileName)
 
 		_, err = cli.PushArtifacts("workflow", fileName, path.Join("test-results", pipelineID+".json"), cmd)
 		if err != nil {
 			return err
 		}
 
-		defer os.Remove(fileName)
+		return pushSummaries(result.TestResults, "workflow", path.Join("test-results", pipelineID+"-summary.json"), cmd)
+	},
+}
 
+func pushSummaries(testResult []parser.TestResults, level, path string, cmd *cobra.Command) error {
+	if len(testResult) == 0 {
+		logger.Info("no test results to process")
 		return nil
-	},
+	}
+
+	logger.Info("starting to generate summary")
+	summaryReport := parser.Summary{}
+	for _, results := range testResult {
+		summaryReport.Merge(&results.Summary)
+	}
+
+	jsonSummary, err := json.Marshal(summaryReport)
+	if err != nil {
+		return err
+	}
+
+	summaryFileName, err := cli.WriteToTmpFile(jsonSummary)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(summaryFileName)
+
+	_, err = cli.PushArtifacts(level, summaryFileName, path, cmd)
+	return err
 }
 
 func init() {

cmd/publish.go

@@ -34,7 +34,7 @@ var publishCmd = &cobra.Command{
 	Short: "parses xml file to well defined json schema and publishes results to artifacts storage",
 	Long: `Parses xml file to well defined json schema and publishes results to artifacts storage
 
-	It traverses through directory sturcture specificed by <xml-file-path>, compiles
+	It traverses through directory structure specified by <xml-file-path>, compiles
 	every .xml file and publishes it as one artifact.
 	`,
 	Args: cobra.MinimumNArgs(1),
@@ -104,6 +104,10 @@ var publishCmd = &cobra.Command{
 			return err
 		}
 
+		if err = pushSummaries(result.TestResults, "job", path.Join("test-results", "summary.json"), cmd); err != nil {
+			return err
+		}
+
 		pipelineID, found := os.LookupEnv("SEMAPHORE_PIPELINE_ID")
 		if !found {
 			logger.Error("SEMAPHORE_PIPELINE_ID env is missing")

pkg/cli/cli.go

@@ -319,7 +319,6 @@ func MergeFiles(path string, cmd *cobra.Command) (*parser.Result, error) {
 }
 
 // Load ...
-// [TODO]: Test this
 func Load(path string) (*parser.Result, error) {
 	var result parser.Result
 	jsonFile, err := os.Open(path) // #nosec

pkg/parser/types.go

@@ -39,7 +39,6 @@ const (
 )
 
 // Result ...
-// [TODO] Better name is required...
 type Result struct {
 	TestResults []TestResults `json:"testResults"`
 }
@@ -316,7 +315,6 @@ func (me *Suite) hasTest(test Test) bool {
 }
 
 // Aggregate all tests in suite
-// TODO: add flag to skip aggregating already present data
 func (me *Suite) Aggregate() {
 	summary := Summary{}
 
@@ -440,6 +438,17 @@ type Summary struct {
 	Duration time.Duration `json:"duration"`
 }
 
+//Merge merges two summaries together summing each field
+func (s *Summary) Merge(withSummary *Summary) {
+	s.Total += withSummary.Total
+	s.Passed += withSummary.Passed
+	s.Skipped += withSummary.Skipped
+	s.Error += withSummary.Error
+	s.Failed += withSummary.Failed
+	s.Disabled += withSummary.Disabled
+	s.Duration += withSummary.Duration
+}
+
 // UUID ...
 func UUID(id uuid.UUID, str string) uuid.UUID {
 	return uuid.NewMD5(id, []byte(str))

pkg/parser/types_test.go

@@ -309,6 +309,33 @@ func Test_Suite_Aggregate(t *testing.T) {
 	assert.Equal(t, Summary{Total: 5, Passed: 1, Failed: 1, Skipped: 1, Error: 1, Disabled: 1, Duration: 110}, suite.Summary, "should sum up tests duration when there is no suite duration present")
 }
 
+func Test_Summary_Merge(t *testing.T) {
+	summary1 := Summary{Total: 10, Passed: 6, Failed: 1, Skipped: 1, Error: 1, Disabled: 1, Duration: 10}
+	summary2 := Summary{Total: 20, Passed: 1, Failed: 16, Skipped: 1, Error: 1, Disabled: 1, Duration: 100}
+	summary3 := Summary{Total: 15, Passed: 10, Failed: 2, Skipped: 1, Error: 1, Disabled: 1, Duration: 10}
+	summary4 := Summary{Total: 25, Passed: 2, Failed: 1, Skipped: 20, Error: 1, Disabled: 1, Duration: 105}
+
+	result := Summary{}
+	for _, s := range []Summary{summary1, summary2, summary3, summary4} {
+		result.Merge(&s)
+	}
+
+	assert.Equal(t, Summary{
+		Total:    70,
+		Passed:   19,
+		Skipped:  23,
+		Error:    4,
+		Failed:   20,
+		Disabled: 4,
+		Duration: 225,
+	}, result)
+
+	result = Summary{}
+	result.Merge(&Summary{})
+	assert.Equal(t, Summary{}, result, "empty summaries should be zeroed")
+
+}
+
 func Test_NewTest(t *testing.T) {
 	test := NewTest()
 

priv/workflow/summary-out.json

@@ -0,0 +1 @@
+{"total":92,"passed":88,"skipped":0,"error":0,"failed":4,"disabled":0,"duration":188211000}