Extend test ID generation by using test classname if present

Author: skipi

.semaphore/integration.yml

@@ -56,9 +56,4 @@ promotions:
     auto_promote_on:
       - result: passed
         branch:
-          - "^refs/tags/v*"
-
-  - name: Reports
-    pipeline_file: reports.yml
-    auto_promote:
-      when: true
+          - "^refs/tags/v*"

Makefile

@@ -16,6 +16,9 @@ regen:
 test:
 	gotestsum ./...
 
+test.watch:
+	gotestsum --watch ./...
+
 build:
 	CGO_ENABLED=0 go build -o bin/test-results
 

docs/id-generation.md

@@ -1,24 +1,24 @@
 ## ID generation
 
-This PR introduces `id` generator for tests results, test suites and tests.
+This PR introduces `id` generator for tests results, test suites, and tests.
 
 `id`'s are being generated in form of UUID strings.
 
-In order to generate consistent `id`'s between builds following method is implemented for all parsers:
+To generate consistent `id`'s between builds following method is implemented for all parsers:
 
-- ID generation for `Test results`(top level element)
+- ID generation for `Test results`(top-level element)
 
-  1. If element has an ID, generate UUID based on that ID
-  2. If element doesn't have an ID - generate UUID based on the `name` attribute
-  3. If element has framework name - generate UUID based on the `name` attribute and `framework`
-  4. Otherwise, generate uuid based on empty string `""`
+  1. If the element has an ID, generate UUID based on that ID
+  2. If the element doesn't have an ID - generate UUID based on the `name` attribute
+  3. If the element has a framework name - generate UUID based on the `name` attribute and `framework`
+  4. Otherwise, generate uuid based on an empty string `""`
 
 - ID generation for `Suites`
 
-  Same rules apply as for `Test results`, however every `Suite ID` is namespaced by `Test results` ID
+  The same rules apply as for `Test results` however every `Suite ID` is namespaced by `Test results` ID
 
 - ID generation for `Tests`
 
-  Same rules apply as for `Test results`, however every `Test ID` is namespaced by `Suite` ID
+  The same rules apply as for `Test results` however every `Test ID` is namespaced by `Suite` ID and `Test classname` if present
 
 For generating IDs we're using [UUID v3 generator](https://pkg.go.dev/github.com/google/uuid#NewMD5).

pkg/parser/types.go

@@ -265,14 +265,47 @@ func (me *Suite) Combine(other Suite) {
 			if me.hasTest(other.Tests[i]) == false {
 				me.Tests = append(me.Tests, other.Tests[i])
 			}
+
+			shouldReplace, indexToReplace := me.shouldReplaceTest(other.Tests[i])
+
+			if shouldReplace && indexToReplace != -1 {
+				me.Tests[indexToReplace] = other.Tests[i]
+			}
+
 		}
 
 		sort.SliceStable(me.Tests, func(i, j int) bool {
 			return me.Tests[i].ID < me.Tests[j].ID
 		})
 	}
 }
+func (me *Suite) shouldReplaceTest(test Test) (shouldReplace bool, foundIndex int) {
+	foundIndex = -1
+	shouldReplace = false
+	for i := range me.Tests {
+		if me.Tests[i].ID == test.ID {
+			foundIndex = i
+			break
+		}
+	}
 
+	if foundIndex == -1 {
+		return
+	} else {
+		foundTest := me.Tests[foundIndex]
+
+		if foundTest.State == StateSkipped {
+			shouldReplace = true
+			return
+		}
+		if foundTest.State == StatePassed && test.State == StateFailed || test.State == StateError {
+			shouldReplace = true
+			return
+		}
+
+		return
+	}
+}
 func (me *Suite) hasTest(test Test) bool {
 	for i := range me.Tests {
 		if me.Tests[i].ID == test.ID {
@@ -355,6 +388,10 @@ func (me *Test) EnsureID(s Suite) {
 		me.ID = me.Name
 	}
 
+	if me.Classname != "" {
+		me.ID = fmt.Sprintf("%s.%s", me.Classname, me.ID)
+	}
+
 	me.ID = UUID(uuid.MustParse(s.ID), me.ID).String()
 }
 

pkg/parser/types_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/semaphoreci/test-results/pkg/logger"
 	"github.com/stretchr/testify/assert"
 )
@@ -86,7 +87,32 @@ func Test_Suite_Combine(t *testing.T) {
 
 	suite.Combine(suiteToMerge)
 
-	assert.Equal(t, 3, len(suite.Tests))
+	assert.Equal(t, 3, len(suite.Tests), "Should combine tests properly given test name and test file")
+
+	test := NewTest()
+	test.ID = "4"
+	test.Name = "4.foo.4"
+	test.File = "foo.4"
+	test.State = StateSkipped
+
+	suiteToMerge = newSuite("1", "foo")
+	suiteToMerge.AppendTest(test)
+	suite.Combine(suiteToMerge)
+
+	assert.Equal(t, 4, len(suite.Tests))
+	assert.Equal(t, true, suite.Tests[len(suite.Tests)-1].State == StateSkipped)
+
+	test.State = StatePassed
+	suiteToMerge.AppendTest(test)
+	suite.Combine(suiteToMerge)
+	assert.Equal(t, 4, len(suite.Tests))
+	assert.Equal(t, true, suite.Tests[len(suite.Tests)-1].State == StatePassed, "If tests are the same, passed state should take priority over skipped state")
+
+	test.State = StateFailed
+	suiteToMerge.AppendTest(test)
+	suite.Combine(suiteToMerge)
+	assert.Equal(t, 4, len(suite.Tests))
+	assert.Equal(t, true, suite.Tests[len(suite.Tests)-1].State == StateFailed, "If tests are the same, failed state should take priority over passed state")
 }
 
 func Test_NewTest_Results(t *testing.T) {
@@ -298,6 +324,25 @@ func Test_NewFailure(t *testing.T) {
 	assert.IsType(t, obj, Failure{})
 }
 
+func Test_EnsureID(t *testing.T) {
+	suite := NewSuite()
+	suite.ID = uuid.NewString()
+
+	test := NewTest()
+	test.Name = "foo"
+	test.EnsureID(suite)
+
+	testToCompare := NewTest()
+	testToCompare.Name = "foo"
+	testToCompare.EnsureID(suite)
+
+	assert.Equal(t, true, test.ID == testToCompare.ID, "should have the same ID in the same suite")
+
+	testToCompare.Classname = "bar"
+	testToCompare.EnsureID(suite)
+	assert.Equal(t, false, test.ID == testToCompare.ID, "should have different ID in the same suite when classname differs")
+}
+
 func newTest(suite *Suite, id string, file string) {
 	test := NewTest()
 	test.ID = id

pkg/parsers/rspec_test.go

@@ -52,7 +52,7 @@ func Test_RSpec_ParseTestSuite(t *testing.T) {
 	assert.Equal(t, "bar", testResults.Suites[0].Tests[2].Name)
 
 	assert.Equal(t, "foo bar", testResults.Suites[1].Tests[0].Name)
-	assert.Equal(t, "977c3162-ea09-3747-8c87-6f8c78585f51", testResults.Suites[1].Tests[0].ID)
+	assert.Equal(t, "96eef1d9-d6ee-32c3-8de2-801a5c64e5c2", testResults.Suites[1].Tests[0].ID)
 	assert.Equal(t, "Foo", testResults.Suites[1].Tests[0].Classname)
 	assert.Equal(t, "foo/bar.o", testResults.Suites[1].Tests[0].File)
 	assert.Equal(t, time.Duration(123400000), testResults.Suites[1].Tests[0].Duration)

priv/merging/out.json

@@ -1 +1 @@
-{"testResults":[{"id":"4ae71336-e44b-39bf-b9d2-752e234818a5","name":"","framework":"","isDisabled":false,"suites":[{"id":"5adf0dab-e505-39fe-99c7-298ef43a8f09","name":"foo","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"03038e64-7367-39b9-8e6c-5f503139a200","file":"","classname":"","package":"","name":"foo.2","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"08a2a4c8-158c-300f-b951-9bb99f6f5a49","file":"","classname":"","package":"","name":"foo.6","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"58f2ffeb-dc64-3100-b81f-cfc2e9646b8b","file":"","classname":"","package":"","name":"foo.1","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"69fda129-a322-36c7-85b6-215eb8ce5c77","file":"","classname":"","package":"","name":"foo.5","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"7c947655-ee72-38d1-965d-35a6bdb0da6d","file":"","classname":"","package":"","name":"foo.4","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"b6941c19-0674-36aa-965c-badc07c512bd","file":"","classname":"","package":"","name":"foo.3","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":6,"passed":6,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""}],"summary":{"total":6,"passed":6,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"status":"success","statusMessage":""}]}
+{"testResults":[{"id":"4ae71336-e44b-39bf-b9d2-752e234818a5","name":"","framework":"","isDisabled":false,"suites":[{"id":"5adf0dab-e505-39fe-99c7-298ef43a8f09","name":"foo","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"03038e64-7367-39b9-8e6c-5f503139a200","file":"","classname":"","package":"","name":"foo.2","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"08a2a4c8-158c-300f-b951-9bb99f6f5a49","file":"","classname":"","package":"","name":"foo.6","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"58f2ffeb-dc64-3100-b81f-cfc2e9646b8b","file":"","classname":"","package":"","name":"foo.1","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"69fda129-a322-36c7-85b6-215eb8ce5c77","file":"","classname":"","package":"","name":"foo.5","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"7c947655-ee72-38d1-965d-35a6bdb0da6d","file":"","classname":"","package":"","name":"foo.4","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"b6941c19-0674-36aa-965c-badc07c512bd","file":"","classname":"","package":"","name":"foo.3","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":6,"passed":6,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""},{"id":"c348fb08-df86-3e06-a356-b951c48ea5a4","name":"bar","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"0727aa44-1fc5-3024-804f-6e5936caa480","file":"","classname":"","package":"","name":"bar.1","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":1,"passed":1,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"systemOut":"","systemErr":""}],"summary":{"total":7,"passed":7,"skipped":0,"error":0,"failed":0,"disabled":0,"duration":0},"status":"success","statusMessage":""}]}

priv/parsers/exunit/out.json

@@ -1 +1 @@
-{"testResults":[{"id":"31c127ba-6edc-33b1-9f53-fc4a54e74c82","name":"","framework":"exunit","isDisabled":false,"suites":[{"id":"75d8a69b-dd7b-3d03-8900-8bdaeb969469","name":"Calculator.AdderTest","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"18e6d9d8-6544-3850-b523-3490f4cdf426","file":"test/calculator/adder_test.exs","classname":"Elixir.Calculator.AdderTest","package":"","name":"test fails with 8531 + 6984 = 1547 0.09411569755491345","duration":0,"state":"failed","failure":{"message":"Assertion with == failed","type":"","body":"  1) test fails with 8531 + 6984 = 1547 0.09411569755491345 (Calculator.AdderTest)\n     test/calculator/adder_test.exs:15\n     Assertion with == failed\n     code:  assert Calculator.Adder.run(8531, 6984) == 1547\n     left:  15515\n     right: 1547\n     stacktrace:\n       test/calculator/adder_test.exs:16: (test)\n"},"error":null,"systemOut":"","systemErr":""},{"id":"cee69a36-c3fb-371f-8138-0fd6bc239b9a","file":"test/calculator/adder_test.exs","classname":"Elixir.Calculator.AdderTest","package":"","name":"test succeeds with 4464 + 2317 = 6781 0.7790935912260967","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"044488e6-1175-349a-9ca3-9dc046b214cc","file":"test/calculator/adder_test.exs","classname":"Elixir.Calculator.AdderTest","package":"","name":"test fails with 6981 + 3883 = 3098 0.5046993437439501","duration":0,"state":"failed","failure":{"message":"Assertion with == failed","type":"","body":"  3) test fails with 6981 + 3883 = 3098 0.5046993437439501 (Calculator.AdderTest)\n     test/calculator/adder_test.exs:15\n     Assertion with == failed\n     code:  assert Calculator.Adder.run(6981, 3883) == 3098\n     left:  10864\n     right: 3098\n     stacktrace:\n       test/calculator/adder_test.exs:16: (test)\n"},"error":null,"systemOut":"","systemErr":""}],"properties":{"date":"2021-03-29T12:19:22.461117Z","seed":"430315"},"summary":{"total":3,"passed":1,"skipped":0,"error":0,"failed":2,"disabled":0,"duration":0},"systemOut":"","systemErr":""}],"summary":{"total":3,"passed":1,"skipped":0,"error":0,"failed":2,"disabled":0,"duration":0},"status":"success","statusMessage":""}]}
+{"testResults":[{"id":"31c127ba-6edc-33b1-9f53-fc4a54e74c82","name":"","framework":"exunit","isDisabled":false,"suites":[{"id":"75d8a69b-dd7b-3d03-8900-8bdaeb969469","name":"Calculator.AdderTest","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"4e0ffbc5-3164-3d0a-9ffd-548c47677086","file":"test/calculator/adder_test.exs","classname":"Elixir.Calculator.AdderTest","package":"","name":"test fails with 8531 + 6984 = 1547 0.09411569755491345","duration":0,"state":"failed","failure":{"message":"Assertion with == failed","type":"","body":"  1) test fails with 8531 + 6984 = 1547 0.09411569755491345 (Calculator.AdderTest)\n     test/calculator/adder_test.exs:15\n     Assertion with == failed\n     code:  assert Calculator.Adder.run(8531, 6984) == 1547\n     left:  15515\n     right: 1547\n     stacktrace:\n       test/calculator/adder_test.exs:16: (test)\n"},"error":null,"systemOut":"","systemErr":""},{"id":"b7852759-c206-3e17-bc12-ab54f28b37dd","file":"test/calculator/adder_test.exs","classname":"Elixir.Calculator.AdderTest","package":"","name":"test succeeds with 4464 + 2317 = 6781 0.7790935912260967","duration":0,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"7b010ca6-1116-380c-a615-49fc04a91e51","file":"test/calculator/adder_test.exs","classname":"Elixir.Calculator.AdderTest","package":"","name":"test fails with 6981 + 3883 = 3098 0.5046993437439501","duration":0,"state":"failed","failure":{"message":"Assertion with == failed","type":"","body":"  3) test fails with 6981 + 3883 = 3098 0.5046993437439501 (Calculator.AdderTest)\n     test/calculator/adder_test.exs:15\n     Assertion with == failed\n     code:  assert Calculator.Adder.run(6981, 3883) == 3098\n     left:  10864\n     right: 3098\n     stacktrace:\n       test/calculator/adder_test.exs:16: (test)\n"},"error":null,"systemOut":"","systemErr":""}],"properties":{"date":"2021-03-29T12:19:22.461117Z","seed":"430315"},"summary":{"total":3,"passed":1,"skipped":0,"error":0,"failed":2,"disabled":0,"duration":0},"systemOut":"","systemErr":""}],"summary":{"total":3,"passed":1,"skipped":0,"error":0,"failed":2,"disabled":0,"duration":0},"status":"success","statusMessage":""}]}

priv/parsers/golang/out.json

@@ -1 +1 @@
-{"testResults":[{"id":"99ec6b78-8d28-33bb-9c4b-e38fd0000bf4","name":"Rspec Suite","framework":"rspec","isDisabled":false,"suites":[{"id":"7ec58d3e-ae4b-3245-964b-4fff0e02fc07","name":"spec/calculator/subtractor_spec.rb","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"96b79360-dc8d-343a-b747-7d79fcaf8829","file":"spec/calculator/subtractor_spec.rb","classname":"spec.calculator.subtractor_spec","package":"","name":"Calculator::Subtractor subtracts arguments #1","duration":97000,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"bf57bb75-ecbc-347f-8b53-25007a5a8952","file":"spec/calculator/subtractor_spec.rb","classname":"spec.calculator.subtractor_spec","package":"","name":"Calculator::Subtractor subtracts arguments #2","duration":146000,"state":"failed","failure":{"message":"\nexpected: 3\n     got: -1\n\n(compared using ==)\n","type":"RSpec::Expectations::ExpectationNotMetError","body":"Failure/Error: expect(result).to eq(3)\n\n  expected: 3\n       got: -1\n\n  (compared using ==)\n./spec/calculator/subtractor_spec.rb:11:in `block (2 levels) in \u003ctop (required)\u003e'"},"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":2,"passed":1,"skipped":0,"error":0,"failed":1,"disabled":0,"duration":243000},"systemOut":"","systemErr":""},{"id":"d1c92324-83c6-3660-b21b-bcb065629fe3","name":"spec/calculator/adder_spec.rb","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"79e0641c-37ab-37ed-815c-a81a1cf653cd","file":"spec/calculator/adder_spec.rb","classname":"spec.calculator.adder_spec","package":"","name":"Calculator::Adder sums arguments #1","duration":436000,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"619bba0e-4b2f-3210-9814-aee2f176126f","file":"spec/calculator/adder_spec.rb","classname":"spec.calculator.adder_spec","package":"","name":"Calculator::Adder sums arguments #2","duration":12473000,"state":"failed","failure":{"message":"\nexpected: -1\n     got: 3\n\n(compared using ==)\n","type":"RSpec::Expectations::ExpectationNotMetError","body":"Failure/Error: expect(result).to eq(-1)\n\n  expected: -1\n       got: 3\n\n  (compared using ==)\n./spec/calculator/adder_spec.rb:11:in `block (2 levels) in \u003ctop (required)\u003e'"},"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":2,"passed":1,"skipped":0,"error":0,"failed":1,"disabled":0,"duration":12909000},"systemOut":"","systemErr":""}],"summary":{"total":4,"passed":2,"skipped":0,"error":0,"failed":2,"disabled":0,"duration":13152000},"status":"success","statusMessage":""}]}
+{"testResults":[{"id":"99ec6b78-8d28-33bb-9c4b-e38fd0000bf4","name":"Rspec Suite","framework":"rspec","isDisabled":false,"suites":[{"id":"7ec58d3e-ae4b-3245-964b-4fff0e02fc07","name":"spec/calculator/subtractor_spec.rb","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"1f4b6322-3f70-3202-b267-4195ffd9215a","file":"spec/calculator/subtractor_spec.rb","classname":"spec.calculator.subtractor_spec","package":"","name":"Calculator::Subtractor subtracts arguments #1","duration":97000,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"7bffdfea-f4d8-3d19-a225-29e41760f3cf","file":"spec/calculator/subtractor_spec.rb","classname":"spec.calculator.subtractor_spec","package":"","name":"Calculator::Subtractor subtracts arguments #2","duration":146000,"state":"failed","failure":{"message":"\nexpected: 3\n     got: -1\n\n(compared using ==)\n","type":"RSpec::Expectations::ExpectationNotMetError","body":"Failure/Error: expect(result).to eq(3)\n\n  expected: 3\n       got: -1\n\n  (compared using ==)\n./spec/calculator/subtractor_spec.rb:11:in `block (2 levels) in \u003ctop (required)\u003e'"},"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":2,"passed":1,"skipped":0,"error":0,"failed":1,"disabled":0,"duration":243000},"systemOut":"","systemErr":""},{"id":"d1c92324-83c6-3660-b21b-bcb065629fe3","name":"spec/calculator/adder_spec.rb","isSkipped":false,"isDisabled":false,"timestamp":"","hostname":"","package":"","tests":[{"id":"23fb2b14-f356-3df8-b5d8-1debb5cf6bbd","file":"spec/calculator/adder_spec.rb","classname":"spec.calculator.adder_spec","package":"","name":"Calculator::Adder sums arguments #1","duration":436000,"state":"passed","failure":null,"error":null,"systemOut":"","systemErr":""},{"id":"e64fd812-2f70-35cb-a58b-6984dba36d01","file":"spec/calculator/adder_spec.rb","classname":"spec.calculator.adder_spec","package":"","name":"Calculator::Adder sums arguments #2","duration":12473000,"state":"failed","failure":{"message":"\nexpected: -1\n     got: 3\n\n(compared using ==)\n","type":"RSpec::Expectations::ExpectationNotMetError","body":"Failure/Error: expect(result).to eq(-1)\n\n  expected: -1\n       got: 3\n\n  (compared using ==)\n./spec/calculator/adder_spec.rb:11:in `block (2 levels) in \u003ctop (required)\u003e'"},"error":null,"systemOut":"","systemErr":""}],"properties":null,"summary":{"total":2,"passed":1,"skipped":0,"error":0,"failed":1,"disabled":0,"duration":12909000},"systemOut":"","systemErr":""}],"summary":{"total":4,"passed":2,"skipped":0,"error":0,"failed":2,"disabled":0,"duration":13152000},"status":"success","statusMessage":""}]}