Skip to content

Commit a400260

Browse files
vijaiaeroastrovijaiaeroastro
committed
Include total must pass and must fail in all integration test outputs
1 parent 26a0c05 commit a400260

File tree

1 file changed

+64
-42
lines changed

1 file changed

+64
-42
lines changed

‎.github/workflows/build.yml

Lines changed: 64 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -605,9 +605,13 @@ jobs:
605605
echo "Latest SDK execution time in seconds: ${LATEST_TOTAL_SECONDS}"
606606
607607
LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $1}')
608+
LATEST_TOTAL_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $3}' | tr -d '()')
608609
LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $1}')
609-
echo "MUSTPASS: ${LATEST_MUSTPASS}"
610-
echo "MUSTFAIL: ${LATEST_MUSTFAIL}"
610+
LATEST_TOTAL_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $3}' | tr -d '()')
611+
612+
613+
echo "MUSTPASS: ${LATEST_MUSTPASS} / ${LATEST_TOTAL_MUSTPASS}"
614+
echo "MUSTFAIL: ${LATEST_MUSTFAIL} / ${LATEST_TOTAL_MUSTFAIL}"
611615
612616
613617
integration-tests-last-two-releases:
@@ -699,36 +703,43 @@ jobs:
699703
700704
- name: Compare results (Checks the total python script execution time and must pass/fail counts)
701705
run: |
702-
LATEST_TIME=$(grep "Elapsed (wall clock) time" test_suites/latest_sdk_test.log | awk '{print $8}')
703-
SECOND_LATEST_TIME=$(grep "Elapsed (wall clock) time" test_suites/second_latest_sdk_test.log | awk '{print $8}')
704-
LATEST_TOTAL_SECONDS=$(echo $LATEST_TIME | awk -F: '{ print ($1 * 60) + $2 }')
705-
SECOND_LATEST_TOTAL_SECONDS=$(echo $SECOND_LATEST_TIME | awk -F: '{ print ($1 * 60) + $2 }')
706-
echo "Latest SDK execution time in seconds: ${LATEST_TOTAL_SECONDS}"
707-
echo "Second Latest SDK execution time in seconds: ${SECOND_LATEST_TOTAL_SECONDS}"
708-
709-
LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $1}')
710-
SECOND_LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/second_latest_sdk_test.log | awk '{print $1}')
711-
LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $1}')
712-
SECOND_LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/second_latest_sdk_test.log | awk '{print $1}')
713-
echo "Latest MUSTPASS: ${LATEST_MUSTPASS}"
714-
echo "Second Latest MUSTPASS: ${SECOND_LATEST_MUSTPASS}"
715-
echo "Latest MUSTFAIL: ${LATEST_MUSTFAIL}"
716-
echo "Second Latest MUSTFAIL: ${SECOND_LATEST_MUSTFAIL}"
717-
718-
# Compare the total seconds
719-
if (( $(echo "$LATEST_TOTAL_SECONDS < $SECOND_LATEST_TOTAL_SECONDS" | bc -l) )); then
720-
echo "New release is better in execution time"
721-
else
722-
echo "New release is worse in execution time"
723-
fi
724-
725-
# Compare MUSTPASS and MUSTFAIL counts
726-
if [ "$LATEST_MUSTPASS" != "$SECOND_LATEST_MUSTPASS" ] || [ "$LATEST_MUSTFAIL" != "$SECOND_LATEST_MUSTFAIL" ]; then
727-
echo "MUSTPASS or MUSTFAIL counts have changed"
728-
exit 1
729-
else
730-
echo "MUSTPASS and MUSTFAIL counts are consistent"
731-
fi
706+
LATEST_TIME=$(grep "Elapsed (wall clock) time" test_suites/latest_sdk_test.log | awk '{print $8}')
707+
SECOND_LATEST_TIME=$(grep "Elapsed (wall clock) time" test_suites/second_latest_sdk_test.log | awk '{print $8}')
708+
LATEST_TOTAL_SECONDS=$(echo $LATEST_TIME | awk -F: '{ print ($1 * 60) + $2 }')
709+
SECOND_LATEST_TOTAL_SECONDS=$(echo $SECOND_LATEST_TIME | awk -F: '{ print ($1 * 60) + $2 }')
710+
echo "Latest SDK execution time in seconds: ${LATEST_TOTAL_SECONDS}"
711+
echo "Second Latest SDK execution time in seconds: ${SECOND_LATEST_TOTAL_SECONDS}"
712+
713+
LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $1}')
714+
LATEST_TOTAL_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $3}' | tr -d '()')
715+
SECOND_LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/second_latest_sdk_test.log | awk '{print $1}')
716+
SECOND_LATEST_TOTAL_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/second_latest_sdk_test.log | awk '{print $3}' | tr -d '()')
717+
718+
LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $1}')
719+
LATEST_TOTAL_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $3}' | tr -d '()')
720+
SECOND_LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/second_latest_sdk_test.log | awk '{print $1}')
721+
SECOND_LATEST_TOTAL_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/second_latest_sdk_test.log | awk '{print $3}' | tr -d '()')
722+
723+
echo "Latest MUSTPASS: ${LATEST_MUSTPASS} / ${LATEST_TOTAL_MUSTPASS}"
724+
echo "Second Latest MUSTPASS: ${SECOND_LATEST_MUSTPASS} / ${SECOND_LATEST_TOTAL_MUSTPASS}"
725+
echo "Latest MUSTFAIL: ${LATEST_MUSTFAIL} / ${LATEST_TOTAL_MUSTFAIL}"
726+
echo "Second Latest MUSTFAIL: ${SECOND_LATEST_MUSTFAIL} / ${SECOND_LATEST_TOTAL_MUSTFAIL}"
727+
728+
# Compare the total seconds
729+
if (( $(echo "$LATEST_TOTAL_SECONDS < $SECOND_LATEST_TOTAL_SECONDS" | bc -l) )); then
730+
echo "New release is better in execution time"
731+
else
732+
echo "New release is worse in execution time"
733+
fi
734+
735+
# Compare MUSTPASS and MUSTFAIL counts
736+
if [ "$LATEST_MUSTPASS" != "$SECOND_LATEST_MUSTPASS" ] || [ "$LATEST_MUSTFAIL" != "$SECOND_LATEST_MUSTFAIL" ]; then
737+
echo "MUSTPASS or MUSTFAIL counts have changed"
738+
exit 1
739+
else
740+
echo "MUSTPASS and MUSTFAIL counts are consistent"
741+
fi
742+
732743
733744
734745
integration-tests-latest-commit:
@@ -787,13 +798,18 @@ jobs:
787798
echo "Latest SDK execution time in seconds: ${LATEST_TOTAL_SECONDS}"
788799
789800
LATEST_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $1}')
801+
LATEST_TOTAL_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_sdk_test.log | awk '{print $3}' | tr -d '()')
790802
LATEST_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $1}')
791-
echo "MUSTPASS: ${LATEST_MUSTPASS}"
792-
echo "MUSTFAIL: ${LATEST_MUSTFAIL}"
803+
LATEST_TOTAL_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_sdk_test.log | awk '{print $3}' | tr -d '()')
804+
805+
echo "MUSTPASS: ${LATEST_MUSTPASS} / ${LATEST_TOTAL_MUSTPASS}"
806+
echo "MUSTFAIL: ${LATEST_MUSTFAIL} / ${LATEST_TOTAL_MUSTFAIL}"
793807
794808
# Save counts for comparison in other jobs
795809
echo "LATEST_MUSTPASS=${LATEST_MUSTPASS}" >> $GITHUB_ENV
810+
echo "LATEST_TOTAL_MUSTPASS=${LATEST_TOTAL_MUSTPASS}" >> $GITHUB_ENV
796811
echo "LATEST_MUSTFAIL=${LATEST_MUSTFAIL}" >> $GITHUB_ENV
812+
echo "LATEST_TOTAL_MUSTFAIL=${LATEST_TOTAL_MUSTFAIL}" >> $GITHUB_ENV
797813
798814
integration-test-last-commit-and-last-release:
799815
runs-on: ubuntu-20.04
@@ -894,13 +910,19 @@ jobs:
894910
echo "Latest release SDK execution time in seconds: ${LATEST_RELEASE_TOTAL_SECONDS}"
895911
896912
LATEST_COMMIT_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_commit_sdk_test.log | awk '{print $1}')
913+
LATEST_TOTAL_COMMIT_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_commit_sdk_test.log | awk '{print $3}' | tr -d '()')
897914
LATEST_RELEASE_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_release_sdk_test.log | awk '{print $1}')
915+
LATEST_TOTAL_RELEASE_MUSTPASS=$(grep "MUSTPASS files passed" test_suites/latest_release_sdk_test.log | awk '{print $3}' | tr -d '()')
916+
898917
LATEST_COMMIT_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_commit_sdk_test.log | awk '{print $1}')
918+
LATEST_TOTAL_COMMIT_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_commit_sdk_test.log | awk '{print $3}' | tr -d '()')
899919
LATEST_RELEASE_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_release_sdk_test.log | awk '{print $1}')
900-
echo "Latest commit MUSTPASS: ${LATEST_COMMIT_MUSTPASS}"
901-
echo "Latest release MUSTPASS: ${LATEST_RELEASE_MUSTPASS}"
902-
echo "Latest commit MUSTFAIL: ${LATEST_COMMIT_MUSTFAIL}"
903-
echo "Latest release MUSTFAIL: ${LATEST_RELEASE_MUSTFAIL}"
920+
LATEST_TOTAL_RELEASE_MUSTFAIL=$(grep "MUSTFAIL files failed" test_suites/latest_release_sdk_test.log | awk '{print $3}' | tr -d '()')
921+
922+
echo "Latest commit MUSTPASS: ${LATEST_COMMIT_MUSTPASS} / ${LATEST_TOTAL_COMMIT_MUSTPASS}"
923+
echo "Latest release MUSTPASS: ${LATEST_RELEASE_MUSTPASS} / ${LATEST_TOTAL_RELEASE_MUSTPASS}"
924+
echo "Latest commit MUSTFAIL: ${LATEST_COMMIT_MUSTFAIL} / ${LATEST_TOTAL_COMMIT_MUSTFAIL}"
925+
echo "Latest release MUSTFAIL: ${LATEST_RELEASE_MUSTFAIL} / ${LATEST_TOTAL_RELEASE_MUSTFAIL}"
904926
905927
# Compare MUSTPASS and MUSTFAIL counts
906928
if [ "$LATEST_COMMIT_MUSTPASS" != "$LATEST_RELEASE_MUSTPASS" ] || [ "$LATEST_COMMIT_MUSTFAIL" != "$LATEST_RELEASE_MUSTFAIL" ]; then
@@ -919,10 +941,10 @@ jobs:
919941
920942
echo "Latest commit with SHA ${{ github.sha }} ran in ${LATEST_COMMIT_TOTAL_SECONDS} seconds" > results.txt
921943
echo "Latest release with tag ${{ env.LATEST_RELEASE_NAME }} ran in ${LATEST_RELEASE_TOTAL_SECONDS} seconds" >> results.txt
922-
echo "Latest commit MUSTPASS: ${LATEST_COMMIT_MUSTPASS}" >> results.txt
923-
echo "Latest release MUSTPASS: ${LATEST_RELEASE_MUSTPASS}" >> results.txt
924-
echo "Latest commit MUSTFAIL: ${LATEST_COMMIT_MUSTFAIL}" >> results.txt
925-
echo "Latest release MUSTFAIL: ${LATEST_RELEASE_MUSTFAIL}" >> results.txt
944+
echo "Latest commit MUSTPASS: ${LATEST_COMMIT_MUSTPASS} / ${LATEST_TOTAL_COMMIT_MUSTPASS}" >> results.txt
945+
echo "Latest release MUSTPASS: ${LATEST_RELEASE_MUSTPASS} / ${LATEST_TOTAL_RELEASE_MUSTPASS}" >> results.txt
946+
echo "Latest commit MUSTFAIL: ${LATEST_COMMIT_MUSTFAIL} / ${LATEST_TOTAL_COMMIT_MUSTFAIL}" >> results.txt
947+
echo "Latest release MUSTFAIL: ${LATEST_RELEASE_MUSTFAIL} / ${LATEST_TOTAL_RELEASE_MUSTFAIL}" >> results.txt
926948
927949
- name: Upload results artifact
928950
uses: actions/upload-artifact@v4

0 commit comments

Comments
 (0)