From 0dd9d8f09c630751553b6caccdf05535c8d58091 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Tue, 11 Oct 2022 12:08:42 +0200 Subject: [PATCH 1/6] Update release-go-task using parallelization Introducing the use of a matrix, greatly improves time performances during the build making process, since each build task is performed simultaneously. To support this, a check has been added to avoid creating the same changelog more than once. The calculation of the checksums has also been modified. Previously, it was done three (3) times at different stages of the workflow. This is pointless, since the only checksums that matter are the ones calculated when the files are in their final form. For this reason, it is now only done once during the release creation. --- workflow-templates/release-go-task.yml | 39 +++++++++++++++----------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/workflow-templates/release-go-task.yml b/workflow-templates/release-go-task.yml index b24bfc3f..663b4f50 100644 --- a/workflow-templates/release-go-task.yml +++ b/workflow-templates/release-go-task.yml @@ -19,6 +19,19 @@ jobs: create-release-artifacts: runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -26,6 +39,8 @@ jobs: fetch-depth: 0 - name: Create changelog + # Avoid creating the same changelog for each os + if: matrix.os == 'Windows_32bit' uses: arduino/create-changelog@v1 with: tag-regex: '^[0-9]+\.[0-9]+\.[0-9]+.*$' @@ -132,14 +147,10 @@ jobs: run: | gon "${{ env.GON_CONFIG_PATH }}" - - name: Re-package binary and output checksum + - name: Re-package binary id: re-package working-directory: ${{ env.DIST_DIR }} - # This step performs the following: - # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file) - # 2. Recalculate package checksum - # 3. Output the new checksum to include in the nnnnnn-checksums.txt file - # (it cannot be done there because of workflow job parallelization) + # Repackage the signed binary replaced in place by Gon (ignoring the output zip file) run: | # GitHub's upload/download-artifact actions don't preserve file permissions, # so we need to add execution permission back until the action is made to do this. @@ -149,11 +160,9 @@ jobs: tar -czvf "$PACKAGE_FILENAME" \ -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \ -C ../../ LICENSE.txt - CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)" echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV - echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE" - - name: Upload artifacts + - name: Upload artifact uses: actions/upload-artifact@v3 with: if-no-files-found: error @@ -171,15 +180,11 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.DIST_DIR }} - - name: Update checksum + - name: Create checksum file + working-directory: ${{ env.DIST_DIR}} run: | - declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}") - for checksum_line in "${checksum_lines[@]}" - do - CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1) - PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2) - perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt - done + TAG="${GITHUB_REF/refs\/tags\//}" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt - name: Identify Prerelease # This is a workaround while waiting for create-release action From be6f653d4b817b4cc3aa3a6bc36580d8f888eea7 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Tue, 11 Oct 2022 12:28:49 +0200 Subject: [PATCH 2/6] Update publish-go-nightly-task using parallelization Introducing the use of a matrix, greatly improves time performances during the build making process, since each build task is performed simultaneously. The calculation of the checksums has also been modified. Previously, it was done three (3) times at different stages of the workflow. This is pointless, since the only checksums that matter are the ones calculated when the files are in their final form. For this reason, it is now only done once during the release creation. --- .../publish-go-nightly-task.yml | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/workflow-templates/publish-go-nightly-task.yml b/workflow-templates/publish-go-nightly-task.yml index cec8754b..da07246d 100644 --- a/workflow-templates/publish-go-nightly-task.yml +++ b/workflow-templates/publish-go-nightly-task.yml @@ -22,6 +22,19 @@ jobs: create-nightly-artifacts: runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -35,7 +48,7 @@ jobs: - name: Build env: NIGHTLY: true - run: task dist:all + run: task dist:${{ matrix.os }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -128,14 +141,10 @@ jobs: run: | gon "${{ env.GON_CONFIG_PATH }}" - - name: Re-package binary and output checksum + - name: Re-package binary id: re-package working-directory: ${{ env.DIST_DIR }} - # This step performs the following: - # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file) - # 2. Recalculate package checksum - # 3. Output the new checksum to include in the nnnnnn-checksums.txt file - # (it cannot be done there because of workflow job parallelization) + # Repackage the signed binary replaced in place by Gon (ignoring the output zip file) run: | # GitHub's upload/download-artifact actions don't preserve file permissions, # so we need to add execution permission back until the action is made to do this. @@ -145,11 +154,9 @@ jobs: tar -czvf "$PACKAGE_FILENAME" \ -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \ -C ../../ LICENSE.txt - CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)" echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV - echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE" - - name: Upload artifacts + - name: Upload artifact uses: actions/upload-artifact@v3 with: if-no-files-found: error @@ -167,15 +174,11 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.DIST_DIR }} - - name: Update checksum + - name: Create checksum file + working-directory: ${{ env.DIST_DIR}} run: | - declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}") - for checksum_line in "${checksum_lines[@]}" - do - CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1) - PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2) - perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt - done + TAG="nightly-$(date -u +"%Y%m%d")" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt - name: Upload release files on Arduino downloads servers uses: docker://plugins/s3 From 3be48280419ff35fc4f5f4877e7a73748dba6957 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Tue, 11 Oct 2022 13:12:44 +0200 Subject: [PATCH 3/6] Update publish-go-tester-task using parallelization Introducing the use of a matrix, greatly improves time performances during the build making process, since each build task is performed simultaneously. Furthermore, it allows to directly upload a different artifact for each build. To complete this set of changes, a separate job to calculate the checksums and upload the relative artifact has been added to the workflow. --- workflow-templates/publish-go-tester-task.yml | 124 +++++++++++------- 1 file changed, 73 insertions(+), 51 deletions(-) diff --git a/workflow-templates/publish-go-tester-task.yml b/workflow-templates/publish-go-tester-task.yml index 80e72b32..3b156c32 100644 --- a/workflow-templates/publish-go-tester-task.yml +++ b/workflow-templates/publish-go-tester-task.yml @@ -24,9 +24,10 @@ on: repository_dispatch: env: + # As defined by the Taskfile's PROJECT_NAME variable + PROJECT_NAME: TODO_PROJECT_NAME # As defined by the Taskfile's DIST_DIR variable DIST_DIR: dist - BUILDS_ARTIFACT: build-artifacts jobs: run-determination: @@ -52,86 +53,107 @@ jobs: echo "::set-output name=result::$RESULT" - build: + package-name-prefix: needs: run-determination if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest - + outputs: + prefix: ${{ steps.calculation.outputs.prefix }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Install Task - uses: arduino/setup-task@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - version: 3.x - - - name: Build + - name: package name prefix calculation + id: calculation run: | PACKAGE_NAME_PREFIX="test" if [ "${{ github.event_name }}" = "pull_request" ]; then PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}" fi PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-" - export PACKAGE_NAME_PREFIX - task dist:all - # Transfer builds to artifacts job - - name: Upload combined builds artifact - uses: actions/upload-artifact@v3 - with: - path: ${{ env.DIST_DIR }} - name: ${{ env.BUILDS_ARTIFACT }} + echo "::set-output name=prefix::$PACKAGE_NAME_PREFIX" - artifacts: - name: ${{ matrix.artifact.name }} artifact - needs: build + build: + needs: package-name-prefix + name: Build ${{ matrix.os.name }} runs-on: ubuntu-latest strategy: matrix: - artifact: - - path: "*checksums.txt" - name: checksums - - path: "*Linux_32bit.tar.gz" + os: + - task: Windows_32bit + path: "*Windows_32bit.zip" + name: Windows_X86-32 + - task: Windows_64bit + path: "*Windows_64bit.zip" + name: Windows_X86-64 + - task: Linux_32bit + path: "*Linux_32bit.tar.gz" name: Linux_X86-32 - - path: "*Linux_64bit.tar.gz" + - task: Linux_64bit + path: "*Linux_64bit.tar.gz" name: Linux_X86-64 - - path: "*Linux_ARM64.tar.gz" - name: Linux_ARM64 - - path: "*Linux_ARMv6.tar.gz" + - task: Linux_ARMv6 + path: "*Linux_ARMv6.tar.gz" name: Linux_ARMv6 - - path: "*Linux_ARMv7.tar.gz" + - task: Linux_ARMv7 + path: "*Linux_ARMv7.tar.gz" name: Linux_ARMv7 - - path: "*macOS_64bit.tar.gz" + - task: Linux_ARM64 + path: "*Linux_ARM64.tar.gz" + name: Linux_ARM64 + - task: macOS_64bit + path: "*macOS_64bit.tar.gz" name: macOS_64 - - path: "*macOS_ARM64.tar.gz" + - task: macOS_ARM64 + path: "*macOS_ARM64.tar.gz" name: macOS_ARM64 - - path: "*Windows_32bit.zip" - name: Windows_X86-32 - - path: "*Windows_64bit.zip" - name: Windows_X86-64 steps: - - name: Download combined builds artifact - uses: actions/download-artifact@v3 + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install Task + uses: arduino/setup-task@v1 with: - name: ${{ env.BUILDS_ARTIFACT }} - path: ${{ env.BUILDS_ARTIFACT }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x - - name: Upload individual build artifact + - name: Build + run: | + PACKAGE_NAME_PREFIX=${{ needs.package-name-prefix.outputs.prefix }} + export PACKAGE_NAME_PREFIX + task dist:${{ matrix.os.task }} + + # Transfer builds to artifacts job + - name: Upload build artifact uses: actions/upload-artifact@v3 with: - path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }} - name: ${{ matrix.artifact.name }} + path: ${{ env.DIST_DIR }}/${{ matrix.os.path }} + name: ${{ matrix.os.name }} - clean: - needs: artifacts + checksums: + needs: + - build + - package-name-prefix runs-on: ubuntu-latest steps: - - name: Remove unneeded combined builds artifact - uses: geekyeggo/delete-artifact@v2 + - name: Download build artifacts + uses: actions/download-artifact@v3 + + - name: Create checksum file + run: | + TAG="${{ needs.package-name-prefix.outputs.prefix }}git-snapshot" + declare -a artifacts=($(ls -d */)) + for artifact in ${artifacts[@]} + do + cd $artifact + checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*) + cd .. + echo $checksum >> ${TAG}-checksums.txt + done + + - name: Upload checksum artifact + uses: actions/upload-artifact@v3 with: - name: ${{ env.BUILDS_ARTIFACT }} + path: ./*checksums.txt + name: checksums From 20f7efd80bc8e3e0f43c5a1a6c5ea790cf2027c6 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Tue, 11 Oct 2022 13:25:33 +0200 Subject: [PATCH 4/6] Update documentation regarding publish-go-tester-task --- workflow-templates/publish-go-tester-task.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/workflow-templates/publish-go-tester-task.md b/workflow-templates/publish-go-tester-task.md index 882bd329..ca004e1b 100644 --- a/workflow-templates/publish-go-tester-task.md +++ b/workflow-templates/publish-go-tester-task.md @@ -19,6 +19,12 @@ Install the [`publish-go-tester-task.yml`](publish-go-tester-task.yml) GitHub Ac - [`DistTasks.yml`](assets/release-go-task/DistTasks.yml) - general purpose tasks for making production builds of Go projects using cgo and [elastic docker containers](https://github.com/elastic/golang-crossbuild). - Install to: repository root +### Configuration + +#### Workflow + +`PROJECT_NAME` must be set to the project-specific name in `publish-go-tester-task.yml`. + ### Readme badge Markdown badge: From b64b747a2907bf3364985c7d55e12e1371d77b5c Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Tue, 11 Oct 2022 13:32:23 +0200 Subject: [PATCH 5/6] Remove dist:all task and checksum calculation from DistTasks --- .../assets/release-go-task/DistTasks.yml | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/workflow-templates/assets/release-go-task/DistTasks.yml b/workflow-templates/assets/release-go-task/DistTasks.yml index e81131ff..307cdca9 100644 --- a/workflow-templates/assets/release-go-task/DistTasks.yml +++ b/workflow-templates/assets/release-go-task/DistTasks.yml @@ -20,22 +20,8 @@ version: "3" vars: CONTAINER: "docker.elastic.co/beats-dev/golang-crossbuild" GO_VERSION: "1.17.8" - CHECKSUM_FILE: "{{.VERSION}}-checksums.txt" tasks: - all: - desc: Build for distribution for all platforms - cmds: - - task: Windows_32bit - - task: Windows_64bit - - task: Linux_32bit - - task: Linux_64bit - - task: Linux_ARMv6 - - task: Linux_ARMv7 - - task: Linux_ARM64 - - task: macOS_64bit - - task: macOS_ARM64 - Windows_32bit: desc: Builds Windows 32 bit binaries dir: "{{.DIST_DIR}}" @@ -48,7 +34,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_386" @@ -70,7 +55,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" zip {{.PACKAGE_NAME}} {{.PLATFORM_DIR}}/{{.PROJECT_NAME}}.exe ../LICENSE.txt -j - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_windows_amd64" @@ -92,7 +76,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd32" @@ -114,7 +97,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_amd64" @@ -136,7 +118,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_7" @@ -158,7 +139,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_6" @@ -208,7 +188,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_linux_arm_64" @@ -230,7 +209,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_amd64" @@ -265,7 +243,6 @@ tasks: -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} - sha256sum {{.PACKAGE_NAME}} >> {{.CHECKSUM_FILE}} vars: PLATFORM_DIR: "{{.PROJECT_NAME}}_osx_darwin_arm64" From 5c9debd1e2a586a128e8b5a8afccc96ac40d84f4 Mon Sep 17 00:00:00 2001 From: Matteo Pologruto Date: Tue, 11 Oct 2022 14:55:50 +0200 Subject: [PATCH 6/6] Sync workflow duplicates --- .../workflows/publish-go-nightly-task.yml | 39 +++--- .../workflows/publish-go-tester-task.yml | 124 +++++++++++------- .../.github/workflows/release-go-task.yml | 39 +++--- 3 files changed, 116 insertions(+), 86 deletions(-) diff --git a/workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-nightly-task.yml b/workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-nightly-task.yml index cec8754b..da07246d 100644 --- a/workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-nightly-task.yml +++ b/workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-nightly-task.yml @@ -22,6 +22,19 @@ jobs: create-nightly-artifacts: runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -35,7 +48,7 @@ jobs: - name: Build env: NIGHTLY: true - run: task dist:all + run: task dist:${{ matrix.os }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -128,14 +141,10 @@ jobs: run: | gon "${{ env.GON_CONFIG_PATH }}" - - name: Re-package binary and output checksum + - name: Re-package binary id: re-package working-directory: ${{ env.DIST_DIR }} - # This step performs the following: - # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file) - # 2. Recalculate package checksum - # 3. Output the new checksum to include in the nnnnnn-checksums.txt file - # (it cannot be done there because of workflow job parallelization) + # Repackage the signed binary replaced in place by Gon (ignoring the output zip file) run: | # GitHub's upload/download-artifact actions don't preserve file permissions, # so we need to add execution permission back until the action is made to do this. @@ -145,11 +154,9 @@ jobs: tar -czvf "$PACKAGE_FILENAME" \ -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \ -C ../../ LICENSE.txt - CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)" echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV - echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE" - - name: Upload artifacts + - name: Upload artifact uses: actions/upload-artifact@v3 with: if-no-files-found: error @@ -167,15 +174,11 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.DIST_DIR }} - - name: Update checksum + - name: Create checksum file + working-directory: ${{ env.DIST_DIR}} run: | - declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}") - for checksum_line in "${checksum_lines[@]}" - do - CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1) - PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2) - perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt - done + TAG="nightly-$(date -u +"%Y%m%d")" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt - name: Upload release files on Arduino downloads servers uses: docker://plugins/s3 diff --git a/workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-tester-task.yml b/workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-tester-task.yml index 80e72b32..3b156c32 100644 --- a/workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-tester-task.yml +++ b/workflow-templates/dependabot/workflow-template-copies/.github/workflows/publish-go-tester-task.yml @@ -24,9 +24,10 @@ on: repository_dispatch: env: + # As defined by the Taskfile's PROJECT_NAME variable + PROJECT_NAME: TODO_PROJECT_NAME # As defined by the Taskfile's DIST_DIR variable DIST_DIR: dist - BUILDS_ARTIFACT: build-artifacts jobs: run-determination: @@ -52,86 +53,107 @@ jobs: echo "::set-output name=result::$RESULT" - build: + package-name-prefix: needs: run-determination if: needs.run-determination.outputs.result == 'true' runs-on: ubuntu-latest - + outputs: + prefix: ${{ steps.calculation.outputs.prefix }} steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: Install Task - uses: arduino/setup-task@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - version: 3.x - - - name: Build + - name: package name prefix calculation + id: calculation run: | PACKAGE_NAME_PREFIX="test" if [ "${{ github.event_name }}" = "pull_request" ]; then PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.event.number }}" fi PACKAGE_NAME_PREFIX="$PACKAGE_NAME_PREFIX-${{ github.sha }}-" - export PACKAGE_NAME_PREFIX - task dist:all - # Transfer builds to artifacts job - - name: Upload combined builds artifact - uses: actions/upload-artifact@v3 - with: - path: ${{ env.DIST_DIR }} - name: ${{ env.BUILDS_ARTIFACT }} + echo "::set-output name=prefix::$PACKAGE_NAME_PREFIX" - artifacts: - name: ${{ matrix.artifact.name }} artifact - needs: build + build: + needs: package-name-prefix + name: Build ${{ matrix.os.name }} runs-on: ubuntu-latest strategy: matrix: - artifact: - - path: "*checksums.txt" - name: checksums - - path: "*Linux_32bit.tar.gz" + os: + - task: Windows_32bit + path: "*Windows_32bit.zip" + name: Windows_X86-32 + - task: Windows_64bit + path: "*Windows_64bit.zip" + name: Windows_X86-64 + - task: Linux_32bit + path: "*Linux_32bit.tar.gz" name: Linux_X86-32 - - path: "*Linux_64bit.tar.gz" + - task: Linux_64bit + path: "*Linux_64bit.tar.gz" name: Linux_X86-64 - - path: "*Linux_ARM64.tar.gz" - name: Linux_ARM64 - - path: "*Linux_ARMv6.tar.gz" + - task: Linux_ARMv6 + path: "*Linux_ARMv6.tar.gz" name: Linux_ARMv6 - - path: "*Linux_ARMv7.tar.gz" + - task: Linux_ARMv7 + path: "*Linux_ARMv7.tar.gz" name: Linux_ARMv7 - - path: "*macOS_64bit.tar.gz" + - task: Linux_ARM64 + path: "*Linux_ARM64.tar.gz" + name: Linux_ARM64 + - task: macOS_64bit + path: "*macOS_64bit.tar.gz" name: macOS_64 - - path: "*macOS_ARM64.tar.gz" + - task: macOS_ARM64 + path: "*macOS_ARM64.tar.gz" name: macOS_ARM64 - - path: "*Windows_32bit.zip" - name: Windows_X86-32 - - path: "*Windows_64bit.zip" - name: Windows_X86-64 steps: - - name: Download combined builds artifact - uses: actions/download-artifact@v3 + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install Task + uses: arduino/setup-task@v1 with: - name: ${{ env.BUILDS_ARTIFACT }} - path: ${{ env.BUILDS_ARTIFACT }} + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x - - name: Upload individual build artifact + - name: Build + run: | + PACKAGE_NAME_PREFIX=${{ needs.package-name-prefix.outputs.prefix }} + export PACKAGE_NAME_PREFIX + task dist:${{ matrix.os.task }} + + # Transfer builds to artifacts job + - name: Upload build artifact uses: actions/upload-artifact@v3 with: - path: ${{ env.BUILDS_ARTIFACT }}/${{ matrix.artifact.path }} - name: ${{ matrix.artifact.name }} + path: ${{ env.DIST_DIR }}/${{ matrix.os.path }} + name: ${{ matrix.os.name }} - clean: - needs: artifacts + checksums: + needs: + - build + - package-name-prefix runs-on: ubuntu-latest steps: - - name: Remove unneeded combined builds artifact - uses: geekyeggo/delete-artifact@v2 + - name: Download build artifacts + uses: actions/download-artifact@v3 + + - name: Create checksum file + run: | + TAG="${{ needs.package-name-prefix.outputs.prefix }}git-snapshot" + declare -a artifacts=($(ls -d */)) + for artifact in ${artifacts[@]} + do + cd $artifact + checksum=$(sha256sum ${{ env.PROJECT_NAME }}_${TAG}*) + cd .. + echo $checksum >> ${TAG}-checksums.txt + done + + - name: Upload checksum artifact + uses: actions/upload-artifact@v3 with: - name: ${{ env.BUILDS_ARTIFACT }} + path: ./*checksums.txt + name: checksums diff --git a/workflow-templates/dependabot/workflow-template-copies/.github/workflows/release-go-task.yml b/workflow-templates/dependabot/workflow-template-copies/.github/workflows/release-go-task.yml index b24bfc3f..663b4f50 100644 --- a/workflow-templates/dependabot/workflow-template-copies/.github/workflows/release-go-task.yml +++ b/workflow-templates/dependabot/workflow-template-copies/.github/workflows/release-go-task.yml @@ -19,6 +19,19 @@ jobs: create-release-artifacts: runs-on: ubuntu-latest + strategy: + matrix: + os: + - Windows_32bit + - Windows_64bit + - Linux_32bit + - Linux_64bit + - Linux_ARMv6 + - Linux_ARMv7 + - Linux_ARM64 + - macOS_64bit + - macOS_ARM64 + steps: - name: Checkout repository uses: actions/checkout@v3 @@ -26,6 +39,8 @@ jobs: fetch-depth: 0 - name: Create changelog + # Avoid creating the same changelog for each os + if: matrix.os == 'Windows_32bit' uses: arduino/create-changelog@v1 with: tag-regex: '^[0-9]+\.[0-9]+\.[0-9]+.*$' @@ -132,14 +147,10 @@ jobs: run: | gon "${{ env.GON_CONFIG_PATH }}" - - name: Re-package binary and output checksum + - name: Re-package binary id: re-package working-directory: ${{ env.DIST_DIR }} - # This step performs the following: - # 1. Repackage the signed binary replaced in place by Gon (ignoring the output zip file) - # 2. Recalculate package checksum - # 3. Output the new checksum to include in the nnnnnn-checksums.txt file - # (it cannot be done there because of workflow job parallelization) + # Repackage the signed binary replaced in place by Gon (ignoring the output zip file) run: | # GitHub's upload/download-artifact actions don't preserve file permissions, # so we need to add execution permission back until the action is made to do this. @@ -149,11 +160,9 @@ jobs: tar -czvf "$PACKAGE_FILENAME" \ -C "${{ env.PROJECT_NAME }}_osx_${{ matrix.artifact.name }}/" "${{ env.PROJECT_NAME }}" \ -C ../../ LICENSE.txt - CHECKSUM_LINE="$(shasum -a 256 $PACKAGE_FILENAME)" echo "PACKAGE_FILENAME=$PACKAGE_FILENAME" >> $GITHUB_ENV - echo "::set-output name=checksum-${{ matrix.artifact.name }}::$CHECKSUM_LINE" - - name: Upload artifacts + - name: Upload artifact uses: actions/upload-artifact@v3 with: if-no-files-found: error @@ -171,15 +180,11 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ env.DIST_DIR }} - - name: Update checksum + - name: Create checksum file + working-directory: ${{ env.DIST_DIR}} run: | - declare -a checksum_lines=("${{ needs.notarize-macos.outputs.checksum-darwin_amd64 }}" "${{ needs.notarize-macos.outputs.checksum-darwin_arm64 }}") - for checksum_line in "${checksum_lines[@]}" - do - CHECKSUM=$(echo ${checksum_line} | cut -d " " -f 1) - PACKAGE_FILENAME=$(echo ${checksum_line} | cut -d " " -f 2) - perl -pi -w -e "s/.*${PACKAGE_FILENAME}/${CHECKSUM} ${PACKAGE_FILENAME}/g;" ${{ env.DIST_DIR }}/*-checksums.txt - done + TAG="${GITHUB_REF/refs\/tags\//}" + sha256sum ${{ env.PROJECT_NAME }}_${TAG}* > ${TAG}-checksums.txt - name: Identify Prerelease # This is a workaround while waiting for create-release action