Fix multiarch service image push from main to serverless (#10129)

* Create a manifest image and push to repository.

* Change to only use docker for agentless publish.

* Update secret_path.

* Debug lines.

* Try 3.

* Fix digest variable.

* Cleanups from code review.

Author: blakerouse

.buildkite/pipeline.agentless-app-release.yaml

@@ -1,13 +1,56 @@
 env:
   VERSION: "${BUILDKITE_COMMIT:0:12}"
+
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - docker_login_plugin: &docker_login_plugin
+      elastic/vault-docker-login#v0.5.2:
+        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
+
 steps:
-  - label: "Mirror Elastic-Agent Snapshot DRA to internal registry"
-    key: "mirror-elastic-agent"
-    command: ".buildkite/scripts/steps/ecp-internal-release.sh"
+  - label: "Packaging: Service Container linux/amd64"
+    key: packaging-service-container-amd64
+    env:
+      PACKAGES: "docker"
+      PLATFORMS: "linux/amd64"
+      DOCKER_VARIANTS: "service"
+    command: |
+      .buildkite/scripts/steps/integration-package.sh
+    artifact_paths:
+      - "build/distributions/elastic-agent-service-git-*.docker.tar.gz"
     agents:
-      image: docker.elastic.co/ci-agent-images/serverless-helm-builder:0.0.2@sha256:d00e8a7a0ab3618cfaacb0a7b1e1b06ee29728eb2b44de602374bd8f6b9b92ac
+      provider: "gcp"
+      machineType: "c2-standard-16"
+      diskSizeGb: 400
 
+  - label: "Packaging: Service Container linux/arm64"
+    key: packaging-service-container-arm64
+    env:
+      PACKAGES: "docker"
+      PLATFORMS: "linux/arm64"
+      DOCKER_VARIANTS: "service"
+    command: |
+       .buildkite/scripts/steps/integration-package.sh
+    artifact_paths:
+      - "build/distributions/elastic-agent-service-git-*.docker.tar.gz"
+    agents:
+      provider: "aws"
+      instanceType: "t4g.2xlarge"
+      imagePrefix: "core-ubuntu-2204-aarch64"
+      diskSizeGb: 400
+
+  # wait for packaging to be done
+  - wait: ~
 
+  - label: "Publish to internal registry"
+    key: "mirror-elastic-agent"
+    command: ".buildkite/scripts/steps/ecp-internal-release.sh"
+    agents:
+      provider: "gcp"
+      machineType: "c2-standard-16"
+    plugins:
+      - *docker_login_plugin
 
   # wait for metadata to be set
   - wait: ~

.buildkite/scripts/steps/ecp-internal-release.sh

@@ -20,7 +20,6 @@ set -eu
 _SELF=$(dirname $0)
 source "${_SELF}/../common.sh"
 
-
 # annotate create temp markdown file if not exists
 # this file will be later used to annotate the build
 # it appends to the file the message passed as argument
@@ -33,37 +32,39 @@ write_annotation() {
     cat $BUILDKITE_ANNOTATE_FILE | buildkite-agent annotate --style info
 }
 
+BUILD_VERSION="$(jq -r '.version' .package-version)"
+DOCKER_TAG="git-${VERSION}"
 PRIVATE_REPO="docker.elastic.co/observability-ci/ecp-elastic-agent-service"
-SNAPSHOT_DRA_URL=https://snapshots.elastic.co/latest/master.json
+PRIVATE_IMAGE="${PRIVATE_REPO}:${DOCKER_TAG}"
 
-DRA_RESULT=$(curl -s -X GET "$SNAPSHOT_DRA_URL")
-echo "$DRA_RESULT"
-BUILD_ID=$(echo "$DRA_RESULT" | jq '.build_id' | tr -d '"')
-BUILD_VERSION=$(echo "$DRA_RESULT" | jq '.version' | tr -d '"')
+# download the amd64 and arm64 builds of the image from the previous steps
+buildkite-agent artifact download "build/distributions/**" . --step "packaging-service-container-amd64"
+buildkite-agent artifact download "build/distributions/**" . --step "packaging-service-container-arm64"
 
-MANIFEST_URL="https://snapshots.elastic.co/$BUILD_ID/agent-package/agent-artifacts-$BUILD_VERSION.json"
-GIT_COMMIT=$(curl -s -X GET "$MANIFEST_URL" | jq '.projects["elastic-agent-core"]["commit_hash"]' | tr -d '"')
-GIT_SHORT_COMMIT=$(echo "$GIT_COMMIT" | cut -c1-12)
+# AMD64
+docker load -i ./build/distributions/elastic-agent-service-$DOCKER_TAG-$BUILD_VERSION-linux-amd64.docker.tar.gz
+docker image tag "elastic-agent-service:$DOCKER_TAG" "$PRIVATE_IMAGE"
+docker push "$PRIVATE_IMAGE"
+AMD64_DIGEST=$(docker image inspect --format "{{index .RepoDigests 0}}" "$PRIVATE_IMAGE")
 
-DOCKER_TAG="git-${GIT_SHORT_COMMIT}"
-PRIVATE_IMAGE="${PRIVATE_REPO}:${DOCKER_TAG}"
+# ARM64 (overwrites AMD64 tags)
+docker load -i ./build/distributions/elastic-agent-service-$DOCKER_TAG-$BUILD_VERSION-linux-arm64.docker.tar.gz
+docker image tag "elastic-agent-service:$DOCKER_TAG" "$PRIVATE_IMAGE"
+docker push "$PRIVATE_IMAGE"
+ARM64_DIGEST=$(docker image inspect --format "{{index .RepoDigests 0}}" "$PRIVATE_IMAGE")
 
-# TODO: let's avoid accessing vault directly but use the vault plugin itself
-#       https://github.com/elastic/vault-docker-login-buildkite-plugin does not support
-#       the `skopeo` command by default but looks for the current installed tools in the runner
-#       Let's contribute in a follow-up PR to support `skopeo` as well.
-DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
-DOCKER_REGISTRY="docker.elastic.co"
-DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-skopeo login --username "${DOCKER_USERNAME_SECRET}" --password "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}"
-skopeo copy --all "docker://docker.elastic.co/cloud-release/elastic-agent-service:$BUILD_ID-SNAPSHOT" "docker://$PRIVATE_IMAGE"
+# at this point the $PRIVATE_IMAGE is pointing to only the arm64 based image, we need the image to
+# be a multi-architecture based image so we create an image from the digests and tag it the same and
+# push it to the registry (aka. make the tag now a multi-architecture based image)
+docker buildx imagetools create -t "$PRIVATE_IMAGE" \
+  "$AMD64_DIGEST" \
+  "$ARM64_DIGEST"
+docker push "$PRIVATE_IMAGE"
 
 annotate "* Image: $PRIVATE_IMAGE"
-annotate "* Short commit: $GIT_SHORT_COMMIT"
-annotate "* Commit: https://github.com/elastic/elastic-agent/commit/$GIT_COMMIT"
-annotate "* Manifest: $MANIFEST_URL"
+annotate "* Short commit: $VERSION"
+annotate "* Commit: https://github.com/elastic/elastic-agent/commit/$VERSION"
 
-buildkite-agent meta-data set "git-short-commit" "$GIT_SHORT_COMMIT"
+buildkite-agent meta-data set "git-short-commit" "$VERSION"
 
 write_annotation

.buildkite/scripts/steps/integration-package.sh

@@ -3,6 +3,12 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+# This script is not only used by integration CI, but also the agentless release pipeline to build the docker images
+# that get released to serverless. USE_PACKAGE_VERSION is important here for both CI and agentless, as it validates
+# that the components bundled to serverless are the same components that the CI validated.
+#
+# Making a change here can affect the released images to agentless, so be cautious.
+
 export SNAPSHOT="true"
 export EXTERNAL="true"
 export USE_PACKAGE_VERSION="true"

.github/CODEOWNERS

@@ -7,6 +7,7 @@
 # Sub-directories/files ownership. Remember that order matters; the last matching pattern takes the most precedence.
 /.buildkite @elastic/elastic-agent-control-plane @elastic/observablt-ci
 /.ci @elastic/elastic-agent-control-plane @elastic/observablt-ci
+/.ci/scripts/ecp-internal-release.sh @elastic/elastic-agent-control-plane
 /.ci/scripts/update-otel.sh @elastic/elastic-agent-control-plane
 /.github @elastic/elastic-agent-control-plane @elastic/observablt-ci
 /.github/CODEOWNERS @elastic/ingest-tech-lead