Skip to content

Commit 3d53d57

Browse files
mykola-elasticmykola-elastic
committed
Merge branch 'main' into add-receivers-v130
2 parents 184520e + 2163167 commit 3d53d57

File tree

402 files changed

+27303
-15605
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

402 files changed

+27303
-15605
lines changed

‎.buildkite/bk.integration-fips.pipeline.yml‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,26 +4,38 @@ env:
44
ASDF_MAGE_VERSION: 1.14.0
55
MS_GOTOOLCHAIN_TELEMETRY_ENABLED: "0"
66

7-
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1753491662"
8-
IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1753491662"
9-
IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1753491662"
7+
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1757725254"
8+
IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1757725254"
9+
IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1757725254"
10+
ASDF_TERRAFORM_VERSION: 1.9.2
1011

1112
# This section is used to define the plugins that will be used in the pipeline.
1213
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
1314
common:
15+
# We are temporarily using the Production CFT environment API key instead of the
16+
# Staging GovCloud one. This is being done until issues with creating deployments in
17+
# Staging GovCloud are fixed. Once those are fixed, uncomment the `vault_ec_key_staging_frh_gov`
18+
# section and delete the `vault_ec_key_prod` section below.
19+
# - vault_ec_key_staging_frh_gov: &vault_ec_key_staging_frh_gov
20+
# elastic/vault-secrets#v0.1.0:
21+
# path: "kv/ci-shared/platform-ingest/platform-ingest-ec-staging-gov"
22+
# field: "apiKey"
23+
# env_var: "EC_API_KEY"
1424
- vault_ec_key_prod: &vault_ec_key_prod
1525
elastic/vault-secrets#v0.1.0:
1626
path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
1727
field: "apiKey"
1828
env_var: "EC_API_KEY"
19-
2029
steps:
2130
- label: Start ESS stack for FIPS integration tests
2231
key: integration-fips-ess
2332
env:
24-
ASDF_TERRAFORM_VERSION: 1.9.2
2533
FIPS: "true"
34+
# EC_ENDPOINT: "https://api.staging.elastic-gov.com"
35+
# ESS_REGION: "us-gov-east-1"
36+
# TF_VAR_deployment_template_id: "aws-general-purpose"
2637
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
38+
TF_VAR_docker_images_name_suffix: "-fips"
2739
command: |
2840
source .buildkite/scripts/steps/ess_start.sh
2941
artifact_paths:
@@ -123,6 +135,13 @@ steps:
123135
- label: ESS FIPS stack cleanup
124136
depends_on:
125137
- integration-tests-ubuntu-fips
138+
env:
139+
FIPS: "true"
140+
# EC_ENDPOINT: "https://api.staging.elastic-gov.com"
141+
# ESS_REGION: "us-gov-east-1"
142+
# TF_VAR_deployment_template_id: "aws-general-purpose"
143+
TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
144+
TF_VAR_docker_images_name_suffix: "-fips"
126145
allow_dependency_failure: true
127146
command: |
128147
buildkite-agent artifact download "test_infra/ess/**" . --step "integration-fips-ess"

‎.buildkite/bk.integration.pipeline.yml‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@ env:
66

77
# The following images are defined here and their values will be updated by updatecli
88
# Please do not change them manually.
9-
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1753491662"
10-
IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1753491662"
11-
IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1753491662"
12-
IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1753491662"
13-
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1753491662"
14-
IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1753491662"
9+
IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1757725254"
10+
IMAGE_UBUNTU_2404_ARM_64: "platform-ingest-elastic-agent-ubuntu-2404-aarch64-1757725254"
11+
IMAGE_RHEL_8: "platform-ingest-elastic-agent-rhel-8-1757725254"
12+
IMAGE_DEBIAN_12: "platform-ingest-elastic-agent-debian-12-1757725254"
13+
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1757725254"
14+
IMAGE_WIN_2025: "platform-ingest-elastic-agent-windows-2025-1757725254"
15+
ASDF_TERRAFORM_VERSION: 1.9.2
1516

1617
# This section is used to define the plugins that will be used in the pipeline.
1718
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
@@ -74,8 +75,6 @@ steps:
7475
notify:
7576
- github_commit_status:
7677
context: "buildkite/elastic-agent-extended-testing - ESS stack provision"
77-
env:
78-
ASDF_TERRAFORM_VERSION: 1.9.2
7978
command: .buildkite/scripts/steps/ess_start.sh
8079
retry:
8180
automatic:

‎.buildkite/integration.pipeline.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ env:
66

77
# The following images are defined here and their values will be updated by updatecli
88
# Please do not change them manually.
9-
IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1753491662"
10-
IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1753491662"
9+
IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1757725254"
10+
IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1757725254"
1111

1212
common:
1313
- vault_docker_login: &vault_docker_login

‎.buildkite/pipeline.agentless-app-release.yaml‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,46 @@
11
env:
22
VERSION: "${BUILDKITE_COMMIT:0:12}"
33
steps:
4-
- label: "Mirror Elastic-Agent Snapshot DRA to internal registry"
4+
- label: "Packaging: Service Container linux/amd64"
5+
key: packaging-service-container-amd64
6+
env:
7+
PACKAGES: "docker"
8+
PLATFORMS: "linux/amd64"
9+
DOCKER_VARIANTS: "service"
10+
command: |
11+
.buildkite/scripts/steps/integration-package.sh
12+
artifact_paths:
13+
- "build/distributions/elastic-agent-service-git-*.docker.tar.gz"
14+
agents:
15+
provider: "gcp"
16+
machineType: "c2-standard-16"
17+
diskSizeGb: 400
18+
19+
- label: "Packaging: Service Container linux/arm64"
20+
key: packaging-service-container-arm64
21+
env:
22+
PACKAGES: "docker"
23+
PLATFORMS: "linux/arm64"
24+
DOCKER_VARIANTS: "service"
25+
command: |
26+
.buildkite/scripts/steps/integration-package.sh
27+
artifact_paths:
28+
- "build/distributions/elastic-agent-service-git-*.docker.tar.gz"
29+
agents:
30+
provider: "aws"
31+
instanceType: "t4g.2xlarge"
32+
imagePrefix: "core-ubuntu-2204-aarch64"
33+
diskSizeGb: 400
34+
35+
# wait for packaging to be done
36+
- wait: ~
37+
38+
- label: "Publish to internal registry"
539
key: "mirror-elastic-agent"
640
command: ".buildkite/scripts/steps/ecp-internal-release.sh"
741
agents:
842
image: docker.elastic.co/ci-agent-images/serverless-helm-builder:0.0.2@sha256:d00e8a7a0ab3618cfaacb0a7b1e1b06ee29728eb2b44de602374bd8f6b9b92ac
943

10-
11-
1244
# wait for metadata to be set
1345
- wait: ~
1446

‎.buildkite/pipeline.yml‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ env:
55

66
# The following images are defined here and their values will be updated by updatecli
77
# Please do not change them manually.
8-
IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1753491662"
9-
IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1753491662"
10-
IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1753491662"
11-
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1753491662"
12-
IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1753491662"
13-
IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1753491662"
8+
IMAGE_UBUNTU_2204_X86_64: "platform-ingest-elastic-agent-ubuntu-2204-1757725254"
9+
IMAGE_UBUNTU_2204_ARM_64: "platform-ingest-elastic-agent-ubuntu-2204-aarch64-1757725254"
10+
IMAGE_WIN_2016: "platform-ingest-elastic-agent-windows-2016-1757725254"
11+
IMAGE_WIN_2022: "platform-ingest-elastic-agent-windows-2022-1757725254"
12+
IMAGE_WIN_10: "platform-ingest-elastic-agent-windows-10-1757725254"
13+
IMAGE_WIN_11: "platform-ingest-elastic-agent-windows-11-1757725254"
1414

1515
steps:
1616
- label: "check-ci"
@@ -255,7 +255,7 @@ steps:
255255
artifact_paths:
256256
- "build/TEST-go-unit.cov"
257257
agents:
258-
image: "golang:1.24.5"
258+
image: "golang:1.24.7"
259259
depends_on:
260260
- unit-tests
261261
- extended-windows
@@ -352,6 +352,7 @@ steps:
352352

353353
- .buildkite/integration.pipeline.yml
354354
- .buildkite/bk.integration.pipeline.yml
355+
- .buildkite/bk.integration-fips.pipeline.yml
355356
- .buildkite/pipeline.yml
356357
- .buildkite/scripts/
357358
- .buildkite/hooks/
@@ -371,3 +372,10 @@ steps:
371372
build:
372373
commit: "${BUILDKITE_COMMIT}"
373374
branch: "${BUILDKITE_BRANCH}"
375+
376+
# wait for packaging to be done
377+
- wait: ~
378+
379+
- label: "Publish to serverless"
380+
branches: main
381+
trigger: "agentless-serverless-release"

‎.buildkite/scripts/steps/ecp-internal-release.sh‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ set -eu
2020
_SELF=$(dirname $0)
2121
source "${_SELF}/../common.sh"
2222

23-
2423
# annotate create temp markdown file if not exists
2524
# this file will be later used to annotate the build
2625
# it appends to the file the message passed as argument
@@ -33,19 +32,9 @@ write_annotation() {
3332
cat $BUILDKITE_ANNOTATE_FILE | buildkite-agent annotate --style info
3433
}
3534

35+
BUILD_VERSION="$(jq -r '.version' .package-version)"
36+
DOCKER_TAG="git-${VERSION}"
3637
PRIVATE_REPO="docker.elastic.co/observability-ci/ecp-elastic-agent-service"
37-
SNAPSHOT_DRA_URL=https://snapshots.elastic.co/latest/master.json
38-
39-
DRA_RESULT=$(curl -s -X GET "$SNAPSHOT_DRA_URL")
40-
echo "$DRA_RESULT"
41-
BUILD_ID=$(echo "$DRA_RESULT" | jq '.build_id' | tr -d '"')
42-
BUILD_VERSION=$(echo "$DRA_RESULT" | jq '.version' | tr -d '"')
43-
44-
MANIFEST_URL="https://snapshots.elastic.co/$BUILD_ID/agent-package/agent-artifacts-$BUILD_VERSION.json"
45-
GIT_COMMIT=$(curl -s -X GET "$MANIFEST_URL" | jq '.projects["elastic-agent-core"]["commit_hash"]' | tr -d '"')
46-
GIT_SHORT_COMMIT=$(echo "$GIT_COMMIT" | cut -c1-12)
47-
48-
DOCKER_TAG="git-${GIT_SHORT_COMMIT}"
4938
PRIVATE_IMAGE="${PRIVATE_REPO}:${DOCKER_TAG}"
5039

5140
# TODO: let's avoid accessing vault directly but use the vault plugin itself
@@ -57,13 +46,19 @@ DOCKER_REGISTRY="docker.elastic.co"
5746
DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
5847
DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
5948
skopeo login --username "${DOCKER_USERNAME_SECRET}" --password "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}"
60-
skopeo copy --all "docker://docker.elastic.co/cloud-release/elastic-agent-service:$BUILD_ID-SNAPSHOT" "docker://$PRIVATE_IMAGE"
49+
50+
# download the amd64 and arm64 builds of the image from the previous steps
51+
buildkite-agent artifact download "build/distributions/**" . --step "packaging-service-container-amd64"
52+
buildkite-agent artifact download "build/distributions/**" . --step "packaging-service-container-arm64"
53+
54+
# copy the images into the private image location
55+
skopeo copy --all "docker-archive:./build/distributions/elastic-agent-service-$DOCKER_TAG-$BUILD_VERSION-linux-amd64.docker.tar.gz" "docker://$PRIVATE_IMAGE"
56+
skopeo copy --all "docker-archive:./build/distributions/elastic-agent-service-$DOCKER_TAG-$BUILD_VERSION-linux-arm64.docker.tar.gz" "docker://$PRIVATE_IMAGE"
6157

6258
annotate "* Image: $PRIVATE_IMAGE"
63-
annotate "* Short commit: $GIT_SHORT_COMMIT"
64-
annotate "* Commit: https://github.com/elastic/elastic-agent/commit/$GIT_COMMIT"
65-
annotate "* Manifest: $MANIFEST_URL"
59+
annotate "* Short commit: $VERSION"
60+
annotate "* Commit: https://github.com/elastic/elastic-agent/commit/$VERSION"
6661

67-
buildkite-agent meta-data set "git-short-commit" "$GIT_SHORT_COMMIT"
62+
buildkite-agent meta-data set "git-short-commit" "$VERSION"
6863

6964
write_annotation

‎.buildkite/scripts/steps/ess.ps1‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ function ess_up {
4040
}
4141

4242
function ess_down {
43+
param (
44+
[string]$EssRegion = "gcp-us-west2"
45+
)
4346
$Workspace = & git rev-parse --show-toplevel
4447
$TfDir = Join-Path -Path $Workspace -ChildPath "test_infra/ess/"
4548
$stateFilePath = Join-Path -Path $TfDir -ChildPath "terraform.tfstate"
@@ -52,7 +55,8 @@ function ess_down {
5255
try {
5356
Push-Location -Path $TfDir
5457
& terraform init
55-
& terraform destroy -auto-approve
58+
& terraform destroy -auto-approve `
59+
-var="ess_region=$EssRegion"
5660
Pop-Location
5761
} catch {
5862
Write-Output "Error: Failed to destroy ESS stack(it will be auto-deleted later): $_"

‎.buildkite/scripts/steps/ess.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,15 @@ function ess_up() {
4040
}
4141

4242
function ess_down() {
43-
echo "~~~ Tearing down the ESS Stack"
43+
echo "~~~ Tearing down the ESS Stack"
44+
local ESS_REGION=${1:-"gcp-us-west2"}
4445
local WORKSPACE=$(git rev-parse --show-toplevel)
4546
local TF_DIR="${WORKSPACE}/test_infra/ess/"
4647

4748
pushd "${TF_DIR}"
4849
terraform init
49-
terraform destroy -auto-approve
50+
terraform destroy -auto-approve \
51+
-var="ess_region=${ESS_REGION}"
5052
popd
5153
}
5254

‎.buildkite/scripts/steps/ess_down.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,6 @@ set -euo pipefail
33

44
source .buildkite/scripts/steps/ess.sh
55

6-
ess_down || echo "Failed to stop ESS stack" >&2
6+
ESS_REGION="${ESS_REGION:-gcp-us-west2}"
7+
8+
ess_down "$ESS_REGION"

‎.buildkite/scripts/steps/ess_start.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,14 @@ source .buildkite/scripts/steps/fleet.sh
66

77
STACK_VERSION="$(jq -r '.version' .package-version)"
88
STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)"
9+
if [[ "${FIPS:-false}" == "true" ]]; then
10+
# FRH testing environment does not have same stack build IDs as CFT environment so
11+
# we just go with the STACK_VERSION.
12+
STACK_BUILD_ID=""
13+
fi
14+
ESS_REGION="${ESS_REGION:-gcp-us-west2}"
915

10-
ess_up "$STACK_VERSION" "$STACK_BUILD_ID"
16+
ess_up "$STACK_VERSION" "$STACK_BUILD_ID" "$ESS_REGION"
1117

1218
preinstall_fleet_packages
1319

0 commit comments

Comments
 (0)