Merge remote-tracking branch 'origin/main' into stack-cloud-provider

Author: jsoriano

.buildkite/hooks/post-checkout

@@ -47,7 +47,7 @@ MERGE_BRANCH="pr_merge_${PR_ID}"
 checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}"
 
 echo "Commit information"
-git log --format=%B -n 1
+git --no-pager log --format=%B -n 1
 
 # Ensure buildkite groups are rendered
 echo ""

.buildkite/hooks/pre-command

@@ -33,7 +33,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifact
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =~ ^integration-parallel ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
 fi
 
@@ -46,7 +46,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =
     export GCP_PROJECT_ID=${ELASTIC_PACKAGE_GCP_PROJECT_SECRET}
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" == "integration-parallel-aws" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" == "integration-parallel-aws" || "$BUILDKITE_STEP_KEY" == "integration-parallel-aws_logs") ]]; then
     export ELASTIC_PACKAGE_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
     export ELASTIC_PACKAGE_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH})
 
@@ -70,3 +70,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-package-storage-publish" &&
     # publishing job
     export PACKAGE_UPLOADER_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field value ${PACKAGE_UPLOADER_GCS_CREDENTIALS_PATH})
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package-test-with-integrations" && "$BUILDKITE_STEP_KEY" == "pr-integrations" ]]; then
+    export GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+    export GITHUB_TOKEN=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+fi

.buildkite/pipeline.test-with-integrations-repo.yml

@@ -1,5 +1,8 @@
 env:
+  SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   GOLANG_VERSION: "1.20.3"
+  GH_CLI_VERSION: "2.29.0"
+  JQ_VERSION: "1.6"
 
 steps:
   - label: ":go: Run check-static"
@@ -9,4 +12,22 @@ steps:
       image: "golang:${GOLANG_VERSION}"
       cpu: "8"
       memory: "4G"
+  - label: ":go: :linux: Run unit tests"
+    key: unit-tests-linux
+    command: "make test-go-ci"
+    artifact_paths:
+      - "build/test-results/*.xml"
+      - "build/test-coverage/*.xml"
+    agents:
+      image: "golang:${GOLANG_VERSION}"
+      cpu: "8"
+      memory: "4G"
+  - label: ":hammer: Create PR in integrations"
+    key: pr-integrations
+    command: ".buildkite/scripts/test-with-integrations.sh"
+    agents:
+      provider: "gcp"
+    depends_on:
+      - check-static
+      - unit-tests-linux
 

.buildkite/pipeline.trigger.integration.tests.sh

@@ -42,14 +42,29 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*"
-    echo "          - build/elastic-stack-status/*/*"
+    echo "          - build/elastic-stack-dump/check-*/logs/*.log"
+    echo "          - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*"
     if [[ $test =~ with-kind$ ]]; then
         echo "          - build/kubectl-dump.txt"
     fi
 done
 
+pushd test/packages/false_positives > /dev/null
+for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
+    package_name=$(basename ${package})
+    echo "      - label: \":go: Running integration test (false positive): ${package_name}\""
+    echo "        key: \"integration-false_positives-${package_name}\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
+    echo "        env:"
+    echo "          UPLOAD_SAFE_LOGS: 1"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "        artifact_paths:"
+    echo "          - build/test-results/*.xml"
+done
+
+ popd > /dev/null
+
 pushd test/packages/parallel > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename ${package})
@@ -62,8 +77,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/*.log"
-    echo "          - build/elastic-stack-dump/stack/check-*/logs/fleet-server-internal/**/*"
 done
 
 popd > /dev/null

.buildkite/pipeline.yml

@@ -2,6 +2,7 @@ env:
   SETUP_GVM_VERSION: 'v0.5.0' # https://github.com/andrewkroh/gvm/issues/44#issuecomment-1013231151
   DOCKER_COMPOSE_VERSION: "v2.17.2"
   ELASTIC_PACKAGE_COMPOSE_DISABLE_ANSI: "true"
+  ELASTIC_PACKAGE_COMPOSE_DISABLE_PULL_PROGRESS_INFORMATION: "true"
   KIND_VERSION: 'v0.17.0'
   K8S_VERSION: 'v1.26.0'
 

.buildkite/scripts/install_deps.sh

@@ -30,7 +30,7 @@ with_go() {
     eval "$(gvm $(cat .go-version))"
     go version
     which go
-    export PATH="$(go env GOPATH)/bin:${PATH}"
+    export PATH="${PATH}:$(go env GOPATH)/bin"
 }
 
 with_docker_compose() {
@@ -39,3 +39,30 @@ with_docker_compose() {
     chmod +x ${WORKSPACE}/bin/docker-compose
     docker-compose version
 }
+
+with_github_cli() {
+    mkdir -p ${WORKSPACE}/bin
+    mkdir -p ${WORKSPACE}/tmp
+
+    local gh_filename="gh_${GH_CLI_VERSION}_linux_amd64"
+    local gh_tar_file="${gh_filename}.tar.gz"
+    local gh_tar_full_path="${WORKSPACE}/tmp/${gh_tar_file}"
+
+    retry 5 curl -sL -o ${gh_tar_full_path} "https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/${gh_tar_file}"
+
+    # just extract the binary file from the tar.gz
+    tar -C ${WORKSPACE}/bin -xpf ${gh_tar_full_path} ${gh_filename}/bin/gh --strip-components=2
+
+    chmod +x ${WORKSPACE}/bin/gh
+    rm -rf ${WORKSPACE}/tmp
+
+    gh version
+}
+
+with_jq() {
+    mkdir -p ${WORKSPACE}/bin
+    retry 5 curl -sL -o ${WORKSPACE}/bin/jq "https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64"
+
+    chmod +x ${WORKSPACE}/bin/jq
+    jq --version
+}

.buildkite/scripts/integration_tests.sh

@@ -24,15 +24,15 @@ source .buildkite/scripts/install_deps.sh
 source .buildkite/scripts/tooling.sh
 
 PARALLEL_TARGET="test-check-packages-parallel"
+FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
 KIND_TARGET="test-check-packages-with-kind"
 TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 
 JOB_GCS_BUCKET_INTERNAL="fleet-ci-temp-internal"
 
-REPO_NAME=$(repoName "${BUILDKITE_REPO}")
-REPO_BUILD_TAG="${REPO_NAME}/${BUILDKITE_BUILD_NUMBER}"
-
+REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
+REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 TARGET=""
 PACKAGE=""
 while getopts ":t:p:h" o; do
@@ -109,7 +109,7 @@ if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then
 fi
 
 echo "--- Run integration test ${TARGET}"
-if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
+if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
     make install
 
     # allow to fail this command, to be able to upload safe logs

.buildkite/scripts/signAndPublishPackage.sh

@@ -16,7 +16,7 @@ cleanup() {
 
 trap cleanup EXIT
 
-isAlreadyPublished() {
+is_already_published() {
     local packageZip=$1
 
     if curl -s --head https://package-storage.elastic.co/artifacts/packages/${packageZip} | grep -q "HTTP/2 200" ; then
@@ -34,7 +34,7 @@ if ! command -v gsutil &> /dev/null ; then
 fi
 
 
-REPO_NAME=$(repoName "${BUILDKITE_REPO}")
+REPO_NAME=$(repo_name "${BUILDKITE_REPO}")
 BUILD_TAG="buildkite-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_NUMBER}"
 
 REPO_BUILD_TAG="${REPO_NAME}/${BUILD_TAG}"
@@ -76,7 +76,7 @@ google_cloud_auth_publishing() {
     echo "${gsUtilLocation}"
 }
 
-signPackage() {
+sign_package() {
     local package=${1}
     local packageZip=$(basename ${package})
 
@@ -110,7 +110,7 @@ signPackage() {
     rm -r "${gsUtilLocation}"
 }
 
-publishPackage() {
+publish_package() {
     local package=$1
     local packageZip=$(basename ${package})
 
@@ -154,14 +154,14 @@ ls -l "${BUILD_PACKAGES_PATH}"
 for package in $(ls ${BUILD_PACKAGES_PATH}/*.zip); do
     echo "isAlreadyInstalled ${package}?"
     packageZip=$(basename ${package})
-    if isAlreadyPublished ${packageZip} ; then
+    if is_already_published ${packageZip} ; then
         echo "Skipping. ${packageZip} already published"
         continue
     fi
 
     echo "Signing package ${packageZip}"
-    signPackage "${package}"
+    sign_package "${package}"
 
     echo "Publishing package ${packageZip}"
-    publishPackage "${package}"
+    publish_package "${package}"
 done