[v2] Merge main on Oct. 18

.ci/Jenkinsfile

@@ -19,7 +19,7 @@ pipeline {
     DEVELOPER_MODE=true
   }
   options {
-    timeout(time: 2, unit: 'HOURS')
+    timeout(time: 3, unit: 'HOURS')
     buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
@@ -39,6 +39,14 @@ pipeline {
     // disabled by default, but required for merge:
     // opt-in with 'ci:end-to-end' tag on PR
     booleanParam(name: 'end_to_end_tests_ci', defaultValue: false, description: 'Enable End-to-End tests')
+
+    // disabled by default, but required for merge:
+    // opt-in with 'ci:extended-windows' tag on PR
+    booleanParam(name: 'extended_windows_ci', defaultValue: false, description: 'Enable Extended Windows tests')
+
+    // disabled by default, but required for merge:
+    // opt-in with 'ci:extended-m1' tag on PR
+    booleanParam(name: 'extended_m1_ci', defaultValue: false, description: 'Enable M1 tests')
   }
   stages {
     stage('Checkout') {
@@ -51,6 +59,10 @@ pipeline {
           setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString())
           setEnvVar('PACKAGING_CHANGES', isGitRegionMatch(patterns: [ '(^dev-tools/packaging/.*|.ci/Jenkinsfile)' ], shouldMatchAll: false).toString())
           setEnvVar('K8S_CHANGES', isGitRegionMatch(patterns: [ '(^deploy/kubernetes/.*|^version/docs/version.asciidoc|.ci/Jenkinsfile)' ], shouldMatchAll: false).toString())
+          setEnvVar('EXT_WINDOWS_CHANGES', isGitRegionMatch(patterns: [ '.ci/Jenkinsfile' ], shouldMatchAll: false).toString())
+          setEnvVar('EXT_M1_CHANGES', isGitRegionMatch(patterns: [ '.ci/Jenkinsfile' ], shouldMatchAll: false).toString())
+          // set the GO_VERSION env variable with the go version to be used in withMageEnv
+          setEnvVar('GO_VERSION', readFile(file: '.go-version')?.trim())
         }
       }
     }
@@ -79,7 +91,8 @@ pipeline {
         axes {
           axis {
             name 'PLATFORM'
-            values 'ubuntu-20.04 && immutable', 'aws && aarch64', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable', 'macos12 && x86_64'
+            // Orka workers are not healthy (memory and connectivity issues)
+            values 'ubuntu-20.04 && immutable', 'aws && aarch64', 'windows-2016 && windows-immutable', 'windows-2022 && windows-immutable' //, 'macos12 && x86_64'
           }
         }
         stages {
@@ -147,7 +160,7 @@ pipeline {
               }
             }
             steps {
-              runK8s(k8sVersion: 'v1.23.0', kindVersion: 'v0.11.1', context: "K8s-${PLATFORM}")
+              runK8s(k8sVersion: 'v1.25.0-beta.0', kindVersion: 'v0.14.0', context: "K8s-${PLATFORM}")
             }
           }
           stage('Package') {
@@ -219,7 +232,7 @@ pipeline {
         axes {
           axis {
             name 'K8S_VERSION'
-            values "v1.24.0", "v1.23.6", "v1.22.9", "v1.21.12"
+            values "v1.25.0","v1.24.3", "v1.23.6", "v1.22.9"
           }
         }
         stages {
@@ -238,6 +251,153 @@ pipeline {
         }
       }
     }
+    stage('Sync K8s') { //This stage opens a PR to kibana Repository in order to sync k8s manifests
+      when {
+        // Only on main branch
+        // Enable if k8s related changes.
+        allOf {
+          branch 'main'         // Only runs for branch main
+          expression { return env.K8S_CHANGES == "true" }   // If k8s changes
+        }
+      }
+      failFast false
+        agent {label 'ubuntu-20.04 && immutable'}
+        options { skipDefaultCheckout() }
+        stages {
+          stage('OpenKibanaPR') {
+            steps {
+              withGhEnv(version: '2.4.0') {
+                deleteDir()
+                unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+                  dir("${BASE_DIR}/deploy/kubernetes"){
+                  sh(label: '[File Creation] Create-Needed-Manifest', script: """
+                    WITHOUTCONFIG=true make generate-k8s
+                    ./creator_k8s_manifest.sh . """)
+                  sh(label: '[Clone] Kibana-Repository', script: """
+                    make ci-clone-kibana-repository
+                    cp Makefile ./kibana
+                    cd kibana
+                    make ci-create-kubernetes-templates-pull-request """)
+                  }
+              }
+            }
+            post {
+              always {
+                junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
+              }
+            }
+          }
+      }
+    }
+    stage('extended windows') {
+      when {
+        // Always when running builds on branches/tags
+        // Enable if extended windows support related changes.
+        beforeAgent true
+        anyOf {
+          not { changeRequest() }
+          expression { return isExtendedWindowsEnabled() && env.ONLY_DOCS == "false"}
+        }
+      }
+      failFast false
+      matrix {
+        agent {label "${PLATFORM} && windows-immutable"}
+        options { skipDefaultCheckout() }
+        axes {
+          axis {
+            name 'PLATFORM'
+            values 'windows-8', 'windows-10', 'windows-11'
+          }
+        }
+        stages {
+          stage('build'){
+            options { skipDefaultCheckout() }
+            steps {
+              withGithubNotify(context: "Build-${PLATFORM}") {
+                deleteDir()
+                unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+                withMageEnv(){
+                  dir("${BASE_DIR}"){
+                    cmd(label: 'Go build', script: 'mage build')
+                  }
+                }
+              }
+            }
+          }
+          stage('Test') {
+            options { skipDefaultCheckout() }
+            steps {
+              withGithubNotify(context: "Test-${PLATFORM}") {
+                withMageEnv(){
+                  dir("${BASE_DIR}"){
+                    withEnv(["TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
+                      cmd(label: 'Go unitTest', script: 'mage unitTest')
+                    }
+                  }
+                }
+              }
+            }
+            post {
+              always {
+                junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
+                whenTrue(isCodeCoverageEnabled()) {
+                  coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml')
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+    stage('m1') {
+      agent { label 'orka && darwin && aarch64' }
+      options { skipDefaultCheckout() }
+      when {
+        // Always when running builds on branches/tags
+        // Enable if extended M1 support related changes.
+        beforeAgent true
+        anyOf {
+          not { changeRequest() }
+          expression { return isExtendedM1Enabled() && env.ONLY_DOCS == "false"}
+        }
+      }
+      stages {
+        stage('build'){
+          steps {
+            withGithubNotify(context: "Build-darwin-aarch64") {
+              deleteDir()
+              unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+              withMageEnv(){
+                dir("${BASE_DIR}"){
+                  cmd(label: 'Go build', script: 'mage build')
+                }
+              }
+            }
+          }
+        }
+        stage('Test') {
+          steps {
+            withGithubNotify(context: "Test-darwin-aarch64") {
+              withMageEnv(){
+                dir("${BASE_DIR}"){
+                  withEnv(["TEST_COVERAGE=${isCodeCoverageEnabled()}"]) {
+                    cmd(label: 'Go unitTest', script: 'mage unitTest')
+                  }
+                }
+              }
+            }
+          }
+          post {
+            always {
+              junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/build/TEST-*.xml")
+              whenTrue(isCodeCoverageEnabled()) {
+                coverageReport(baseDir: "**/build", reportFiles: 'TEST-go-unit.html', coverageFiles: 'TEST-go-unit-cov.xml')
+              }
+            }
+          }
+        }
+      }
+    }
     stage('e2e tests') {
       when {
         // Always when running builds on branches/tags
@@ -250,7 +410,6 @@ pipeline {
         }
       }
       steps {
-        // TODO: what's the testMatrixFile to be used if any
         runE2E(testMatrixFile: '.ci/.e2e-tests-for-elastic-agent.yaml',
                beatVersion: "${env.BEAT_VERSION}-SNAPSHOT",
                elasticAgentVersion: "${env.BEAT_VERSION}-SNAPSHOT",
@@ -374,3 +533,17 @@ def isE2eEnabled() {
 def isPackageEnabled() {
   return env.PACKAGING_CHANGES == "true" || env.GITHUB_COMMENT?.contains('package') || matchesPrLabel(label: 'ci:package')
 }
+
+/**
+* Wrapper to know if the build should enable the windows extended support
+*/
+def isExtendedWindowsEnabled() {
+  return env.EXT_WINDOWS_CHANGES == "true" || params.extended_windows_ci || env.GITHUB_COMMENT?.contains('extended windows') || matchesPrLabel(label: 'ci:extended-windows')
+}
+
+/**
+* Wrapper to know if the build should enable the M1 extended support
+*/
+def isExtendedM1Enabled() {
+  return env.EXT_M1_CHANGES == "true" || params.extended_m1_ci || env.GITHUB_COMMENT?.contains('extended m1') || matchesPrLabel(label: 'ci:extended-m1')
+}

.ci/schedule-daily.groovy

@@ -20,7 +20,7 @@ pipeline {
   stages {
     stage('Nighly beats builds') {
       steps {
-        runBuilds(quietPeriodFactor: 2000, branches: ['main', '8.<minor>', '8.<next-patch>', '7.<minor>'])
+        runBuilds(quietPeriodFactor: 2000, branches: ['main', '8.<minor>', '8.<next-patch>'])
       }
     }
   }

.github/workflows/changelog.yml

@@ -0,0 +1,17 @@
+name: Changelog
+on: [pull_request]
+
+jobs:
+  fragments:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Fetch Go version from .go-version
+      run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV
+    - uses: actions/setup-go@v3
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: check pr-has-fragment
+      run: |
+        GOBIN=$PWD/bin go install github.com/elastic/elastic-agent-changelog-tool@latest
+        ./bin/elastic-agent-changelog-tool pr-has-fragment --repo ${{ github.event.repository.name }} ${{github.event.number}}

.github/workflows/golangci-lint.yml

@@ -18,22 +18,22 @@ jobs:
     name: lint
     runs-on: ${{  matrix.os }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # Uses Go version from the repository.
       - name: Read .go-version file
         id: goversion
         run: echo "::set-output name=version::$(cat .go-version)"
 
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: "${{ steps.goversion.outputs.version }}"
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.45.2
+          version: v1.47.2
 
           # Give the job more time to execute.
           # Regarding `--whole-files`, the linter is supposed to support linting of changed a patch only but,

.github/workflows/macos.yml

@@ -0,0 +1,25 @@
+name: macos
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - 8.*
+
+jobs:
+  macos:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Fetch Go version from .go-version
+      run: echo "GO_VERSION=$(cat .go-version)" >> $GITHUB_ENV
+    - uses: actions/setup-go@v3
+      with:
+        go-version: ${{ env.GO_VERSION }}
+    - name: Install dependencies
+      run:  go install github.com/magefile/mage
+    - name: Run build
+      run: mage build
+    - name: Run test
+      run: mage unitTest