[chore] add a smoke test for the TA (#5505)

Author: hughesjj

.github/workflows/ansible.yml

@@ -17,6 +17,7 @@ on:
       - '.github/workflows/ansible.yml'
       - 'deployments/ansible/**'
       - '!**.md'
+      - '!packaging/technical-addon/**'
   schedule:
     - cron: '0 0 * * 1,4' # Every Monday and Thrusday at midnight UTC
 

.github/workflows/auto-instrumentation.yml

@@ -11,6 +11,7 @@ on:
       - 'instrumentation/**'
       - 'internal/**'
       - '!**.md'
+      - '!packaging/technical-addon/**'
 
 concurrency:
   group: auto-instrumentation-${{ github.event.pull_request.number || github.ref }}

.github/workflows/integration-test.yml

@@ -17,6 +17,7 @@ on:
       - 'go.mod'
       - 'go.sum'
       - '!**.md'
+      - '!packaging/technical-addon/**'
 
 concurrency:
   group: integration-test-${{ github.event.pull_request.number || github.ref }}

.github/workflows/linux-package-test.yml

@@ -12,6 +12,7 @@ on:
       - 'Makefile'
       - 'Makefile.Common'
       - '!**.md'
+      - '!packaging/technical-addon/**'
 
 concurrency:
   group: linux-package-test-${{ github.event.pull_request.number || github.ref }}

.github/workflows/splunk-ta-otel.yml

@@ -0,0 +1,62 @@
+name: splunk-ta-otel-build
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - '.github/workflows/splunk-ta-otel.yml'
+      - 'packaging/technical-addon/**'
+      - 'cmd/**'
+      - 'internal/**'
+      - 'pkg/**'
+      - 'tests/**'
+      - '!**.md'
+
+concurrency:
+  group: splunk-technical-addon-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  GO_VERSION: 1.22.7
+
+jobs:
+  setup-environment:
+    name: setup-environment
+    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check out the codebase.
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: '**/go.sum'
+
+      - name: Installing dependency
+        run: |
+          make install-tools
+
+
+  test:
+    name: test
+    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
+    runs-on: ubuntu-20.04
+    needs: [setup-environment]
+    steps:
+      - name: Check out the codebase.
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: '**/go.sum'
+
+      - name: Build & Package TA
+        run: |
+          set -o pipefail
+          make -e package-technical-addon

Makefile

@@ -271,3 +271,7 @@ endif
 	docker create --platform linux/$(GOARCH) --name otelcol-fips-builder-$(GOOS)-$(GOARCH) otelcol-fips-builder-$(GOOS)-$(GOARCH) true >/dev/null
 	docker cp otelcol-fips-builder-$(GOOS)-$(GOARCH):/src/bin/otelcol_$(GOOS)_$(GOARCH)$(EXTENSION) ./bin/otelcol-fips_$(GOOS)_$(GOARCH)$(EXTENSION)
 	@docker rm -f otelcol-fips-builder-$(GOOS)-$(GOARCH) >/dev/null
+
+
+.PHONY: package-technical-addon
+package-technical-addon: bundle.d otelcol generate-technical-addon package-ta smoketest-ta

packaging/technical-addon/Makefile

@@ -22,6 +22,12 @@ generate-technical-addon: env-guard-all
 	BUILD_DIR="$(BUILD_DIR)" \
 	SOURCE_DIR="$(SOURCE_DIR)" \
 	$(SOURCE_DIR)/packaging-scripts/make-buildspace.sh
+	# Should only have affect if otelcol or download make targets were run first
+	# Running make download-otelcol will overwrite anything locally created
+	mkdir -p $(BUILD_DIR)/out/bin/
+	cp -Rv bin/* $(BUILD_DIR)/out/bin/
+	mkdir -p $(BUILD_DIR)/out/smart-agent
+	cp ./dist/agent-bundle_linux_amd64.tar.gz $(BUILD_DIR)/out/smart-agent
 
 .PHONY: env-guard-all
 env-guard-all:
@@ -43,8 +49,8 @@ env-guard-verify: env-guard-ta
 	UF_VERSION="$(UF_VERSION)" \
 	$(SOURCE_DIR)/packaging-scripts/env/verify.sh
 
-.PHONY: download
-download: env-guard-ta
+.PHONY: download-otelcol
+download-otelcol: env-guard-ta
 	BUILD_DIR="$(BUILD_DIR)" \
 	SOURCE_DIR="$(SOURCE_DIR)" \
 	OTEL_COLLECTOR_VERSION="$(OTEL_COLLECTOR_VERSION)" \
@@ -54,8 +60,8 @@ download: env-guard-ta
 	$(SOURCE_DIR)/packaging-scripts/download-release.sh
 
 
-.PHONY: package
-package: env-guard-ta
+.PHONY: package-ta
+package-ta: env-guard-ta
 	BUILD_DIR="$(BUILD_DIR)" \
 	SOURCE_DIR="$(SOURCE_DIR)" \
 	ARCH="$(ARCH)" \
@@ -64,10 +70,10 @@ package: env-guard-ta
 	$(SOURCE_DIR)/packaging-scripts/package-ta.sh
 
 .PHONY: distribute-ta
-distribute-ta: generate-technical-addon download package
+distribute-ta: generate-technical-addon download-otelcol package-ta
 
-.PHONY: verify
-verify: env-guard-verify
+.PHONY: verify-ta
+verify-ta: env-guard-verify
 	SOURCE_DIR="$(SOURCE_DIR)" \
 	BUILD_DIR="$(BUILD_DIR)" \
 	ARCH="$(ARCH)" \
@@ -79,8 +85,16 @@ verify: env-guard-verify
 	OLLY_ACCESS_TOKEN="$(OLLY_ACCESS_TOKEN)" \
 	$(SOURCE_DIR)/packaging-scripts/deploy-and-verify.sh
 
-.PHONY: update-deps
-update-deps:
+.PHONY: smoketest-ta
+smoketest-ta:
+	SOURCE_DIR="$(SOURCE_DIR)" \
+	BUILD_DIR="$(BUILD_DIR)" \
+	ARCH="$(ARCH)" \
+	PLATFORM="$(PLATFORM)" \
+	$(SOURCE_DIR)/packaging-scripts/cicd-tests/smoketests/smoketest.sh
+
+.PHONY: update-ta-deps
+update-ta-deps:
 	SOURCE_DIR="$(SOURCE_DIR)" \
 	SPLUNK_OTEL_VERSION="$(SPLUNK_OTEL_VERSION)" \
 	$(SOURCE_DIR)/packaging-scripts/update-otel-version.sh
@@ -89,14 +103,14 @@ update-deps:
 	TA_VERSION="$(TA_VERSION)" \
 	$(SOURCE_DIR)/packaging-scripts/update-ta-version.sh
 
-.PHONY: release
-release:
+.PHONY: release-ta
+release-ta:
 	SOURCE_DIR="$(SOURCE_DIR)" \
 	$(SOURCE_DIR)/packaging-scripts/release-ta-version.sh
 
-.PHONY: update-and-release
-update-and-release: update-deps release
+.PHONY: update-and-release-ta
+update-and-release: update-ta-deps release-ta
 
-.PHONY: clean
-clean:
+.PHONY: clean-ta
+clean-ta:
 	rm -rf build

packaging/technical-addon/packaging-scripts/cicd-tests/add-access-token.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -eu
+
+repack_with_access_token() {
+    local token="${SPLUNK_O11Y_ACCESS_TOKEN:-$1}"
+    local path="${TA_TGZ_PATH:-$2}"
+
+    echo "Adding access token to: $path"
+
+    TEMP_DIR="$BUILD_DIR/repack"
+    mkdir -p "$TEMP_DIR"
+    tar xzvf "$BUILD_DIR/out/distribution/Splunk_TA_otel.tgz" -C "$TEMP_DIR"
+    cp -r "$TEMP_DIR/Splunk_TA_otel/default/" "$TEMP_DIR/Splunk_TA_otel/local/"
+    echo "$token" > "$TEMP_DIR/Splunk_TA_otel/local/access_token"
+
+    random_suffix="$(tr -dc 'A-Za-z0-9' < /dev/urandom | head -c 6)"
+    repacked="$TEMP_DIR/Splunk_TA_otel-${random_suffix}.tgz"
+    tar -C "$TEMP_DIR" -hcz --file "$repacked"  "Splunk_TA_otel"
+
+    echo "$repacked"
+    return 0
+}

packaging/technical-addon/packaging-scripts/cicd-tests/smoketests/docker-compose.yml

@@ -0,0 +1,20 @@
+version: "3.6"
+# See https://splunk.github.io/docker-splunk/advanced/APP_INSTALL.html#apps-on-filesystem
+services:
+  so1:
+    image: ${SPLUNK_IMAGE:-splunk/splunk:latest}
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/en-US/account/login"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 60s
+    hostname: so1
+    environment:
+      - SPLUNK_START_ARGS=--accept-license
+      - SPLUNK_APPS_URL=/addon-dir/$REPACKED_TA_NAME
+      - SPLUNK_PASSWORD=Chang3d!
+    ports:
+      - 8000
+    volumes:
+      - $ADDON_DIR:/addon-dir

packaging/technical-addon/packaging-scripts/cicd-tests/smoketests/smoketest.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -eu
+set -o pipefail
+
+[[ -z "$BUILD_DIR" ]] && echo "BUILD_DIR not set" && exit 1
+[[ -z "$SOURCE_DIR" ]] && echo "SOURCE_DIR not set" && exit 1
+
+source "${SOURCE_DIR}/packaging-scripts/cicd-tests/add-access-token.sh"
+SPLUNK_APPS_URL="$(repack_with_access_token "foobar" "$BUILD_DIR/out/distribution/Splunk_TA_otel.tgz" | tail -n 1)"
+REPACKED_TA_NAME="$(basename "$SPLUNK_APPS_URL")"
+ADDON_DIR="$(realpath "$(dirname "$SPLUNK_APPS_URL")")"
+echo "Testing with hot TA $SPLUNK_APPS_URL ($ADDON_DIR and $REPACKED_TA_NAME)"
+DOCKER_COMPOSE_CONFIG="$SOURCE_DIR/packaging-scripts/cicd-tests/smoketests/docker-compose.yml"
+ADDON_DIR="$ADDON_DIR" REPACKED_TA_NAME="$REPACKED_TA_NAME" docker compose --file "$DOCKER_COMPOSE_CONFIG" up --detach --wait
+
+
+# Should trap this
+ADDON_DIR="$ADDON_DIR" REPACKED_TA_NAME="$REPACKED_TA_NAME" docker compose --file "$DOCKER_COMPOSE_CONFIG" down

packaging/technical-addon/packaging-scripts/package-ta.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -eux
+#!/bin/bash -eu
 set -o pipefail
 
 TA_NAME="Splunk_TA_otel"
@@ -37,13 +37,15 @@ if [ "$PLATFORM" == "darwin" ] ; then  # NOTE Darwin not used yet
 fi
 
 # Copy smart agent bundle into addon package directory
+version=""
+if [ "$OTEL_COLLECTOR_VERSION" == "" ]; then 
+    version="${OTEL_COLLECTOR_VERSION}_"
+fi
 if [ "$PLATFORM" == "windows" ] || [ "$PLATFORM" == "all" ] ; then
-    SMART_AGENT_BUNDLE="agent-bundle_${OTEL_COLLECTOR_VERSION}_windows_${ARCH}.zip"
-    cp "$BUILD_DIR/out/smart-agent/$SMART_AGENT_BUNDLE" "$TA_PACKAGING_DIR/$TA_NAME/windows_$SPLUNK_ARCH/bin/agent-bundle_windows_${ARCH}.zip"
+    cp "$BUILD_DIR/out/smart-agent/agent-bundle_${version}windows_${ARCH}.zip" "$TA_PACKAGING_DIR/$TA_NAME/windows_$SPLUNK_ARCH/bin/agent-bundle_windows_${ARCH}.zip"
 fi
 if [ "$PLATFORM" == "linux" ] || [ "$PLATFORM" == "all" ] ; then
-    SMART_AGENT_BUNDLE="agent-bundle_${OTEL_COLLECTOR_VERSION}_linux_${ARCH}.tar.gz"
-    cp "$BUILD_DIR/out/smart-agent/$SMART_AGENT_BUNDLE" "$TA_PACKAGING_DIR/$TA_NAME/linux_$SPLUNK_ARCH/bin/agent-bundle_linux_${ARCH}.tar.gz"
+    cp "$BUILD_DIR/out/smart-agent/agent-bundle_${version}linux_${ARCH}.tar.gz" "$TA_PACKAGING_DIR/$TA_NAME/linux_$SPLUNK_ARCH/bin/agent-bundle_linux_${ARCH}.tar.gz"
 fi
 
 # Prepare artifact directory structure