Reapply grouping workflows and adding support for integration tests

.github/workflows/ci.yml

@@ -1,20 +1,20 @@
-name: CI
+# This workflow builds, tests and lints the project
+name: Build, Test & Lint
 
 on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
+  workflow_call:
+  workflow_dispatch:
 
 jobs:
   test:
+    name: Run
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.12"]
 
     steps:
-    - name: Checkout github repo
+    - name: Checkout
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
       with:
         lfs: true

.github/workflows/image-build.yml

@@ -1,29 +1,48 @@
-# Attempt to build (but not push) the Docker image on Pull Requests
-name: Image build
+# This workflow build (but does not push) the OCI image
+name: OCI Image - Build
+
 on:
-  pull_request:
-    branches:
-      - main
-    paths-ignore:
-      - "**.md"
-      - "docs/**"
-      - "static/**"
-      - "LICENSE"
+  workflow_call:
+    inputs:
+      artifact-name:
+        description: 'The name of the artifact to upload'
+        required: true
+        type: string
+
 permissions:
   contents: read
+
 jobs:
   docker-image:
     name: Check docker image build
     runs-on: ubuntu-latest
+    env:
+      IMAGE_NAME: stacklok/codegate
+      IMAGE_TAG: dev
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3
+      - name: Download artifact
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
+        with:
+          github_token: ${{ github.token }}
+          workflow: ".github/workflows/import_packages.yml"
+          workflow_conclusion: success
+          name: sqlite_data
+          name_is_regexp: true
+          skip_unpack: false
+          if_no_artifact_found: ignore
       - name: Fetch latest release URL
         id: fetch_release_url
         run: |
           echo "LATEST_RELEASE=$(curl -s "https://api.github.com/repos/stacklok/codegate-ui/releases/latest" | jq -r '.zipball_url')" >> $GITHUB_ENV
+      - name: Download git lfs dependencies
+        run: |
+          git lfs install
+          git lfs pull
       - name: Test build on x86
         id: docker_build
         uses: docker/build-push-action@67a2d409c0a876cbe6b11854e3e25193efe4e62d # v5
@@ -37,3 +56,15 @@ jobs:
           cache-to: type=gha,mode=max
           build-args: |
             LATEST_RELEASE=${{ env.LATEST_RELEASE }}
+          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+      - name: Save Docker image as a tarball
+        run: |
+          # List all images
+          docker images
+          # Save the image as a tarball
+          docker save -o image.tar ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
+      - name: Upload Docker image artifact
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
+        with:
+          name: ${{ inputs.artifact-name }}
+          path: image.tar

.github/workflows/image-publish.yml

@@ -1,8 +1,8 @@
-name: Publish Docker Image
+# This workflow builds and publishes the OCI image
+name: OCI Image - Publish
+
 on:
-  release:
-    types:
-      - published
+  workflow_call:
   workflow_dispatch:
 
 jobs:

.github/workflows/import_packages.yml

@@ -1,3 +1,4 @@
+# This workflow syncs the vector database
 name: Sync vector DB
 
 on:

.github/workflows/integration-tests.yml

@@ -0,0 +1,144 @@
+# This workflow will run the integration tests for the project
+name: Tests - Integration
+
+on:
+  workflow_call:
+    inputs:
+      artifact-name:
+        description: 'The name of the artifact to download'
+        required: true
+        type: string
+    secrets:
+      copilot-key:
+        description: 'The Copilot key to use for integration tests'
+        required: true
+
+jobs:
+  integration-tests:
+    name: Run
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.12" ]
+    env:
+      CONTAINER_NAME: "codegate"
+      CERT_FILE: "/app/codegate_volume/certs/ca.crt"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          lfs: true
+
+      - name: Checkout LFS objects
+        run: git lfs pull
+
+      - name: Ensure file permissions for mounted volume
+        run: |
+          mkdir -p ./codegate_volume/certs ./codegate_volume/models ./codegate_volume/db
+          chmod -R 777 ./codegate_volume
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
+        with:
+          name: ${{ inputs.artifact-name }}
+
+      - name: Load Docker image
+        run: |
+          docker load -i image.tar
+          echo "Loaded image:"
+          docker images
+
+      - name: Run container from the loaded image
+        run: |
+          # Get the image name
+          DOCKER_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" | head -n 1)
+          echo "Running container from image: $DOCKER_IMAGE"
+
+          # Run the container
+          docker run --name $CONTAINER_NAME -d -p 8989:8989 -p 9090:9090 \
+            -p 8990:8990 \
+            -v "$(pwd)"/codegate_volume:/app/codegate_volume \
+            -e CODEGATE_APP_LOG_LEVEL=DEBUG \
+            --restart unless-stopped $DOCKER_IMAGE
+
+          # Confirm the container started
+          echo "Container started:"
+          docker ps
+
+          # Verify container is running with correct ports
+          docker ps -f name=$CONTAINER_NAME
+
+          # Check mount configuration
+          docker inspect $CONTAINER_NAME -f '{{ json .Mounts }}' | jq
+
+      - name: Ensure certificates are available in the container
+        timeout-minutes: 4
+        run: |
+          # Wait for the cert file to be available in the container
+          while true; do
+            echo "Checking for $CERT_FILE in container $CONTAINER_NAME..."
+
+            if docker exec "$CONTAINER_NAME" test -f "$CERT_FILE"; then
+              echo "Cert file found: $CERT_FILE"
+              break
+            else
+              echo "Cert file not found. Retrying in 5 seconds..."
+              sleep 5
+            fi
+          done
+
+          # Verify volume contents are accessible
+          docker exec $CONTAINER_NAME ls -la /app/codegate_volume
+
+          # Print the container logs we got so far
+          docker logs $CONTAINER_NAME
+
+      - name: Install the CodeGate certificate
+        run: |
+          docker cp codegate:/app/codegate_volume/certs/ca.crt ./codegate.crt
+          sudo cp ./codegate.crt /usr/local/share/ca-certificates/codegate.crt
+          sudo update-ca-certificates
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
+        with:
+          version: 2.0.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        run: poetry install --with dev
+
+      - name: Run integration tests - Copilot
+        env:
+          CODEGATE_PROVIDERS: "copilot"
+          CA_CERT_FILE: "/home/runner/work/codegate/codegate/codegate_volume/certs/ca.crt"
+          ENV_COPILOT_KEY: ${{ secrets.copilot-key }}
+        run: |
+          poetry run python tests/integration/integration_tests.py
+
+      - name: Print the container logs (useful for debugging)
+        if: always()
+        run: |
+          docker logs $CONTAINER_NAME
+          echo "Models contents:"
+          ls -la codegate_volume/models
+          docker exec $CONTAINER_NAME ls -la /app/codegate_volume/models
+          echo "Certs contents:"
+          ls -la codegate_volume/certs
+          docker exec $CONTAINER_NAME ls -la /app/codegate_volume/certs
+          echo "DB contents:"
+          ls -la codegate_volume/db
+          docker exec $CONTAINER_NAME ls -la /app/codegate_volume/db

.github/workflows/openapi.yml

@@ -1,10 +1,9 @@
+# This workflow generates the OpenAPI docs
 name: Generate OpenAPI Documentation
 
 on:
+  workflow_call:
   workflow_dispatch:
-  push:
-    branches:
-      - main
 
 jobs:
   generate_openapi:

.github/workflows/run-on-pr.yml

@@ -0,0 +1,29 @@
+# These set of workflows run on every pull request
+name: Run - pull_request
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  security:
+    name: Security Checks
+    uses: ./.github/workflows/security.yml
+  ci:
+    name: Build, Test & Lint
+    uses: ./.github/workflows/ci.yml
+  image-build:
+    name: OCI Image - Build
+    needs: [ci] # No need to build the image if the CI fails
+    uses: ./.github/workflows/image-build.yml
+    with:
+      artifact-name: "codegate-image"
+  integration-tests:
+    name: Integration Tests
+    needs: [ci, image-build] # We need the image available in order to run the integration tests
+    uses: ./.github/workflows/integration-tests.yml
+    with:
+      artifact-name: "codegate-image"
+    secrets:
+      copilot-key: ${{ secrets.INTEGRATION_TESTS_COPILOT_KEY }}

.github/workflows/run-on-push.yml

@@ -0,0 +1,19 @@
+# These set of workflows run on every push to the main branch
+name: Run - push_to_main
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+
+jobs:
+  security:
+    name: Security Checks
+    uses: ./.github/workflows/security.yml
+  ci:
+    name: Build, Test & Lint
+    uses: ./.github/workflows/ci.yml
+  openapi:
+    name: Generate the OpenAPI docs
+    needs: [ci]
+    uses: ./.github/workflows/openapi.yml      

.github/workflows/run-on-release-published.yml

@@ -0,0 +1,13 @@
+# These set of workflows run on every published release
+name: Run - release_published
+
+on:
+  workflow_dispatch:
+  release:
+    types:
+      - published
+
+jobs:
+  image-publish:
+    name: OCI Image - Publish
+    uses: ./.github/workflows/image-publish.yml      

.github/workflows/security.yml

@@ -1,17 +1,14 @@
 name: Security
 
 on:
-  pull_request:
-  push:
-    branches:
-      - main
+  workflow_call:
   schedule:
     - cron: '0 0 * * *'
 
 jobs:
   dependencies:
     runs-on: ubuntu-latest
-    name: Dependencies & Secrets Scan
+    name: Dependencies & Secrets Scan via Trivy
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

README.md

@@ -3,8 +3,10 @@
   <img alt="CodeGate logo" src="./static/codegate-logo-dark.svg" width="800px" style="max-width: 100%;">
 </picture>
 
-[![CI](https://github.com/stacklok/codegate/actions/workflows/ci.yml/badge.svg)](https://github.com/stacklok/codegate/actions/workflows/ci.yml) | [![](https://dcbadge.vercel.app/api/server/RkzVuTp3WK?logo=discord&label=Discord&color=5865&style=flat)](https://discord.gg/RkzVuTp3WK)
+---
+[![CI](https://github.com/stacklok/codegate/actions/workflows/run-on-push.yml/badge.svg)](https://github.com/stacklok/codegate/actions/workflows/run-on-push.yml) | [![License: Apache 2.0](https://img.shields.io/badge/License-Apache2.0-brightgreen.svg)](https://opensource.org/licenses/Apache-2.0) | [![](https://dcbadge.vercel.app/api/server/RkzVuTp3WK?logo=discord&label=Discord&color=5865&style=flat)](https://discord.gg/RkzVuTp3WK) 
 
+---
 ## Introduction
 
 <img src="./assets/codegate.gif" style="width: 70%; height: 70%;" />