Merge pull request #1 from coopTilleuls/feature/cronjob-image

Cronjob cert secrets garbage collector

Author: bcobzh

.github/workflows/build.yml

@@ -9,6 +9,19 @@ on:
         required: true
 
 jobs:
+  build-cronjob-cert-secrets-gc:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Build-cronjob-cert
+      uses: coopTilleuls/action-docker-build-push@v12
+      with:
+        IMAGE_NAME: cronjob-cert-secrets-gc
+        BUILD_CONTEXT: ./cronjob/cert
+        BUILD_TARGET: cronjob
+        REGISTRY_JSON_KEY: ${{ secrets.GITHUB_TOKEN }}
+        IMAGE_REPOSITORY: ghcr.io/cooptilleuls/sre/minimal
+
   build-kubectl:
     runs-on: ubuntu-latest
 

.github/workflows/cd.yml

@@ -1,5 +1,7 @@
 name: CD
 on:
+  pull_request: 
+    types: [ opened, synchronize, reopened, labeled ]
   push:
     branches:
       - main

cronjob/cert/Dockerfile

@@ -0,0 +1,24 @@
+FROM debian:bookworm-slim AS build
+RUN apt-get update -y  && apt-get upgrade -y  && apt-get install curl -y
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"  && \
+    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl.sha256" && \
+    echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check && \
+    chmod +x kubectl && mv kubectl /usr/bin/
+
+FROM debian:bookworm-slim AS cronjob
+COPY --from=build /usr/bin/kubectl /usr/bin/kubectl
+COPY --from=ghcr.io/jqlang/jq:latest /jq /usr/bin/jq
+COPY --chmod=0755 cert-secrets-gc.sh /usr/local/bin/cert-secrets-gc.sh 
+ENV PATH="/usr/local/bin:${PATH}"
+ENTRYPOINT [ "cert-secrets-gc.sh" ]
+CMD [ "--dry" ]
+
+# Fat image includes gcloud
+FROM google/cloud-sdk:latest AS cloud-sdk-base
+RUN apt-get update && apt-get install -y google-cloud-sdk-gke-gcloud-auth-plugin
+
+FROM cronjob AS cronjob-gcloud
+RUN apt-get update -y  && apt-get upgrade -y  && apt-get install python3 -y
+COPY --from=cloud-sdk-base /usr/lib/google-cloud-sdk /usr/lib/google-cloud-sdk
+ENV CLOUDSDK_ROOT_DIR="/usr/lib/google-cloud-sdk"
+ENV PATH="${CLOUDSDK_ROOT_DIR}/bin:${PATH}"

cronjob/cert/Event.template.json

@@ -0,0 +1,25 @@
+{
+    "apiVersion": "v1",
+    "kind": "Event",
+    "metadata": {
+        "name": "{{event-name}}",
+        "namespace": "kube-system"
+    },
+    "involvedObject": {
+        "apiVersion": "v1",
+        "kind": "Secret",
+        "name": "{{secret-name}}",
+        "namespace": "{{secret-namespace}}"
+    },
+    "reportingComponent": "cronjob-cert-secrets-gc",
+    "reportingInstance": "{{host-name}}",
+    "source": {
+        "component": "cronjob-cert-secrets-gc",
+        "host": "{{host-name}}"
+    },
+    "firstTimestamp": "{{first-timestamp}}",
+    "lastTimestamp": "{{last-timestamp}}",
+    "message": "{{message}}",
+    "reason": "UnusedSecret",
+    "type": "{{warning-level}}"
+}

cronjob/cert/README.md

@@ -0,0 +1,29 @@
+# Cert Secrets Garbage Collector
+
+This image deletes unused secrets from the cluster
+
+## Run the image in a cluster
+
+This image needs access secrets and ingresses. So here we use service account `my-account` in namespace `my-namespace` which has the needed rights
+
+```
+kubectl run -n my-namespace test-cert-secrets-gc --rm -i --image=ghcr.io/cooptilleuls/sre/minimal/cronjob-cert-secrets-gc:latest --overrides='{ "spec": { "serviceAccountName": "my-account" } }'
+```
+
+## Run the image from local environment (outside the cluster)
+
+### Build the image which includes gcloud sdk
+
+The minimal image on the registry can not authenticate on google cloud. If you want to use it, you need to build the fat image which includes gcloud sdk
+
+```
+docker build --network host -t cert-secrets-gc-fat .
+```
+
+### Run the image with gcloud config
+
+You can now run the image. Be sure to mount any relevant config as a volume in the container
+
+```
+docker run --network host -v ~/.kube:/root/.kube -v ~/.config/gcloud:/root/.config/gcloud -it --rm cert-secrets-gc-fat
+```

cronjob/cert/cert-secrets-gc.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+set -e
+set -x
+
+if [[ "$1" == "--help" ]]; then
+    echo "Usage: cert-secrets-gc.sh [option]"
+    echo "Option:"
+    echo "  --help            Show this help message"
+    echo "  --dry             Dry run. Dont delete secrets"
+    echo "  --filter <regex>  A regex expression used to filter secrets"
+    exit 0
+fi
+
+if [[ "$1" == "--filter" ]]; then
+    SECRET_ANNOTATION_FILTER=$2
+fi
+
+if [[ "$1" == "--dry" ]]; then
+    echo "Argument --dry is set. Will NOT be deleting anything"
+fi
+
+log_event() {
+    local message="$1"
+    local level="$2"
+    local now=$(date -u --date='now' +%Y-%m-%dT%H:%M:%SZ)
+    local secret_name="${3:-"default"}"
+    local secret_namespace="${4:-"default"}"
+    local event_name="cert-secrets-gc-$(date -u +%s%N)"
+    local host_name=$(hostname -s)
+    local template=$(cat Event.template.json)
+    template=${template//\{\{message\}\}/$message}
+    template=${template//\{\{warning-level\}\}/$level}
+    template=${template//\{\{first-timestamp\}\}/$JOBSTART}
+    template=${template//\{\{last-timestamp\}\}/$now}
+    template=${template//\{\{secret-name\}\}/$secret_name}
+    template=${template//\{\{secret-namespace\}\}/$secret_namespace}
+    template=${template//\{\{event-name\}\}/$event_name}
+    template=${template//\{\{host-name\}\}/$host_name}
+    echo "$template" | kubectl create -f -
+}
+
+# cert should have been renewed before this date
+# if it's not we delete the secret
+RENEWAL_DATE=$(date -d "@$(( $(date +%s) - 14 * 24 * 60 * 60 ))" +%s)
+
+# secrets referenced in certificates
+kubectl get certificates  -A  -o json  | jq  -r ".items[] | ( .metadata.namespace + \"/\" + .spec.secretName  )" >/tmp/secrets_from_certs
+
+# secrets managed by cert manager and NOT referenced in certs
+kubectl get secrets -A -o json | jq  -r '.items[] | select( .metadata.annotations["cert-manager.io/issuer-group"] == "cert-manager.io" and .type == "kubernetes.io/tls" ) | (.metadata.namespace + "/" + .metadata.name )' | grep -x -v -f /tmp/secrets_from_certs >/tmp/abandoned  || [ $? -eq 1 ]
+
+# secrets currently used by ingress
+kubectl get ing -A -o  go-template='{{range .items}}{{$namespace := .metadata.namespace}}{{range .spec.tls }}{{$namespace}}/{{ .secretName }}{{ end }}{{printf "\n"}}{{ end}}' | sed -e '/^$/d; /<no value>/d' > /tmp/used
+
+# secrets from certificates not renewed
+# we ignore certificates without renewalTime
+kubectl get certificates  -A  -o json  | jq  -r ".items[] | select( .status.renewalTime | select(type == \"string\") | fromdateiso8601 < $RENEWAL_DATE ) | ( .metadata.namespace + \"/\" + .spec.secretName  )" >/tmp/not_renewed
+
+#  get list of unused secrets
+grep -v -x -f /tmp/used /tmp/not_renewed > /tmp/unused || echo "secrets not renewed not found"
+grep -v -x -f /tmp/used /tmp/abandoned >> /tmp/unused || echo "secrets without certs not found"
+# if there's a filter we use it restrictively
+# if it returns nothing, we don't delete secrets
+if [ -n "$SECRET_ANNOTATION_FILTER" ]; then cat /tmp/unused | grep "$SECRET_ANNOTATION_FILTER" >/tmp/unused_filtered ; mv /tmp/unused_filtered /tmp/unused; fi
+
+for CERT in $(cat /tmp/unused ) ; do
+  NAMESPACE=$${CERT%%/*}
+  SECRET=$${CERT##*/}
+  if kubectl -n $${NAMESPACE} get secret/$${SECRET} &>/dev/null ; then
+     if [[ "$1" == "--dry" ]]; then
+         echo "Not deleting $${NAMESPACE}}/$${SECRET}"
+     else
+         echo "Deleting $${NAMESPACE}}/$${SECRET}"
+         log_event "Deleting secret  $${NAMESPACE}}/$${SECRET}" "Warning" $${SECRET} $${NAMESPACE}
+         kubectl -n $${NAMESPACE} delete secret/$${SECRET}
+     fi
+  fi
+done