Docker CI caching

.github/workflows/ci.yml

@@ -28,6 +28,7 @@ name: CI
       - "**"
 permissions:
   contents: read
+  packages: write
 defaults:
   run:
     shell: bash
@@ -42,6 +43,7 @@ jobs:
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
       HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
+      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
       CACHE_DOMAIN: ci-caches.rust-lang.org
@@ -50,18 +52,8 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: mingw-check
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: mingw-check-tidy
-            os: ubuntu-20.04-4core-16gb
-            env: {}
-          - name: x86_64-gnu-llvm-16
-            env:
-              ENABLE_GCC_CODEGEN: "1"
-            os: ubuntu-20.04-16core-64gb
-          - name: x86_64-gnu-tools
-            os: ubuntu-20.04-16core-64gb
+          - name: dist-quick
+            os: ubuntu-20.04
             env: {}
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
@@ -172,6 +164,7 @@ jobs:
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
       HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
+      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       DEPLOY_BUCKET: rust-lang-ci2
       TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
@@ -204,7 +197,7 @@ jobs:
             os: ubuntu-20.04-8core-32gb
             env: {}
           - name: dist-arm-linux
-            os: ubuntu-20.04-16core-64gb
+            os: ubuntu-20.04
             env: {}
           - name: dist-armhf-linux
             os: ubuntu-20.04-8core-32gb
@@ -251,12 +244,12 @@ jobs:
           - name: dist-x86_64-linux
             env:
               CODEGEN_BACKENDS: "llvm,cranelift"
-            os: ubuntu-20.04-16core-64gb
+            os: ubuntu-20.04
           - name: dist-x86_64-linux-alt
             env:
               IMAGE: dist-x86_64-linux
               CODEGEN_BACKENDS: "llvm,cranelift"
-            os: ubuntu-20.04-16core-64gb
+            os: ubuntu-20.04
           - name: dist-x86_64-musl
             env:
               CODEGEN_BACKENDS: "llvm,cranelift"
@@ -554,6 +547,7 @@ jobs:
       CI_JOB_NAME: "${{ matrix.name }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
       HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
+      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       DEPLOY_BUCKET: rust-lang-ci2
       TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
@@ -567,10 +561,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: "llvm,cranelift"
-            os: ubuntu-20.04-16core-64gb
+          - name: dist-quick
+            os: ubuntu-20.04
+            env: {}
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:

src/ci/docker/host-x86_64/dist-quick/Dockerfile

@@ -0,0 +1,54 @@
+# We document platform support for minimum glibc 2.17 and kernel 3.2.
+# CentOS 7 has headers for kernel 3.10, but that's fine as long as we don't
+# actually use newer APIs in rustc or std without a fallback. It's more
+# important that we match glibc for ELF symbol versioning.
+FROM centos:7
+
+WORKDIR /build
+
+RUN yum upgrade -y && \
+    yum install -y epel-release && \
+    yum install -y \
+      bzip2 \
+      file \
+      git \
+      python3 && \
+    yum clean all
+
+# Foo 2
+
+RUN mkdir -p /rustroot/bin
+
+ENV PATH=/rustroot/bin:$PATH
+ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib32:/rustroot/lib
+ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
+WORKDIR /tmp
+RUN mkdir /home/user
+COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
+
+# rustc-perf version from 2023-05-30
+# Should also be changed in the opt-dist tool for other environments.
+ENV PERF_COMMIT 8b2ac3042e1ff2c0074455a0a3618adef97156b1
+
+ENV PGO_HOST=x86_64-unknown-linux-gnu
+
+ENV HOSTS=x86_64-unknown-linux-gnu
+
+ENV RUST_CONFIGURE_ARGS \
+      --enable-full-tools \
+      --enable-sanitizers \
+      --enable-profiler \
+      --enable-compiler-docs \
+      --set llvm.download-ci-llvm=if-available
+
+ENV SCRIPT ls
+ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
+
+# This is the only builder which will create source tarballs
+ENV DIST_SRC 1
+
+# When we build cargo in this container, we don't want it to use the system
+# libcurl, instead it should compile its own.
+ENV LIBCURL_NO_PKG_CONFIG 1
+
+ENV DIST_REQUIRE_ALL_TOOLS 1

src/ci/docker/run.sh

@@ -74,25 +74,6 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
 
       cksum=$(sha512sum $hash_key | \
         awk '{print $1}')
-
-      url="https://$CACHE_DOMAIN/docker/$cksum"
-
-      echo "Attempting to download $url"
-      rm -f /tmp/rustci_docker_cache
-      set +e
-      retry curl --max-time 600 -y 30 -Y 10 --connect-timeout 30 -f -L -C - \
-        -o /tmp/rustci_docker_cache "$url"
-
-      docker_archive_hash=$(sha512sum /tmp/rustci_docker_cache | awk '{print $1}')
-      echo "Downloaded archive hash: ${docker_archive_hash}"
-
-      echo "Loading images into docker"
-      # docker load sometimes hangs in the CI, so time out after 10 minutes with TERM,
-      # KILL after 12 minutes
-      loaded_images=$(/usr/bin/timeout -k 720 600 docker load -i /tmp/rustci_docker_cache \
-        | sed 's/.* sha/sha/')
-      set -e
-      printf "Downloaded containers:\n$loaded_images\n"
     fi
 
     dockerfile="$docker_dir/$image/Dockerfile"
@@ -103,39 +84,30 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then
         context="$script_dir"
     fi
     echo "::group::Building docker image for $image"
+    echo ${cksum}
 
-    # As of August 2023, Github Actions have updated Docker to 23.X,
-    # which uses the BuildKit by default. It currently throws aways all
-    # intermediate layers, which breaks our usage of S3 layer caching.
-    # Therefore we opt-in to the old build backend for now.
-    export DOCKER_BUILDKIT=0
+    echo ${DOCKER_TOKEN} | docker login ghcr.io --username kobzol --password-stdin
+
+    docker buildx create --use --driver docker-container
     retry docker \
+      buildx \
       build \
       --rm \
       -t rust-ci \
       -f "$dockerfile" \
+      --cache-from type=registry,ref=ghcr.io/kobzol/rust-ci:${cksum} \
+      --cache-to type=registry,ref=ghcr.io/kobzol/rust-ci:${cksum},compression=zstd \
+      --output=type=docker \
       "$context"
+
     echo "::endgroup::"
+    docker images
+    docker tag rust-ci ghcr.io/kobzol/rust-ci:${cksum}
+    docker push ghcr.io/kobzol/rust-ci:${cksum}
 
+    # https://lipanski.com/posts/speed-up-your-docker-builds-with-cache-from
     if [ "$CI" != "" ]; then
-      s3url="s3://$SCCACHE_BUCKET/docker/$cksum"
-      upload="aws s3 cp - $s3url"
       digest=$(docker inspect rust-ci --format '{{.Id}}')
-      echo "Built container $digest"
-      if ! grep -q "$digest" <(echo "$loaded_images"); then
-        echo "Uploading finished image $digest to $url"
-        set +e
-        # Print image history for easier debugging of layer SHAs
-        docker history rust-ci
-        docker history -q rust-ci | \
-          grep -v missing | \
-          xargs docker save | \
-          gzip | \
-          $upload
-        set -e
-      else
-        echo "Looks like docker image is the same as before, not uploading"
-      fi
       # Record the container image for reuse, e.g. by rustup.rs builds
       info="$dist/image-$image.txt"
       mkdir -p "$dist"

src/ci/github-actions/ci.yml

@@ -34,6 +34,7 @@ x--expand-yaml-anchors--remove:
     CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
     # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
     HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+    DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   - &public-variables
     SCCACHE_BUCKET: rust-lang-ci-sccache2
@@ -84,7 +85,7 @@ x--expand-yaml-anchors--remove:
     <<: *base-job
 
   - &job-linux-16c
-    os: ubuntu-20.04-16core-64gb
+    os: ubuntu-20.04
     <<: *base-job
 
   - &job-macos-xl
@@ -301,6 +302,7 @@ on:
 
 permissions:
   contents: read
+  packages: write
 
 defaults:
   run:
@@ -328,19 +330,19 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: mingw-check
-            <<: *job-linux-4c
-
-          - name: mingw-check-tidy
-            <<: *job-linux-4c
-
-          - name: x86_64-gnu-llvm-16
-            env:
-              ENABLE_GCC_CODEGEN: "1"
-            <<: *job-linux-16c
-
-          - name: x86_64-gnu-tools
+          - name: dist-quick
             <<: *job-linux-16c
+#          - name: mingw-check
+#            <<: *job-linux-16c
+#
+#          - name: mingw-check-tidy
+#            <<: *job-linux-16c
+#
+#          - name: x86_64-gnu-llvm-15
+#            <<: *job-linux-16c
+#
+#          - name: x86_64-gnu-tools
+#            <<: *job-linux-16c
 
   auto:
     <<: *base-ci-job
@@ -724,10 +726,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - &dist-x86_64-linux
-            name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
+          - name: dist-quick
             <<: *job-linux-16c