Fix rustls integration CI (#915)

* Fix Rustls integration CI

* Also fix --latest-release

Author: justsmth

.github/workflows/integration.yml

@@ -61,11 +61,11 @@ jobs:
       - name: Set Rust toolchain override
         run: rustup override set ${{ steps.toolchain.outputs.name }}
       - name: Install cargo-download
-        run: cargo install cargo-download
+        run: cargo install cargo-download cargo-show
       - name: Run rustls integration
         working-directory: ./aws-lc-rs
         run: |
-          ./scripts/run-rustls-integration.sh ${{ matrix.latest == 1 && '--latest-release' }}
+          ./scripts/run-rustls-integration.sh ${{ matrix.latest == 1 && '--latest-release' }} --cleanup
 
   windows-debug-crt-static-test:
     if: github.repository_owner == 'aws'

aws-lc-rs/scripts/run-rustls-integration.sh

@@ -5,18 +5,28 @@
 ROOT="${GITHUB_WORKSPACE:-$(git rev-parse --show-toplevel)}"
 
 latest_release=0
+cleanup=0
 for arg in "$@"; do
   if [ "$arg" = "--latest-release" ]; then
     latest_release=1
   fi
+  if [ "$arg" = "--cleanup" ]; then
+    cleanup=1
+  fi
 done
 
 CLEANUP_ON_EXIT=()
 
 function cleanup() {
+  if [ "$cleanup" -eq 0 ]; then
+    echo "You can delete the following directories:"
+    echo "${CLEANUP_ON_EXIT[@]}"
+  else
     for x in "${CLEANUP_ON_EXIT[@]}"; do
-        rm -rf "${x}"
+      echo "Deleting: ${x}"
+      rm -rf "${x}"
     done
+  fi
 }
 
 trap cleanup EXIT
@@ -27,11 +37,7 @@ function sed_replace {
   shift
   while [[ $# -gt 0 ]]; do
       local pattern="${1}";
-      if [[ "$(uname)" == "Darwin" ]]; then
-      	sed -i '' -e "${pattern}" "${filepath}"
-      else
-      	sed -i -e "${pattern}" "${filepath}"
-      fi
+     	perl -0777 -pi -e "${pattern}" "${filepath}"
       shift
   done
 }
@@ -44,24 +50,46 @@ popd &>/dev/null # "${ROOT}/aws-lc-rs"
 
 RUSTLS_RCGEN_DIR="$(mktemp -d)"
 CLEANUP_ON_EXIT+=("${RUSTLS_RCGEN_DIR}")
-cargo download -o "${RUSTLS_RCGEN_DIR}"/rcgen.tar.gz rcgen
-tar xvzf "${RUSTLS_RCGEN_DIR}"/rcgen.tar.gz -C "${RUSTLS_RCGEN_DIR}" --strip-components=1
-rm "${RUSTLS_RCGEN_DIR}"/rcgen.tar.gz
-RUSTLS_RCGEN_COMMIT="$(jq -r '.git.sha1' ${RUSTLS_RCGEN_DIR}/.cargo_vcs_info.json)"
-rm -rf "${RUSTLS_RCGEN_DIR}" # Cleanup before we clone
+if [[ $latest_release == "1" ]]; then
+  RUSTLS_RCGEN_STABLE_VERSION="$(cargo show --json rcgen | jq -r '
+    [.versions[] |
+     select(.yanked == false and (.num | test("alpha|beta|rc") | not))
+    ][0].num
+  ')"
+
+  cargo download -o "${RUSTLS_RCGEN_DIR}"/rcgen.tar.gz "rcgen=${RUSTLS_RCGEN_STABLE_VERSION}"
+  tar xvzf "${RUSTLS_RCGEN_DIR}"/rcgen.tar.gz -C "${RUSTLS_RCGEN_DIR}" --strip-components=1
+  rm "${RUSTLS_RCGEN_DIR}"/rcgen.tar.gz
+  RUSTLS_RCGEN_COMMIT="$(jq -r '.git.sha1' ${RUSTLS_RCGEN_DIR}/.cargo_vcs_info.json)"
+  rm -rf "${RUSTLS_RCGEN_DIR}" # Cleanup before we clone
+fi
 
 RUSTLS_WEBPKI_DIR="$(mktemp -d)"
 CLEANUP_ON_EXIT+=("${RUSTLS_WEBPKI_DIR}")
-cargo download -o "${RUSTLS_WEBPKI_DIR}"/rustls-webpki.tar.gz rustls-webpki
-tar xvzf "${RUSTLS_WEBPKI_DIR}"/rustls-webpki.tar.gz -C "${RUSTLS_WEBPKI_DIR}" --strip-components=1
-rm "${RUSTLS_WEBPKI_DIR}"/rustls-webpki.tar.gz
-RUSTLS_WEBPKI_COMMIT="$(jq -r '.git.sha1' ${RUSTLS_WEBPKI_DIR}/.cargo_vcs_info.json)"
-rm -rf "${RUSTLS_WEBPKI_DIR}" # Cleanup before we clone
+if [[ $latest_release == "1" ]]; then
+  RUSTLS_WEBPKI_STABLE_VERSION="$(cargo show --json rustls-webpki | jq -r '
+    [.versions[] |
+     select(.yanked == false and (.num | test("alpha|beta|rc") | not))
+    ][0].num
+  ')"
+
+  cargo download  -o "${RUSTLS_WEBPKI_DIR}"/rustls-webpki.tar.gz "rustls-webpki=${RUSTLS_WEBPKI_STABLE_VERSION}"
+  tar xvzf "${RUSTLS_WEBPKI_DIR}"/rustls-webpki.tar.gz -C "${RUSTLS_WEBPKI_DIR}" --strip-components=1
+  rm "${RUSTLS_WEBPKI_DIR}"/rustls-webpki.tar.gz
+  RUSTLS_WEBPKI_COMMIT="$(jq -r '.git.sha1' ${RUSTLS_WEBPKI_DIR}/.cargo_vcs_info.json)"
+  rm -rf "${RUSTLS_WEBPKI_DIR}" # Cleanup before we clone
+fi
 
 RUSTLS_DIR="$(mktemp -d)"
 CLEANUP_ON_EXIT+=("${RUSTLS_DIR}")
 if [[ $latest_release == "1" ]]; then
-  cargo download -o "${RUSTLS_DIR}"/rustls.tar.gz rustls
+  RUSTLS_STABLE_VERSION="$(cargo show --json rustls | jq -r '
+    [.versions[] |
+     select(.yanked == false and (.num | test("alpha|beta|rc") | not))
+    ][0].num
+  ')"
+
+  cargo download -o "${RUSTLS_DIR}"/rustls.tar.gz "rustls=${RUSTLS_STABLE_VERSION}"
   tar xvzf "${RUSTLS_DIR}"/rustls.tar.gz -C "${RUSTLS_DIR}" --strip-components=1
   rm "${RUSTLS_DIR}"/rustls.tar.gz
   RUSTLS_COMMIT="$(jq -r '.git.sha1' ${RUSTLS_DIR}/.cargo_vcs_info.json)"
@@ -74,7 +102,9 @@ git clone https://github.com/rustls/rustls.git "${RUSTLS_DIR}"
 
 # Update rcgen to use the GitHub repository reference under test.
 pushd "${RUSTLS_RCGEN_DIR}"
-git checkout "${RUSTLS_RCGEN_COMMIT}"
+if [[ $latest_release == "1" ]]; then
+  git checkout "${RUSTLS_RCGEN_COMMIT}"
+fi
 rm Cargo.lock
 RCGEN_AWS_LC_RS_STRING="^aws-lc-rs = .*"
 RCGEN_AWS_LC_RS_PATH_STRING="aws-lc-rs = { path = \"${ROOT}/aws-lc-rs\", default-features = false, features = [\"aws-lc-sys\"] }"
@@ -87,17 +117,16 @@ popd &>/dev/null # "${RUSTLS_RCGEN_DIR}"
 
 # Update rustls-webpki to use the GitHub repository reference under test.
 pushd "${RUSTLS_WEBPKI_DIR}"
-git checkout "${RUSTLS_WEBPKI_COMMIT}"
+if [[ $latest_release == "1" ]]; then
+  git checkout "${RUSTLS_WEBPKI_COMMIT}"
+fi
 rm Cargo.lock
 WEBPKI_RCGEN_STRING="^rcgen = { .* }"
 WEBPKI_RCGEN_PATH_STRING="rcgen = { path = \"${RUSTLS_RCGEN_DIR}/rcgen\" , default-features = false, features = [\"aws_lc_rs\"] }"
 WEBPKI_AWS_LC_RS_STRING="^aws-lc-rs = { version.* }"
 WEBPKI_AWS_LC_RS_PATH_STRING="aws-lc-rs = { path = \"${ROOT}/aws-lc-rs\", optional = true, default-features = false, features = [\"aws-lc-sys\"] }"
-if [[ "$(uname)" == "Darwin" ]]; then
-	find ./ -type f  -name "Cargo.toml" | xargs sed -i '' -e "s|${WEBPKI_RCGEN_STRING}|${WEBPKI_RCGEN_PATH_STRING}|g" -e "s|${WEBPKI_AWS_LC_RS_STRING}|${WEBPKI_AWS_LC_RS_PATH_STRING}|g"
-else
-	find ./ -type f  -name "Cargo.toml" | xargs sed -i -e "s|${WEBPKI_RCGEN_STRING}|${WEBPKI_RCGEN_PATH_STRING}|g" -e "s|${WEBPKI_AWS_LC_RS_STRING}|${WEBPKI_AWS_LC_RS_PATH_STRING}|g"
-fi
+find ./ -type f  -name "Cargo.toml" | xargs perl -0777 -pi -e 's|'"${WEBPKI_RCGEN_STRING}"'|'"${WEBPKI_RCGEN_PATH_STRING}"'|g; s|'"${WEBPKI_AWS_LC_RS_STRING}"'|'"${WEBPKI_AWS_LC_RS_PATH_STRING}"'|g'
+
 # Trigger Cargo to generate the lock file
 cargo update
 cargo update "aws-lc-rs@${AWS_LC_RS_VERSION}"
@@ -116,14 +145,11 @@ RUSTLS_RCGEN_PATH_STRING="rcgen = { path = \"${RUSTLS_RCGEN_DIR}/rcgen\" , defau
 RUSTLS_AWS_LC_RS_STRING="^aws-lc-rs = { version.* }"
 RUSTLS_AWS_LC_RS_PATH_STRING="aws-lc-rs = { path = \"${ROOT}/aws-lc-rs\", default-features = false, features = [\"aws-lc-sys\"] }"
 RUSTLS_AWS_LC_RS_NON_OPTIONAL_PATH_STRING="aws-lc-rs = { path = \"${ROOT}/aws-lc-rs\", default-features = false, features = [\"unstable\", \"aws-lc-sys\"] }"
-RUSTLS_WEBPKI_STRING="^webpki = { package.* }"
+RUSTLS_WEBPKI_STRING="^webpki = { package = \"rustls-webpki\",.*}"
 RUSTLS_WEBPKI_PATH_STRING="webpki = { package = \"rustls-webpki\", path = \"${RUSTLS_WEBPKI_DIR}\", features = [\"alloc\"], default-features = false }"
 sed_replace ./rustls-post-quantum/Cargo.toml "s|${RUSTLS_AWS_LC_RS_STRING}|${RUSTLS_AWS_LC_RS_NON_OPTIONAL_PATH_STRING}|g"
-if [[ "$(uname)" == "Darwin" ]]; then
-	find ./ -type f  -name "Cargo.toml" | xargs sed -i '' -e "s|${RUSTLS_RCGEN_STRING}|${RUSTLS_RCGEN_PATH_STRING}|g" -e "s|${RUSTLS_AWS_LC_RS_STRING}|${RUSTLS_AWS_LC_RS_PATH_STRING}|g" -e "s|${RUSTLS_WEBPKI_STRING}|${RUSTLS_WEBPKI_PATH_STRING}|g"
-else
-	find ./ -type f  -name "Cargo.toml" | xargs sed -i -e "s|${RUSTLS_RCGEN_STRING}|${RUSTLS_RCGEN_PATH_STRING}|g" -e "s|${RUSTLS_AWS_LC_RS_STRING}|${RUSTLS_AWS_LC_RS_PATH_STRING}|g" -e "s|${RUSTLS_WEBPKI_STRING}|${RUSTLS_WEBPKI_PATH_STRING}|g"
-fi
+find ./ -type f  -name "Cargo.toml" | xargs perl -0777 -pi -e 's|'"${RUSTLS_RCGEN_STRING}"'|'"${RUSTLS_RCGEN_PATH_STRING}"'|g; s|'"${RUSTLS_AWS_LC_RS_STRING}"'|'"${RUSTLS_AWS_LC_RS_PATH_STRING}"'|g; s|'"${RUSTLS_WEBPKI_STRING}"'|'"${RUSTLS_WEBPKI_PATH_STRING}"'|g'
+
 # Trigger Cargo to generate the lock file
 cargo update
 cargo update "aws-lc-rs@${AWS_LC_RS_VERSION}"