Use llvm-strip

Author: charliermarsh

.github/workflows/release.yml

@@ -31,10 +31,6 @@ jobs:
 
       - uses: extractions/setup-just@v2
 
-      # Install `llvm-strip`.
-      - name: Install llvm-strip
-        run: sudo apt-get install -y llvm
-
       # Perform a release in dry-run mode.
       - run: just release-dry-run ${{ secrets.GITHUB_TOKEN }} ${{ github.event.inputs.sha }} ${{ github.event.inputs.tag }}
         if: ${{ github.event.inputs.dry-run == 'true' }}

.gitignore

@@ -4,4 +4,5 @@ docs/_build/
 dist/
 target/
 venv/
+llvm/
 __pycache__/

Cargo.lock

@@ -24,7 +24,7 @@ octocrab = { version = "0.34.1", features = ["rustls", "stream"] }
 once_cell = "1.19.0"
 pdb = "0.8.0"
 rayon = "1.8.1"
-reqwest = { version = "0.11.24", features = ["rustls"] }
+reqwest = { version = "0.11.24", features = ["rustls", "stream"] }
 scroll = "0.12.0"
 semver = "1.0.22"
 serde = { version = "1.0.197", features = ["derive"] }

Cargo.toml

@@ -175,18 +175,21 @@
         "sha256": "04cb77c660f09df017a57738ae9635ef23a506024789f2f18da1304b45af2023",
         "version": "14.0.3+20220508",
     },
+    # Remember to update LLVM_URL in src/release.rs whenever upgrading.
     "llvm-17-x86_64-linux": {
         "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-gnu_only-x86_64-unknown-linux-gnu.tar.zst",
         "size": 229404408,
         "sha256": "fc5e9092a8915dde438c3b491c5e6321594de541245b619f391edba719e4bd4f",
         "version": "17.0.6+20240222",
     },
+    # Remember to update LLVM_URL in src/release.rs whenever upgrading.
     "llvm-aarch64-macos": {
         "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-aarch64-apple-darwin.tar.zst",
         "size": 131303875,
         "sha256": "e1acf616780f32787b37b5534cb342c0e1e4a56b80cb9283f537fd8c9976e0d1",
         "version": "17.0.6+20240222",
     },
+    # Remember to update LLVM_URL in src/release.rs whenever upgrading.
     "llvm-x86_64-macos": {
         "url": "https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-x86_64-apple-darwin.tar.zst",
         "size": 131306317,

pythonbuild/downloads.py

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use crate::release::produce_install_only_stripped;
+use crate::release::{bootstrap_llvm, produce_install_only_stripped};
 use {
     crate::release::{produce_install_only, RELEASE_TRIPLES},
     anyhow::{anyhow, Result},
@@ -257,6 +257,8 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<()
         }
     }
 
+    let llvm_dir = bootstrap_llvm().await?;
+
     install_paths
         .par_iter()
         .try_for_each(|path| -> Result<()> {
@@ -287,7 +289,7 @@ pub async fn command_fetch_release_distributions(args: &ArgMatches) -> Result<()
                     .to_string_lossy()
             );
 
-            let dest_path = produce_install_only_stripped(&dest_path)?;
+            let dest_path = produce_install_only_stripped(&dest_path, &llvm_dir)?;
 
             println!(
                 "releasing {}",

src/github.rs

@@ -193,8 +193,13 @@ fn main_impl() -> Result<()> {
             Ok(())
         }
         Some(("convert-install-only-stripped", args)) => {
+            let llvm_dir = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .unwrap()
+                .block_on(release::bootstrap_llvm())?;
             for path in args.get_many::<PathBuf>("path").unwrap() {
-                let dest_path = release::produce_install_only_stripped(path)?;
+                let dest_path = release::produce_install_only_stripped(path, &llvm_dir)?;
                 println!("wrote {}", dest_path.display());
             }
 

src/main.rs

@@ -3,11 +3,11 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 use anyhow::Context;
-use object::elf::{FileHeader32, FileHeader64};
-use object::macho::{MachHeader32, MachHeader64};
-use object::read::pe::{PeFile32, PeFile64};
+use futures::StreamExt;
+
 use object::FileKind;
 use std::process::{Command, Stdio};
+use url::Url;
 use {
     crate::json::parse_python_json,
     anyhow::{anyhow, Result},
@@ -286,8 +286,8 @@ pub fn convert_to_install_only<W: Write>(reader: impl BufRead, writer: W) -> Res
 }
 
 /// Run `llvm-strip` over the given data, returning the stripped data.
-fn llvm_strip(data: &[u8]) -> Result<Vec<u8>> {
-    let mut command = Command::new("/opt/homebrew/opt/llvm/bin/llvm-strip")
+fn llvm_strip(data: &[u8], llvm_dir: &Path) -> Result<Vec<u8>> {
+    let mut command = Command::new(llvm_dir.join("bin/llvm-strip"))
         .arg("--strip-debug")
         .arg("-")
         .stdin(Stdio::piped())
@@ -313,7 +313,11 @@ fn llvm_strip(data: &[u8]) -> Result<Vec<u8>> {
 }
 
 /// Given an install-only .tar.gz archive, strip the underlying build.
-pub fn convert_to_stripped<W: Write>(reader: impl BufRead, writer: W) -> Result<W> {
+pub fn convert_to_stripped<W: Write>(
+    reader: impl BufRead,
+    writer: W,
+    llvm_dir: &Path,
+) -> Result<W> {
     let dctx = flate2::read::GzDecoder::new(reader);
 
     let mut tar_in = tar::Archive::new(dctx);
@@ -331,18 +335,12 @@ pub fn convert_to_stripped<W: Write>(reader: impl BufRead, writer: W) -> Result<
         let path = entry.path()?;
 
         // Drop PDB files.
-        match pdb::PDB::open(std::io::Cursor::new(&data)) {
-            Ok(_) => {
-                println!("removed PDB file: {}", path.display());
-                continue;
-            }
-            Err(err) => {
-                if path.extension().is_some_and(|ext| ext == "pdb") {
-                    println!(
-                        "file with `.pdb` extension ({}) failed to parse as PDB :{err}",
-                        path.display()
-                    );
-                }
+        if let Err(err) = pdb::PDB::open(std::io::Cursor::new(&data)) {
+            if path.extension().is_some_and(|ext| ext == "pdb") {
+                println!(
+                    "file with `.pdb` extension ({}) failed to parse as PDB :{err}",
+                    path.display()
+                );
             }
         }
 
@@ -359,21 +357,11 @@ pub fn convert_to_stripped<W: Write>(reader: impl BufRead, writer: W) -> Result<
                 | FileKind::Pe32
                 | FileKind::Pe64)
         ) {
-            let size_before = data.len();
-
-            let data =
-                llvm_strip(&data).with_context(|| format!("failed to strip {}", path.display()))?;
-
-            let size_after = data.len();
-
-            println!(
-                "stripped {} from {size_before} to {size_after} bytes",
-                path.display()
-            );
+            data = llvm_strip(&data, llvm_dir)
+                .with_context(|| format!("failed to strip {}", path.display()))?;
         }
 
         let header = entry.header().clone();
-
         builder.append(&header, std::io::Cursor::new(data))?;
     }
 
@@ -409,11 +397,24 @@ pub fn produce_install_only(tar_zst_path: &Path) -> Result<PathBuf> {
     Ok(dest_path)
 }
 
-pub fn produce_install_only_stripped(tar_gz_path: &Path) -> Result<PathBuf> {
+pub fn produce_install_only_stripped(tar_gz_path: &Path, llvm_dir: &Path) -> Result<PathBuf> {
     let buf = std::fs::read(tar_gz_path)?;
 
-    let gz_data =
-        convert_to_stripped(std::io::Cursor::new(buf), std::io::Cursor::new(vec![]))?.into_inner();
+    let size_before = buf.len();
+
+    let gz_data = convert_to_stripped(
+        std::io::Cursor::new(buf),
+        std::io::Cursor::new(vec![]),
+        llvm_dir,
+    )?
+    .into_inner();
+
+    let size_after = gz_data.len();
+
+    println!(
+        "stripped {} from {size_before} to {size_after} bytes",
+        tar_gz_path.display()
+    );
 
     let filename = tar_gz_path
         .file_name()
@@ -436,3 +437,72 @@ pub fn produce_install_only_stripped(tar_gz_path: &Path) -> Result<PathBuf> {
 
     Ok(dest_path)
 }
+
+/// URL from which to download LLVM.
+///
+/// To be kept in sync with `pythonbuild/downloads.py`.
+static LLVM_URL: Lazy<Url> = Lazy::new(|| {
+    if cfg!(target_os = "macos") {
+        if std::env::consts::ARCH == "aarch64" {
+            Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-aarch64-apple-darwin.tar.zst").unwrap()
+        } else if std::env::consts::ARCH == "x86_64" {
+            Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-x86_64-apple-darwin.tar.zst").unwrap()
+        } else {
+            panic!("unsupported macOS architecture");
+        }
+    } else if cfg!(target_os = "linux") {
+        Url::parse("https://github.com/indygreg/toolchain-tools/releases/download/toolchain-bootstrap%2F20240222/llvm-17.0.6+20240222-gnu_only-x86_64-unknown-linux-gnu.tar.zst").unwrap()
+    } else {
+        panic!("unsupported platform");
+    }
+});
+
+/// Bootstrap `llvm` for the current platform.
+///
+/// Returns the path to the top-level `llvm` directory.
+pub async fn bootstrap_llvm() -> Result<PathBuf> {
+    let url = &*LLVM_URL;
+    let filename = url.path_segments().unwrap().last().unwrap();
+
+    let llvm_dir = PathBuf::from("llvm");
+
+    // If `llvm` is already available with the target version, return it.
+    if llvm_dir.join(filename).exists() {
+        return Ok(llvm_dir.join("llvm"));
+    }
+
+    println!("Downloading LLVM tarball from: {url}");
+
+    // Create a temporary directory to download and extract the LLVM tarball.
+    let temp_dir = tempfile::TempDir::new()?;
+
+    // Download the tarball.
+    let tarball_path = temp_dir
+        .path()
+        .join(url.path_segments().unwrap().last().unwrap());
+    let mut tarball_file = tokio::fs::File::create(&tarball_path).await?;
+    let mut bytes_stream = reqwest::Client::new()
+        .get(url.clone())
+        .send()
+        .await?
+        .bytes_stream();
+    while let Some(chunk) = bytes_stream.next().await {
+        tokio::io::copy(&mut chunk?.as_ref(), &mut tarball_file).await?;
+    }
+
+    // Decompress the tarball.
+    let tarball = std::fs::File::open(&tarball_path)?;
+    let tar = zstd::stream::Decoder::new(std::io::BufReader::new(tarball))?;
+    let mut archive = tar::Archive::new(tar);
+    archive.unpack(temp_dir.path())?;
+
+    // Persist the directory.
+    match tokio::fs::remove_dir_all(&llvm_dir).await {
+        Ok(_) => {}
+        Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
+        Err(err) => return Err(err).context("failed to remove existing llvm directory"),
+    }
+    tokio::fs::rename(temp_dir.into_path(), &llvm_dir).await?;
+
+    Ok(llvm_dir.join("llvm"))
+}