diff --git a/src/build_manifest.rs b/src/build_manifest.rs index 8776071..421d8c7 100644 --- a/src/build_manifest.rs +++ b/src/build_manifest.rs @@ -1,13 +1,14 @@ -use crate::{config::Channel, Context}; +use crate::Context; use anyhow::{Context as _, Error}; use std::{ + collections::HashSet, fs::File, io::BufReader, path::{Path, PathBuf}, process::Command, }; use tar::Archive; -use tempfile::NamedTempFile; +use tempfile::{NamedTempFile, TempDir}; use xz2::read::XzDecoder; pub(crate) struct BuildManifest<'a> { @@ -19,10 +20,7 @@ pub(crate) struct BuildManifest<'a> { impl<'a> BuildManifest<'a> { pub(crate) fn new(builder: &'a Context) -> Self { // Precalculate paths used later. - let release = match builder.config.channel { - Channel::Stable => builder.current_version.clone().unwrap(), - channel => channel.to_string(), - }; + let release = builder.config.channel.release_name(builder); let tarball_name = format!("build-manifest-{}-{}", release, crate::TARGET); let tarball_path = builder.dl_dir().join(format!("{}.tar.xz", tarball_name)); @@ -37,12 +35,15 @@ impl<'a> BuildManifest<'a> { self.tarball_path.is_file() } - pub(crate) fn run(&self) -> Result<(), Error> { + pub(crate) fn run(&self) -> Result { let config = &self.builder.config; let bin = self .extract() .context("failed to extract build-manifest from the tarball")?; + let metadata_dir = TempDir::new()?; + let shipped_files_path = metadata_dir.path().join("shipped-files.txt"); + println!("running build-manifest..."); let upload_addr = format!("{}/{}", config.upload_addr, config.upload_dir); // build-manifest @@ -52,11 +53,12 @@ impl<'a> BuildManifest<'a> { .arg(&self.builder.date) .arg(upload_addr) .arg(config.channel.to_string()) + .env("BUILD_MANIFEST_SHIPPED_FILES_PATH", &shipped_files_path) .status() .context("failed to execute build-manifest")?; if status.success() { - Ok(()) + Execution::new(&shipped_files_path) } else { anyhow::bail!("build-manifest failed with status {:?}", status); } @@ -82,3 +84,27 @@ impl<'a> BuildManifest<'a> { Ok(bin) } } + +pub(crate) struct Execution { + pub(crate) shipped_files: Option>, +} + +impl Execution { + fn new(shipped_files_path: &Path) -> Result { + // Once https://github.com/rust-lang/rust/pull/78196 reaches stable we can assume the + // "shipped files" file is always generated, and we can remove the Option<_>. + let shipped_files = if shipped_files_path.is_file() { + Some( + std::fs::read_to_string(shipped_files_path)? + .lines() + .filter(|line| !line.trim().is_empty()) + .map(PathBuf::from) + .collect(), + ) + } else { + None + }; + + Ok(Execution { shipped_files }) + } +} diff --git a/src/config.rs b/src/config.rs index cc873be..b37ef8b 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,5 @@ -use anyhow::{Context, Error}; +use crate::Context; +use anyhow::{Context as _, Error}; use std::env::VarError; use std::str::FromStr; @@ -11,6 +12,16 @@ pub(crate) enum Channel { Nightly, } +impl Channel { + pub(crate) fn release_name(&self, ctx: &Context) -> String { + if *self == Channel::Stable { + ctx.current_version.clone().unwrap() + } else { + self.to_string() + } + } +} + impl FromStr for Channel { type Err = Error; @@ -70,6 +81,9 @@ pub(crate) struct Config { /// Whether to allow multiple releases on the same channel in the same day or not. pub(crate) allow_multiple_today: bool, + /// Whether to allow the work-in-progress pruning code for this release. + pub(crate) wip_prune_unused_files: bool, + /// The compression level to use when recompressing tarballs with gzip. pub(crate) gzip_compression_level: u32, /// Custom sha of the commit to release, instead of the latest commit in the channel's branch. @@ -105,6 +119,7 @@ impl Config { upload_addr: require_env("UPLOAD_ADDR")?, upload_bucket: require_env("UPLOAD_BUCKET")?, upload_dir: require_env("UPLOAD_DIR")?, + wip_prune_unused_files: bool_env("WIP_PRUNE_UNUSED_FILES")?, }) } } diff --git a/src/main.rs b/src/main.rs index 5ed320e..28a0827 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,12 +2,12 @@ mod build_manifest; mod config; mod sign; -use std::env; use std::fs::{self, File, OpenOptions}; use std::io::{self, Read}; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::Instant; +use std::{collections::HashSet, env}; use anyhow::Error; use build_manifest::BuildManifest; @@ -22,6 +22,14 @@ use crate::config::{Channel, Config}; const TARGET: &str = env!("TARGET"); +/// List of files that should not be pruned even if they're not referenced in the manifest. +const AVOID_PRUNING_UNUSED_FILES: &[&str] = &[ + // Source code tarballs, which are not distributed through rustup and thus are mistakenly + // marked as "unused" by build-manifest. + "rustc-{release}-src.tar.gz", + "rustc-{release}-src.tar.xz", +]; + struct Context { work: PathBuf, handle: Easy, @@ -177,7 +185,14 @@ impl Context { let build_manifest = BuildManifest::new(self); if build_manifest.exists() { // Generate the channel manifest - build_manifest.run()?; + let execution = build_manifest.run()?; + + if self.config.wip_prune_unused_files { + // Removes files that we are not shipping from the files we're about to upload. + if let Some(shipped_files) = &execution.shipped_files { + self.prune_unused_files(&shipped_files)?; + } + } // Generate checksums and sign all the files we're about to ship. let signer = Signer::new(&self.config)?; @@ -459,6 +474,28 @@ upload-addr = \"{}/{}\" Ok(()) } + fn prune_unused_files(&self, shipped_files: &HashSet) -> Result<(), Error> { + let release = self.config.channel.release_name(self); + let allowed_files = AVOID_PRUNING_UNUSED_FILES + .iter() + .map(|pattern| pattern.replace("{release}", &release).into()) + .collect::>(); + + for entry in std::fs::read_dir(self.dl_dir())? { + let entry = entry?; + + if let Some(name) = entry.path().file_name() { + let name = Path::new(name); + if !allowed_files.contains(name) && !shipped_files.contains(name) { + std::fs::remove_file(entry.path())?; + println!("pruned unused file {}", name.display()); + } + } + } + + Ok(()) + } + fn publish_archive(&mut self) -> Result<(), Error> { let bucket = &self.config.upload_bucket; let dir = &self.config.upload_dir;