From 39a5d3f409b4e9b610129e02625b45124e1e55a0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 28 Jun 2016 13:31:30 -0700 Subject: [PATCH] rustbuild: Implement testing for Android This commit enhances the rustbuild support for testing Android to the same level of parity as the makefiles. This involved: * A new step to copy the standard library and other shared objects to the emulator. This is injected as a dependency of all test suites for Android. * Appropriate arguments are now passed through to compiletest to ensure that it can run tests. * When testing the standard library the test executables are probed for and shipped to the emulator to run for each test. * Fixing compilation of compiler-rt a bit All support added here is modeled after what's found in the makefiles, just translating one strategy to another. As an added bonus this commit adds support for the "check" step to automatically run tests for all targets, and the "check-target" step now runs all tests for a particular target, automatically filtering the tests if the target is detected as a cross-compile. Note that we don't (and probably won't) have a bot which is actually going to exercise any of this just yet, but all tests have passed locally for me at least. --- src/bootstrap/build/cc.rs | 1 + src/bootstrap/build/check.rs | 128 +++++++++++++++++++++++++++++++--- src/bootstrap/build/config.rs | 4 +- src/bootstrap/build/dist.rs | 4 +- src/bootstrap/build/mod.rs | 12 +++- src/bootstrap/build/native.rs | 8 ++- src/bootstrap/build/sanity.rs | 4 ++ src/bootstrap/build/step.rs | 118 +++++++++++++++++++++++-------- 8 files changed, 227 insertions(+), 52 deletions(-) diff --git a/src/bootstrap/build/cc.rs b/src/bootstrap/build/cc.rs index 7eb50b8b86dac..ff0941a97dce1 100644 --- a/src/bootstrap/build/cc.rs +++ b/src/bootstrap/build/cc.rs @@ -90,6 +90,7 @@ fn set_compiler(cfg: &mut gcc::Config, // compiler already takes into account the triple in question. t if t.contains("android") => { if let Some(ndk) = config.and_then(|c| c.ndk.as_ref()) { + let target = target.replace("armv7", "arm"); let compiler = format!("{}-{}", target, gnu_compiler); cfg.compiler(ndk.join("bin").join(compiler)); } diff --git a/src/bootstrap/build/check.rs b/src/bootstrap/build/check.rs index 154d9556fd7ba..0a096f8e4de41 100644 --- a/src/bootstrap/build/check.rs +++ b/src/bootstrap/build/check.rs @@ -23,6 +23,9 @@ use build_helper::output; use bootstrap::{dylib_path, dylib_path_var}; use build::{Build, Compiler, Mode}; +use build::util; + +const ADB_TEST_DIR: &'static str = "/data/tmp"; /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. /// @@ -88,6 +91,7 @@ pub fn compiletest(build: &Build, target: &str, mode: &str, suite: &str) { + println!("Check compiletest {} ({} -> {})", suite, compiler.host, target); let mut cmd = build.tool_cmd(compiler, "compiletest"); // compiletest currently has... a lot of arguments, so let's just pass all @@ -105,21 +109,23 @@ pub fn compiletest(build: &Build, cmd.arg("--host").arg(compiler.host); cmd.arg("--llvm-filecheck").arg(build.llvm_filecheck(&build.config.build)); - let mut flags = format!("-Crpath"); + let mut flags = vec!["-Crpath".to_string()]; if build.config.rust_optimize_tests { - flags.push_str(" -O"); + flags.push("-O".to_string()); } if build.config.rust_debuginfo_tests { - flags.push_str(" -g"); + flags.push("-g".to_string()); } - cmd.arg("--host-rustcflags").arg(&flags); - - let linkflag = format!("-Lnative={}", build.test_helpers_out(target).display()); - cmd.arg("--target-rustcflags").arg(format!("{} {}", flags, linkflag)); + let mut hostflags = build.rustc_flags(&compiler.host); + hostflags.extend(flags.clone()); + cmd.arg("--host-rustcflags").arg(hostflags.join(" ")); - // FIXME: needs android support - cmd.arg("--android-cross-path").arg(""); + let mut targetflags = build.rustc_flags(&target); + targetflags.extend(flags); + targetflags.push(format!("-Lnative={}", + build.test_helpers_out(target).display())); + cmd.arg("--target-rustcflags").arg(targetflags.join(" ")); // FIXME: CFG_PYTHON should probably be detected more robustly elsewhere let python_default = "python"; @@ -180,6 +186,16 @@ pub fn compiletest(build: &Build, } build.add_bootstrap_key(compiler, &mut cmd); + cmd.arg("--adb-path").arg("adb"); + cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); + if target.contains("android") { + // Assume that cc for this target comes from the android sysroot + cmd.arg("--android-cross-path") + .arg(build.cc(target).parent().unwrap().parent().unwrap()); + } else { + cmd.arg("--android-cross-path").arg(""); + } + build.run(&mut cmd); } @@ -302,7 +318,97 @@ pub fn krate(build: &Build, let mut dylib_path = dylib_path(); dylib_path.insert(0, build.sysroot_libdir(compiler, target)); cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - cargo.args(&build.flags.args); - build.run(&mut cargo); + if target.contains("android") { + build.run(cargo.arg("--no-run")); + krate_android(build, compiler, target, mode); + } else { + cargo.args(&build.flags.args); + build.run(&mut cargo); + } +} + +fn krate_android(build: &Build, + compiler: &Compiler, + target: &str, + mode: Mode) { + let mut tests = Vec::new(); + let out_dir = build.cargo_out(compiler, mode, target); + find_tests(&out_dir, target, &mut tests); + find_tests(&out_dir.join("deps"), target, &mut tests); + + for test in tests { + build.run(Command::new("adb").arg("push").arg(&test).arg(ADB_TEST_DIR)); + + let test_file_name = test.file_name().unwrap().to_string_lossy(); + let log = format!("{}/check-stage{}-T-{}-H-{}-{}.log", + ADB_TEST_DIR, + compiler.stage, + target, + compiler.host, + test_file_name); + let program = format!("(cd {dir}; \ + LD_LIBRARY_PATH=./{target} ./{test} \ + --logfile {log} \ + {args})", + dir = ADB_TEST_DIR, + target = target, + test = test_file_name, + log = log, + args = build.flags.args.join(" ")); + + let output = output(Command::new("adb").arg("shell").arg(&program)); + println!("{}", output); + build.run(Command::new("adb") + .arg("pull") + .arg(&log) + .arg(build.out.join("tmp"))); + build.run(Command::new("adb").arg("shell").arg("rm").arg(&log)); + if !output.contains("result: ok") { + panic!("some tests failed"); + } + } +} + +fn find_tests(dir: &Path, + target: &str, + dst: &mut Vec) { + for e in t!(dir.read_dir()).map(|e| t!(e)) { + let file_type = t!(e.file_type()); + if !file_type.is_file() { + continue + } + let filename = e.file_name().into_string().unwrap(); + if (target.contains("windows") && filename.ends_with(".exe")) || + (!target.contains("windows") && !filename.contains(".")) { + dst.push(e.path()); + } + } +} + +pub fn android_copy_libs(build: &Build, + compiler: &Compiler, + target: &str) { + println!("Android copy libs to emulator ({})", target); + build.run(Command::new("adb").arg("remount")); + build.run(Command::new("adb").args(&["shell", "rm", "-r", ADB_TEST_DIR])); + build.run(Command::new("adb").args(&["shell", "mkdir", ADB_TEST_DIR])); + build.run(Command::new("adb") + .arg("push") + .arg(build.src.join("src/etc/adb_run_wrapper.sh")) + .arg(ADB_TEST_DIR)); + + let target_dir = format!("{}/{}", ADB_TEST_DIR, target); + build.run(Command::new("adb").args(&["shell", "mkdir", &target_dir[..]])); + + for f in t!(build.sysroot_libdir(compiler, target).read_dir()) { + let f = t!(f); + let name = f.file_name().into_string().unwrap(); + if util::is_dylib(&name) { + build.run(Command::new("adb") + .arg("push") + .arg(f.path()) + .arg(&target_dir)); + } + } } diff --git a/src/bootstrap/build/config.rs b/src/bootstrap/build/config.rs index e155bf356a0d8..498196e9b6dfc 100644 --- a/src/bootstrap/build/config.rs +++ b/src/bootstrap/build/config.rs @@ -368,13 +368,13 @@ impl Config { target.ndk = Some(PathBuf::from(value)); } "CFG_I686_LINUX_ANDROID_NDK" if value.len() > 0 => { - let target = "i686-linux-androideabi".to_string(); + let target = "i686-linux-android".to_string(); let target = self.target_config.entry(target) .or_insert(Target::default()); target.ndk = Some(PathBuf::from(value)); } "CFG_AARCH64_LINUX_ANDROID_NDK" if value.len() > 0 => { - let target = "aarch64-linux-androideabi".to_string(); + let target = "aarch64-linux-android".to_string(); let target = self.target_config.entry(target) .or_insert(Target::default()); target.ndk = Some(PathBuf::from(value)); diff --git a/src/bootstrap/build/dist.rs b/src/bootstrap/build/dist.rs index 088e89b658d46..6eed7eaf206f4 100644 --- a/src/bootstrap/build/dist.rs +++ b/src/bootstrap/build/dist.rs @@ -135,7 +135,6 @@ pub fn rustc(build: &Build, stage: u32, host: &str) { // Prepare the overlay which is part of the tarball but won't actually be // installed - t!(fs::create_dir_all(&overlay)); let cp = |file: &str| { install(&build.src.join(file), &overlay, 0o644); }; @@ -199,7 +198,6 @@ pub fn rustc(build: &Build, stage: u32, host: &str) { // Copy runtime DLLs needed by the compiler if libdir != "bin" { - t!(fs::create_dir_all(image.join(libdir))); for entry in t!(src.join(libdir).read_dir()).map(|e| t!(e)) { let name = entry.file_name(); if let Some(s) = name.to_str() { @@ -221,7 +219,6 @@ pub fn rustc(build: &Build, stage: u32, host: &str) { let cp = |file: &str| { install(&build.src.join(file), &image.join("share/doc/rust"), 0o644); }; - t!(fs::create_dir_all(&image.join("share/doc/rust"))); cp("COPYRIGHT"); cp("LICENSE-APACHE"); cp("LICENSE-MIT"); @@ -289,6 +286,7 @@ pub fn std(build: &Build, compiler: &Compiler, target: &str) { fn install(src: &Path, dstdir: &Path, perms: u32) { let dst = dstdir.join(src.file_name().unwrap()); + t!(fs::create_dir_all(dstdir)); t!(fs::copy(src, &dst)); chmod(&dst, perms); } diff --git a/src/bootstrap/build/mod.rs b/src/bootstrap/build/mod.rs index dadb0ffa6c98d..195d1bc90c655 100644 --- a/src/bootstrap/build/mod.rs +++ b/src/bootstrap/build/mod.rs @@ -128,6 +128,7 @@ pub struct Build { /// /// These entries currently correspond to the various output directories of the /// build system, with each mod generating output in a different directory. +#[derive(Clone, Copy)] pub enum Mode { /// This cargo is going to build the standard library, placing output in the /// "stageN-std" directory. @@ -383,8 +384,7 @@ impl Build { "ui", "ui"); } CheckDebuginfo { compiler } => { - if target.target.contains("msvc") || - target.target.contains("android") { + if target.target.contains("msvc") { // nothing to do } else if target.target.contains("apple") { check::compiletest(self, &compiler, target.target, @@ -434,8 +434,14 @@ impl Build { target.target); } + AndroidCopyLibs { compiler } => { + check::android_copy_libs(self, &compiler, target.target); + } + + // pseudo-steps Dist { .. } | - Doc { .. } | // pseudo-steps + Doc { .. } | + CheckTarget { .. } | Check { .. } => {} } } diff --git a/src/bootstrap/build/native.rs b/src/bootstrap/build/native.rs index 1e677aa48b0f6..a01eb0d1a6bcb 100644 --- a/src/bootstrap/build/native.rs +++ b/src/bootstrap/build/native.rs @@ -49,6 +49,8 @@ pub fn llvm(build: &Build, target: &str) { return } + println!("Building LLVM for {}", target); + let _ = fs::remove_dir_all(&dst.join("build")); t!(fs::create_dir_all(&dst.join("build"))); let assertions = if build.config.llvm_assertions {"ON"} else {"OFF"}; @@ -165,8 +167,10 @@ pub fn compiler_rt(build: &Build, target: &str) { "arm" if target.contains("eabihf") => "armhf", _ => arch, }; - let target = format!("clang_rt.builtins-{}{}", builtins_arch, os_extra); - ("linux".to_string(), target.clone(), target) + let target = format!("clang_rt.builtins-{}", builtins_arch); + ("linux".to_string(), + target.clone(), + format!("{}{}", target, os_extra)) } else if target.contains("apple-darwin") { let builtins_arch = match arch { "i686" => "i386", diff --git a/src/bootstrap/build/sanity.rs b/src/bootstrap/build/sanity.rs index fd6cdc702cc3b..5eced00e13973 100644 --- a/src/bootstrap/build/sanity.rs +++ b/src/bootstrap/build/sanity.rs @@ -139,6 +139,10 @@ $ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake "); } } + + if target.contains("arm-linux-android") { + need_cmd("adb".as_ref()); + } } for host in build.flags.host.iter() { diff --git a/src/bootstrap/build/step.rs b/src/bootstrap/build/step.rs index 742fd8575bb80..7cbbd6740a265 100644 --- a/src/bootstrap/build/step.rs +++ b/src/bootstrap/build/step.rs @@ -102,6 +102,7 @@ macro_rules! targets { // Steps for running tests. The 'check' target is just a pseudo // target to depend on a bunch of others. (check, Check { stage: u32, compiler: Compiler<'a> }), + (check_target, CheckTarget { stage: u32, compiler: Compiler<'a> }), (check_linkcheck, CheckLinkcheck { stage: u32 }), (check_cargotest, CheckCargoTest { stage: u32 }), (check_tidy, CheckTidy { stage: u32 }), @@ -138,6 +139,9 @@ macro_rules! targets { (dist_mingw, DistMingw { _dummy: () }), (dist_rustc, DistRustc { stage: u32 }), (dist_std, DistStd { compiler: Compiler<'a> }), + + // Misc targets + (android_copy_libs, AndroidCopyLibs { compiler: Compiler<'a> }), } } } @@ -382,37 +386,80 @@ impl<'a> Step<'a> { self.doc_error_index(stage)] } Source::Check { stage, compiler } => { - vec![ + // Check is just a pseudo step which means check all targets, + // so just depend on checking all targets. + build.config.target.iter().map(|t| { + self.target(t).check_target(stage, compiler) + }).collect() + } + Source::CheckTarget { stage, compiler } => { + // CheckTarget here means run all possible test suites for this + // target. Most of the time, however, we can't actually run + // anything if we're not the build triple as we could be cross + // compiling. + // + // As a result, the base set of targets here is quite stripped + // down from the standard set of targets. These suites have + // their own internal logic to run in cross-compiled situations + // if they'll run at all. For example compiletest knows that + // when testing Android targets we ship artifacts to the + // emulator. + // + // When in doubt the rule of thumb for adding to this list is + // "should this test suite run on the android bot?" + let mut base = vec![ self.check_rpass(compiler), - self.check_rpass_full(compiler), self.check_rfail(compiler), - self.check_rfail_full(compiler), - self.check_cfail(compiler), - self.check_cfail_full(compiler), - self.check_pfail(compiler), - self.check_incremental(compiler), - self.check_ui(compiler), self.check_crate_std(compiler), self.check_crate_test(compiler), - self.check_crate_rustc(compiler), - self.check_codegen(compiler), - self.check_codegen_units(compiler), self.check_debuginfo(compiler), - self.check_rustdoc(compiler), - self.check_pretty(compiler), - self.check_pretty_rpass(compiler), - self.check_pretty_rpass_full(compiler), - self.check_pretty_rfail(compiler), - self.check_pretty_rfail_full(compiler), - self.check_pretty_rpass_valgrind(compiler), - self.check_rpass_valgrind(compiler), - self.check_error_index(compiler), - self.check_docs(compiler), - self.check_rmake(compiler), - self.check_linkcheck(stage), - self.check_tidy(stage), self.dist(stage), - ] + ]; + + // If we're testing the build triple, then we know we can + // actually run binaries and such, so we run all possible tests + // that we know about. + if self.target == build.config.build { + base.extend(vec![ + // docs-related + self.check_docs(compiler), + self.check_error_index(compiler), + self.check_rustdoc(compiler), + + // UI-related + self.check_cfail(compiler), + self.check_pfail(compiler), + self.check_ui(compiler), + + // codegen-related + self.check_incremental(compiler), + self.check_codegen(compiler), + self.check_codegen_units(compiler), + + // misc compiletest-test suites + self.check_rpass_full(compiler), + self.check_rfail_full(compiler), + self.check_cfail_full(compiler), + self.check_pretty_rpass_full(compiler), + self.check_pretty_rfail_full(compiler), + self.check_rpass_valgrind(compiler), + self.check_rmake(compiler), + + // crates + self.check_crate_rustc(compiler), + + // pretty + self.check_pretty(compiler), + self.check_pretty_rpass(compiler), + self.check_pretty_rfail(compiler), + self.check_pretty_rpass_valgrind(compiler), + + // misc + self.check_linkcheck(stage), + self.check_tidy(stage), + ]); + } + return base } Source::CheckLinkcheck { stage } => { vec![self.tool_linkchecker(stage), self.doc(stage)] @@ -437,16 +484,20 @@ impl<'a> Step<'a> { Source::CheckCFail { compiler } | Source::CheckRPassValgrind { compiler } | Source::CheckRPass { compiler } => { - vec![ + let mut base = vec![ self.libtest(compiler), - self.tool_compiletest(compiler.stage), + self.target(compiler.host).tool_compiletest(compiler.stage), self.test_helpers(()), - ] + ]; + if self.target.contains("android") { + base.push(self.android_copy_libs(compiler)); + } + base } Source::CheckDebuginfo { compiler } => { vec![ self.libtest(compiler), - self.tool_compiletest(compiler.stage), + self.target(compiler.host).tool_compiletest(compiler.stage), self.test_helpers(()), self.debugger_scripts(compiler.stage), ] @@ -459,13 +510,14 @@ impl<'a> Step<'a> { Source::CheckPrettyRPassValgrind { compiler } | Source::CheckRMake { compiler } => { vec![self.librustc(compiler), - self.tool_compiletest(compiler.stage)] + self.target(compiler.host).tool_compiletest(compiler.stage)] } Source::CheckDocs { compiler } => { vec![self.libstd(compiler)] } Source::CheckErrorIndex { compiler } => { - vec![self.libstd(compiler), self.tool_error_index(compiler.stage)] + vec![self.libstd(compiler), + self.target(compiler.host).tool_error_index(compiler.stage)] } Source::CheckCrateStd { compiler } => { vec![self.libtest(compiler)] @@ -529,6 +581,10 @@ impl<'a> Step<'a> { } return base } + + Source::AndroidCopyLibs { compiler } => { + vec![self.libtest(compiler)] + } } } }