new non-determinism introduced recently

[steven807]

PR 141769 looks like it introduced non-determinism in the generation of dylibs -- it is looking like the tmpdir is being introduced into the file -- perhaps when creating the .rustc section here:

rust/compiler/rustc_codegen_ssa/src/back/metadata.rs

Line 571 in 3129d37

pub fn create_compressed_metadata_file(

We are seeing it in determinism-validation builds in Fuchsia.

Along with .so files being different, we are seeing .so.map files differing, with distinct tmp paths, e.g.:

            f91ef            f91ef      431     1         .../rust/linux-x64/lib/rustlib/x86_64-unknown-fuchsia/lib/libcompiler_builtins-84c11efc675a4b65.rlib(compiler_builtins-84c11efc675a4b65.compiler_builtins.ac0a882909fb5ff0-cgu.018.rcgu.o):(.debug_line)
            f9620            f9620      30d     1         obj/zircon/system/ulib/sync/libsync.a(libsync.mutex.c.o):(.debug_line)
                0                0   1cf28a     1 .rustc
-               0                0   1cf28a     1         /tmp/rustcli35e4/rmeta.o:(.rustc)
+               0                0   1cf28a     1         /tmp/rustcvBrdn3/rmeta.o:(.rustc)
                0                0   1cf28a     1                 rust_metadata_vfs_3772e3b421ca7a16
                0                0      187     1 .debug_loclists
                0                0      22b     1         obj/zircon/system/ulib/sync/libsync.a

Suggested label: A-reproducibility

[workingjubilee]

cc @bjorn3 looks like we need to add a few .sort()s

[steven807]

I added a bit more information to the bug -- it's not just the ordering (that sort()s would fix) -- there are actual tmp paths being embedded in the .so.map files

[fangism]

https://g-issues.fuchsia.dev/issues/427157992#comment2 shows that it is the basename of the temporary object file that is different across runs and leaking into the built outputs, akin to mktemp -t rustcXXXXXX.o.

[workingjubilee]

@fangism I can't see your bug tracker. It's private.

[bjorn3]

This uses the same logic for writing the file to a temporary location as symbols.o. I guess having a non-deterministic location is harmless in that case as it doesn't contain any functions or data that get linked into the final binary. I could try going back to tcx.output_filenames(()).temp_path_for_cgu().

[bjorn3]

By the way what exactly is this .so.map file? What do you use it for? And how exactly is the output .so different? With incr comp enabled we do use random filenames for object files even before my PR ever since #139453.

[workingjubilee]

yes, it would be nice if we had an upstream version of your repro test, ideally complete with comments on why.

[bjorn3]

WIP patch which doesn't handle removing the metadata object after linking

diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 4a2425967e4..772dafeb2cc 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -21,6 +21,7 @@
 use rustc_fs_util::{TempDirBuilder, fix_windows_verbatim_for_gcc, try_canonicalize};
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_macros::LintDiagnostic;
+use rustc_metadata::errors::FailedWriteError;
 use rustc_metadata::fs::{METADATA_FILENAME, copy_to_stdout, emit_wrapper_file};
 use rustc_metadata::{
     EncodedMetadata, NativeLibSearchFallback, find_native_static_library,
@@ -31,6 +32,7 @@
 use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
+use rustc_middle::mir::mono::CodegenUnit;
 use rustc_session::config::{
     self, CFGuard, CrateType, DebugInfo, LinkerFeaturesCli, OutFileName, OutputFilenames,
     OutputType, PrintKind, SplitDwarfKind, Strip,
@@ -151,6 +153,7 @@ pub fn link_binary(
                         &out_filename,
                         &codegen_results,
                         &metadata,
+                        outputs,
                         path.as_ref(),
                     );
                 }
@@ -677,6 +680,7 @@ fn link_natively(
     out_filename: &Path,
     codegen_results: &CodegenResults,
     metadata: &EncodedMetadata,
+    outputs: &OutputFilenames,
     tmpdir: &Path,
 ) {
     info!("preparing {:?} to {:?}", crate_type, out_filename);
@@ -702,6 +706,7 @@ fn link_natively(
         temp_filename,
         codegen_results,
         metadata,
+        outputs,
         self_contained_components,
     );
 
@@ -2082,9 +2087,9 @@ fn add_local_crate_metadata_objects(
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
     crate_type: CrateType,
-    tmpdir: &Path,
     codegen_results: &CodegenResults,
     metadata: &EncodedMetadata,
+    outputs: &OutputFilenames,
 ) {
     // When linking a dynamic library, we put the metadata into a section of the
     // executable. This metadata is in a separate object file from the main
@@ -2095,9 +2100,30 @@ fn add_local_crate_metadata_objects(
             &metadata,
             &codegen_results.crate_info.metadata_symbol,
         );
-        let obj = emit_wrapper_file(sess, &data, tmpdir, "rmeta.o");
 
-        cmd.add_object(&obj);
+        let stable_crate_id = 0; //tcx.stable_crate_id(LOCAL_CRATE);
+        let cgu_name = format!(
+            "{}.{:08x}-crate.metadata",
+            codegen_results.crate_info.crate_name[&LOCAL_CRATE], stable_crate_id
+        );
+        let metadata_cgu_name = if sess.opts.unstable_opts.human_readable_cgu_names {
+            CodegenUnit::shorten_name(&cgu_name).to_string()
+        } else {
+            CodegenUnit::mangle_name(&cgu_name).to_string()
+        };
+        let filename = outputs.temp_path_for_cgu(
+            OutputType::Metadata,
+            &metadata_cgu_name,
+            sess.invocation_temp.as_deref(),
+        );
+
+        if let Err(err) = fs::write(&filename, data) {
+            sess.dcx().emit_fatal(FailedWriteError { filename, err });
+        }
+
+        // FIXME remove file after linking
+
+        cmd.add_object(&filename);
     }
 }
 
@@ -2188,6 +2214,7 @@ fn linker_with_args(
     out_filename: &Path,
     codegen_results: &CodegenResults,
     metadata: &EncodedMetadata,
+    outputs: &OutputFilenames,
     self_contained_components: LinkSelfContainedComponents,
 ) -> Command {
     let self_contained_crt_objects = self_contained_components.is_crt_objects_enabled();
@@ -2267,9 +2294,9 @@ fn linker_with_args(
         sess,
         archive_builder_builder,
         crate_type,
-        tmpdir,
         codegen_results,
         metadata,
+        outputs,
     );
     add_local_crate_allocator_objects(cmd, codegen_results);