Move metadata object generation for dylibs to the linker code

This deduplicates some code between codegen backends and may in the
future allow adding extra metadata that is only known at link time.

Author: bjorn3

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -11,15 +11,13 @@ use std::thread::JoinHandle;
 use cranelift_object::{ObjectBuilder, ObjectModule};
 use rustc_codegen_ssa::assert_module_sources::CguReuse;
 use rustc_codegen_ssa::back::link::ensure_removed;
-use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
 use rustc_codegen_ssa::base::determine_cgu_reuse;
 use rustc_codegen_ssa::{
     CodegenResults, CompiledModule, CrateInfo, ModuleKind, errors as ssa_errors,
 };
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::{IntoDynSyncSend, par_map};
-use rustc_metadata::EncodedMetadata;
 use rustc_metadata::fs::copy_to_stdout;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
@@ -61,7 +59,6 @@ impl<HCX> HashStable<HCX> for OngoingModuleCodegen {
 pub(crate) struct OngoingCodegen {
     modules: Vec<OngoingModuleCodegen>,
     allocator_module: Option<CompiledModule>,
-    metadata_module: Option<CompiledModule>,
     crate_info: CrateInfo,
     concurrency_limiter: ConcurrencyLimiter,
 }
@@ -133,7 +130,6 @@ impl OngoingCodegen {
         let codegen_results = CodegenResults {
             modules,
             allocator_module: self.allocator_module,
-            metadata_module: self.metadata_module,
             crate_info: self.crate_info,
         };
 
@@ -644,42 +640,6 @@ fn module_codegen(
     }))
 }
 
-fn emit_metadata_module(tcx: TyCtxt<'_>, metadata: &EncodedMetadata) -> CompiledModule {
-    use rustc_middle::mir::mono::CodegenUnitNameBuilder;
-
-    let _timer = tcx.sess.timer("write compressed metadata");
-
-    let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
-    let metadata_cgu_name = cgu_name_builder
-        .build_cgu_name(LOCAL_CRATE, ["crate"], Some("metadata"))
-        .as_str()
-        .to_string();
-
-    let tmp_file = tcx.output_filenames(()).temp_path_for_cgu(
-        OutputType::Metadata,
-        &metadata_cgu_name,
-        tcx.sess.invocation_temp.as_deref(),
-    );
-
-    let symbol_name = rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx);
-    let obj = create_compressed_metadata_file(tcx.sess, metadata, &symbol_name);
-
-    if let Err(err) = std::fs::write(&tmp_file, obj) {
-        tcx.dcx().fatal(format!("error writing metadata object file: {}", err));
-    }
-
-    CompiledModule {
-        name: metadata_cgu_name,
-        kind: ModuleKind::Metadata,
-        object: Some(tmp_file),
-        dwarf_object: None,
-        bytecode: None,
-        assembly: None,
-        llvm_ir: None,
-        links_from_incr_cache: Vec::new(),
-    }
-}
-
 fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option<CompiledModule> {
     let mut allocator_module = make_module(tcx.sess, "allocator_shim".to_string());
     let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module);
@@ -704,7 +664,7 @@ fn emit_allocator_module(tcx: TyCtxt<'_>) -> Option<CompiledModule> {
     }
 }
 
-pub(crate) fn run_aot(tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Box<OngoingCodegen> {
+pub(crate) fn run_aot(tcx: TyCtxt<'_>) -> Box<OngoingCodegen> {
     // FIXME handle `-Ctarget-cpu=native`
     let target_cpu = match tcx.sess.opts.cg.target_cpu {
         Some(ref name) => name,
@@ -720,7 +680,6 @@ pub(crate) fn run_aot(tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Bo
         return Box::new(OngoingCodegen {
             modules: vec![],
             allocator_module: None,
-            metadata_module: None,
             crate_info: CrateInfo::new(tcx, target_cpu),
             concurrency_limiter: ConcurrencyLimiter::new(0),
         });
@@ -780,12 +739,9 @@ pub(crate) fn run_aot(tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Bo
 
     let allocator_module = emit_allocator_module(tcx);
 
-    let metadata_module = metadata.map(|metadata| emit_metadata_module(tcx, metadata));
-
     Box::new(OngoingCodegen {
         modules,
         allocator_module,
-        metadata_module,
         crate_info: CrateInfo::new(tcx, target_cpu),
         concurrency_limiter: concurrency_limiter.0,
     })

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -46,7 +46,6 @@ use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::settings::{self, Configurable};
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::{CodegenResults, TargetConfig};
-use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_session::Session;
 use rustc_session::config::OutputFilenames;
@@ -238,7 +237,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
         println!("Cranelift version: {}", cranelift_codegen::VERSION);
     }
 
-    fn codegen_crate(&self, tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Box<dyn Any> {
+    fn codegen_crate(&self, tcx: TyCtxt<'_>) -> Box<dyn Any> {
         info!("codegen crate {}", tcx.crate_name(LOCAL_CRATE));
         let config = self.config.clone().unwrap_or_else(|| {
             BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
@@ -251,7 +250,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
             #[cfg(not(feature = "jit"))]
             tcx.dcx().fatal("jit support was disabled when compiling rustc_codegen_cranelift");
         } else {
-            driver::aot::run_aot(tcx, metadata)
+            driver::aot::run_aot(tcx)
         }
     }
 

compiler/rustc_codegen_gcc/src/lib.rs

@@ -48,7 +48,6 @@ extern crate rustc_index;
 #[cfg(feature = "master")]
 extern crate rustc_interface;
 extern crate rustc_macros;
-extern crate rustc_metadata;
 extern crate rustc_middle;
 extern crate rustc_session;
 extern crate rustc_span;
@@ -106,7 +105,6 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen, TargetCon
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::IntoDynSyncSend;
 use rustc_errors::DiagCtxtHandle;
-use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::TyCtxt;
 use rustc_middle::util::Providers;
@@ -230,9 +228,9 @@ impl CodegenBackend for GccCodegenBackend {
         providers.global_backend_features = |tcx, ()| gcc_util::global_gcc_features(tcx.sess, true)
     }
 
-    fn codegen_crate(&self, tcx: TyCtxt<'_>, metadata: Option<&EncodedMetadata>) -> Box<dyn Any> {
+    fn codegen_crate(&self, tcx: TyCtxt<'_>) -> Box<dyn Any> {
         let target_cpu = target_cpu(tcx.sess);
-        let res = codegen_crate(self.clone(), tcx, target_cpu.to_string(), metadata);
+        let res = codegen_crate(self.clone(), tcx, target_cpu.to_string());
 
         Box::new(res)
     }

compiler/rustc_codegen_llvm/src/lib.rs

@@ -341,16 +341,11 @@ impl CodegenBackend for LlvmCodegenBackend {
         target_config(sess)
     }
 
-    fn codegen_crate<'tcx>(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        metadata: Option<&EncodedMetadata>,
-    ) -> Box<dyn Any> {
+    fn codegen_crate<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Box<dyn Any> {
         Box::new(rustc_codegen_ssa::base::codegen_crate(
             LlvmCodegenBackend(()),
             tcx,
             crate::llvm_util::target_cpu(tcx.sess).to_string(),
-            metadata,
         ))
     }
 

compiler/rustc_codegen_ssa/messages.ftl

@@ -200,8 +200,6 @@ codegen_ssa_linking_failed = linking with `{$linker_path}` failed: {$exit_status
 codegen_ssa_malformed_cgu_name =
     found malformed codegen unit name `{$user_path}`. codegen units names must always start with the name of the crate (`{$crate_name}` in this case).
 
-codegen_ssa_metadata_object_file_write = error writing metadata object file: {$error}
-
 codegen_ssa_missing_cpp_build_tool_component = or a necessary component may be missing from the "C++ build tools" workload
 
 codegen_ssa_missing_features = add the missing features in a `target_feature` attribute

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -14,11 +14,12 @@ use object::read::macho::FatArch;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::memmap::Mmap;
 use rustc_fs_util::TempDirBuilder;
+use rustc_metadata::EncodedMetadata;
 use rustc_session::Session;
 use rustc_span::Symbol;
 use tracing::trace;
 
-use super::metadata::search_for_section;
+use super::metadata::{create_compressed_metadata_file, search_for_section};
 use crate::common;
 // Re-exporting for rustc_codegen_llvm::back::archive
 pub use crate::errors::{ArchiveBuildFailure, ExtractBundledLibsError, UnknownArchiveKind};
@@ -58,6 +59,15 @@ impl From<ImportLibraryItem> for COFFShortExport {
 pub trait ArchiveBuilderBuilder {
     fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a>;
 
+    fn create_dylib_metadata_wrapper(
+        &self,
+        sess: &Session,
+        metadata: &EncodedMetadata,
+        symbol_name: &str,
+    ) -> Vec<u8> {
+        create_compressed_metadata_file(sess, metadata, symbol_name)
+    }
+
     /// Creates a DLL Import Library <https://docs.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-creation#creating-an-import-library>.
     /// and returns the path on disk to that import library.
     /// This functions doesn't take `self` so that it can be called from

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -167,6 +167,7 @@ pub fn link_binary(
                         crate_type,
                         &out_filename,
                         &codegen_results,
+                        &metadata,
                         path.as_ref(),
                     );
                 }
@@ -230,11 +231,7 @@ pub fn link_binary(
         let remove_temps_from_module =
             |module: &CompiledModule| maybe_remove_temps_from_module(false, false, module);
 
-        // Otherwise, always remove the metadata and allocator module temporaries.
-        if let Some(ref metadata_module) = codegen_results.metadata_module {
-            remove_temps_from_module(metadata_module);
-        }
-
+        // Otherwise, always remove the allocator module temporaries.
         if let Some(ref allocator_module) = codegen_results.allocator_module {
             remove_temps_from_module(allocator_module);
         }
@@ -326,7 +323,7 @@ fn link_rlib<'a>(
         RlibFlavor::Normal => {
             let (metadata, metadata_position) =
                 create_wrapper_file(sess, ".rmeta".to_string(), metadata.stub_or_full());
-            let metadata = emit_wrapper_file(sess, &metadata, tmpdir, METADATA_FILENAME);
+            let metadata = emit_wrapper_file(sess, &metadata, tmpdir.as_ref(), METADATA_FILENAME);
             match metadata_position {
                 MetadataPosition::First => {
                     // Most of the time metadata in rlib files is wrapped in a "dummy" object
@@ -394,7 +391,7 @@ fn link_rlib<'a>(
             let src = read(path)
                 .unwrap_or_else(|e| sess.dcx().emit_fatal(errors::ReadFileError { message: e }));
             let (data, _) = create_wrapper_file(sess, ".bundled_lib".to_string(), &src);
-            let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
+            let wrapper_file = emit_wrapper_file(sess, &data, tmpdir.as_ref(), filename.as_str());
             packed_bundled_libs.push(wrapper_file);
         } else {
             let path = find_native_static_library(lib.name.as_str(), lib.verbatim, sess);
@@ -698,6 +695,7 @@ fn link_natively(
     crate_type: CrateType,
     out_filename: &Path,
     codegen_results: &CodegenResults,
+    metadata: &EncodedMetadata,
     tmpdir: &Path,
 ) {
     info!("preparing {:?} to {:?}", crate_type, out_filename);
@@ -722,6 +720,7 @@ fn link_natively(
         tmpdir,
         temp_filename,
         codegen_results,
+        metadata,
         self_contained_components,
     );
 
@@ -2098,17 +2097,25 @@ fn add_local_crate_allocator_objects(cmd: &mut dyn Linker, codegen_results: &Cod
 /// Add object files containing metadata for the current crate.
 fn add_local_crate_metadata_objects(
     cmd: &mut dyn Linker,
+    sess: &Session,
+    archive_builder_builder: &dyn ArchiveBuilderBuilder,
     crate_type: CrateType,
+    tmpdir: &Path,
     codegen_results: &CodegenResults,
+    metadata: &EncodedMetadata,
 ) {
     // When linking a dynamic library, we put the metadata into a section of the
     // executable. This metadata is in a separate object file from the main
-    // object file, so we link that in here.
-    if matches!(crate_type, CrateType::Dylib | CrateType::ProcMacro)
-        && let Some(m) = &codegen_results.metadata_module
-        && let Some(obj) = &m.object
-    {
-        cmd.add_object(obj);
+    // object file, so we create and link it in here.
+    if matches!(crate_type, CrateType::Dylib | CrateType::ProcMacro) {
+        let data = archive_builder_builder.create_dylib_metadata_wrapper(
+            sess,
+            &metadata,
+            &codegen_results.crate_info.metadata_symbol,
+        );
+        let obj = emit_wrapper_file(sess, &data, tmpdir, "rmeta.o");
+
+        cmd.add_object(&obj);
     }
 }
 
@@ -2198,6 +2205,7 @@ fn linker_with_args(
     tmpdir: &Path,
     out_filename: &Path,
     codegen_results: &CodegenResults,
+    metadata: &EncodedMetadata,
     self_contained_components: LinkSelfContainedComponents,
 ) -> Command {
     let self_contained_crt_objects = self_contained_components.is_crt_objects_enabled();
@@ -2272,7 +2280,15 @@ fn linker_with_args(
     // in this DAG so far because they can only depend on other native libraries
     // and such dependencies are also required to be specified.
     add_local_crate_regular_objects(cmd, codegen_results);
-    add_local_crate_metadata_objects(cmd, crate_type, codegen_results);
+    add_local_crate_metadata_objects(
+        cmd,
+        sess,
+        archive_builder_builder,
+        crate_type,
+        tmpdir,
+        codegen_results,
+        metadata,
+    );
     add_local_crate_allocator_objects(cmd, codegen_results);
 
     // Avoid linking to dynamic libraries unless they satisfy some undefined symbols