use rustc_serialize::opaque::FileEncoder

Author: m-ysk

compiler/rustc_metadata/src/fs.rs

@@ -55,27 +55,30 @@ pub fn encode_and_write_metadata(
         .max()
         .unwrap_or(MetadataKind::None);
 
+    let crate_name = tcx.crate_name(LOCAL_CRATE);
+    let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs);
+    // To avoid races with another rustc process scanning the output directory,
+    // we need to write the file somewhere else and atomically move it to its
+    // final destination, with an `fs::rename` call. In order for the rename to
+    // always succeed, the temporary file needs to be on the same filesystem,
+    // which is why we create it inside the output directory specifically.
+    let metadata_tmpdir = TempFileBuilder::new()
+        .prefix("rmeta")
+        .tempdir_in(out_filename.parent().unwrap_or_else(|| Path::new("")))
+        .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err)));
+    let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps);
+    let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME);
     let metadata = match metadata_kind {
         MetadataKind::None => EncodedMetadata::new(),
-        MetadataKind::Uncompressed | MetadataKind::Compressed => encode_metadata(tcx),
+        MetadataKind::Uncompressed | MetadataKind::Compressed => {
+            encode_metadata(tcx, metadata_filename)
+        }
     };
 
     let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata");
 
     let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata);
     if need_metadata_file {
-        let crate_name = tcx.crate_name(LOCAL_CRATE);
-        let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs);
-        // To avoid races with another rustc process scanning the output directory,
-        // we need to write the file somewhere else and atomically move it to its
-        // final destination, with an `fs::rename` call. In order for the rename to
-        // always succeed, the temporary file needs to be on the same filesystem,
-        // which is why we create it inside the output directory specifically.
-        let metadata_tmpdir = TempFileBuilder::new()
-            .prefix("rmeta")
-            .tempdir_in(out_filename.parent().unwrap())
-            .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err)));
-        let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps);
         let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir);
         if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) {
             tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -27,8 +27,7 @@ use rustc_middle::ty::codec::TyEncoder;
 use rustc_middle::ty::fast_reject::{self, SimplifiedType, TreatParams};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, SymbolName, Ty, TyCtxt};
-use rustc_serialize::opaque::MemEncoder;
-use rustc_serialize::{Encodable, Encoder};
+use rustc_serialize::{opaque, Encodable, Encoder};
 use rustc_session::config::CrateType;
 use rustc_session::cstore::{ForeignModule, LinkagePreference, NativeLib};
 use rustc_span::hygiene::{ExpnIndex, HygieneEncodeContext, MacroKind};
@@ -41,10 +40,11 @@ use std::borrow::Borrow;
 use std::hash::Hash;
 use std::iter;
 use std::num::NonZeroUsize;
+use std::path::Path;
 use tracing::{debug, trace};
 
 pub(super) struct EncodeContext<'a, 'tcx> {
-    opaque: MemEncoder,
+    opaque: opaque::FileEncoder,
     tcx: TyCtxt<'tcx>,
     feat: &'tcx rustc_feature::Features,
 
@@ -730,25 +730,26 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         assert_eq!(total_bytes, computed_total_bytes);
 
         if tcx.sess.meta_stats() {
-            let mut zero_bytes = 0;
-            for e in self.opaque.data.iter() {
-                if *e == 0 {
-                    zero_bytes += 1;
-                }
-            }
+            // let mut zero_bytes = 0;
+            // for e in self.opaque.data.iter() {
+            //     if *e == 0 {
+            //         zero_bytes += 1;
+            //     }
+            // }
 
             let perc = |bytes| (bytes * 100) as f64 / total_bytes as f64;
             let p = |label, bytes| {
                 eprintln!("{:>21}: {:>8} bytes ({:4.1}%)", label, bytes, perc(bytes));
             };
 
             eprintln!("");
-            eprintln!(
-                "{} metadata bytes, of which {} bytes ({:.1}%) are zero",
-                total_bytes,
-                zero_bytes,
-                perc(zero_bytes)
-            );
+            // FIXME print zero bytes
+            //eprintln!(
+            //    "{} metadata bytes, of which {} bytes ({:.1}%) are zero",
+            //    total_bytes,
+            //    zero_bytes,
+            //    perc(zero_bytes)
+            //);
             p("preamble", preamble_bytes);
             p("dep", dep_bytes);
             p("lib feature", lib_feature_bytes);
@@ -2151,15 +2152,15 @@ impl EncodedMetadata {
     }
 }
 
-pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata {
+pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef<Path>) -> EncodedMetadata {
     let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata");
 
     // Since encoding metadata is not in a query, and nothing is cached,
     // there's no need to do dep-graph tracking for any of it.
     tcx.dep_graph.assert_ignored();
 
     join(
-        || encode_metadata_impl(tcx),
+        || encode_metadata_impl(tcx, path),
         || {
             if tcx.sess.threads() == 1 {
                 return;
@@ -2173,8 +2174,9 @@ pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata {
     .0
 }
 
-fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
-    let mut encoder = MemEncoder::new();
+fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef<Path>) -> EncodedMetadata {
+    let mut encoder = opaque::FileEncoder::new(path.as_ref())
+        .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err)));
     encoder.emit_raw_bytes(METADATA_HEADER);
 
     // Will be filled with the root position after encoding everything.
@@ -2209,7 +2211,8 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
     // culminating in the `CrateRoot` which points to all of it.
     let root = ecx.encode_crate_root();
 
-    let mut result = ecx.opaque.finish();
+    ecx.opaque.flush();
+    let mut result = std::fs::read(path.as_ref()).unwrap();
 
     // Encode the root position.
     let header = METADATA_HEADER.len();
@@ -2219,6 +2222,8 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata {
     result[header + 2] = (pos >> 8) as u8;
     result[header + 3] = (pos >> 0) as u8;
 
+    std::fs::write(path, &result).unwrap();
+
     // Record metadata size for self-profiling
     tcx.prof.artifact_size("crate_metadata", "crate_metadata", result.len() as u64);
 

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -22,7 +22,7 @@ use rustc_middle::ty::fast_reject::SimplifiedType;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, ReprOptions, Ty};
 use rustc_middle::ty::{GeneratorDiagnosticData, ParameterizedOverTcx, TyCtxt};
-use rustc_serialize::opaque::MemEncoder;
+use rustc_serialize::opaque::FileEncoder;
 use rustc_session::config::SymbolManglingVersion;
 use rustc_session::cstore::{CrateDepKind, ForeignModule, LinkagePreference, NativeLib};
 use rustc_span::edition::Edition;
@@ -323,7 +323,7 @@ macro_rules! define_tables {
         }
 
         impl TableBuilders {
-            fn encode(&self, buf: &mut MemEncoder) -> LazyTables {
+            fn encode(&self, buf: &mut FileEncoder) -> LazyTables {
                 LazyTables {
                     $($name: self.$name.encode(buf)),+
                 }

compiler/rustc_metadata/src/rmeta/table.rs

@@ -4,8 +4,8 @@ use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_hir::def::{CtorKind, CtorOf};
 use rustc_index::vec::Idx;
 use rustc_middle::ty::ParameterizedOverTcx;
-use rustc_serialize::opaque::MemEncoder;
-use rustc_serialize::Encoder;
+use rustc_serialize::opaque::FileEncoder;
+use rustc_serialize::Encoder as _;
 use rustc_span::hygiene::MacroKind;
 use std::convert::TryInto;
 use std::marker::PhantomData;
@@ -281,7 +281,7 @@ where
         Some(value).write_to_bytes(&mut self.blocks[i]);
     }
 
-    pub(crate) fn encode<const N: usize>(&self, buf: &mut MemEncoder) -> LazyTable<I, T>
+    pub(crate) fn encode<const N: usize>(&self, buf: &mut FileEncoder) -> LazyTable<I, T>
     where
         Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
     {