diff --git a/Cargo.lock b/Cargo.lock index ba0098a977534..dd9e4bd9a5f17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3982,7 +3982,6 @@ dependencies = [ "rustc_ty_utils", "rustc_typeck", "smallvec", - "tempfile", "tracing", "winapi", ] @@ -4092,6 +4091,7 @@ dependencies = [ "rustc_type_ir", "smallvec", "snap", + "tempfile", "tracing", ] diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 72aa790c36357..1051b1eb59389 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -6,6 +6,7 @@ use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_errors::{ErrorGuaranteed, Handler}; use rustc_fs_util::fix_windows_verbatim_for_gcc; use rustc_hir::def_id::CrateNum; +use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME}; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::SymbolExportKind; use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip}; @@ -28,10 +29,7 @@ use super::command::Command; use super::linker::{self, Linker}; use super::metadata::{create_rmeta_file, MetadataPosition}; use super::rpath::{self, RPathConfig}; -use crate::{ - looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib, - METADATA_FILENAME, -}; +use crate::{looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib}; use cc::windows_registry; use regex::Regex; @@ -237,22 +235,6 @@ pub fn each_linked_rlib( Ok(()) } -/// We use a temp directory here to avoid races between concurrent rustc processes, -/// such as builds in the same directory using the same filename for metadata while -/// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a -/// directory being searched for `extern crate` (observing an incomplete file). -/// The returned path is the temporary file containing the complete metadata. -pub fn emit_metadata(sess: &Session, metadata: &[u8], tmpdir: &MaybeTempDir) -> PathBuf { - let out_filename = tmpdir.as_ref().join(METADATA_FILENAME); - let result = fs::write(&out_filename, metadata); - - if let Err(e) = result { - sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); - } - - out_filename -} - /// Create an 'rlib'. /// /// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index 3dd607adee501..0302c28815ab8 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -16,14 +16,13 @@ use rustc_data_structures::memmap::Mmap; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; +use rustc_metadata::fs::METADATA_FILENAME; use rustc_metadata::EncodedMetadata; use rustc_session::cstore::MetadataLoader; use rustc_session::Session; use rustc_target::abi::Endian; use rustc_target::spec::{RelocModel, Target}; -use crate::METADATA_FILENAME; - /// The default metadata loader. This is used by cg_llvm and cg_clif. /// /// # Metadata location diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 4be3ae11e4e5b..1802eedf193aa 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -64,9 +64,6 @@ pub struct ModuleCodegen { pub kind: ModuleKind, } -// FIXME(eddyb) maybe include the crate name in this? -pub const METADATA_FILENAME: &str = "lib.rmeta"; - impl ModuleCodegen { pub fn into_compiled_module( self, diff --git a/compiler/rustc_data_structures/src/memmap.rs b/compiler/rustc_data_structures/src/memmap.rs index 26b26415eea0f..917416df6b867 100644 --- a/compiler/rustc_data_structures/src/memmap.rs +++ b/compiler/rustc_data_structures/src/memmap.rs @@ -1,6 +1,6 @@ use std::fs::File; use std::io; -use std::ops::Deref; +use std::ops::{Deref, DerefMut}; use crate::owning_ref::StableAddress; @@ -45,3 +45,64 @@ impl Deref for Mmap { // export any function that can cause the `Vec` to be re-allocated. As such the address of the // bytes inside this `Vec` is stable. unsafe impl StableAddress for Mmap {} + +#[cfg(not(target_arch = "wasm32"))] +pub struct MmapMut(memmap2::MmapMut); + +#[cfg(target_arch = "wasm32")] +pub struct MmapMut(Vec); + +#[cfg(not(target_arch = "wasm32"))] +impl MmapMut { + #[inline] + pub fn map_anon(len: usize) -> io::Result { + let mmap = memmap2::MmapMut::map_anon(len)?; + Ok(MmapMut(mmap)) + } + + #[inline] + pub fn flush(&mut self) -> io::Result<()> { + self.0.flush() + } + + #[inline] + pub fn make_read_only(self) -> std::io::Result { + let mmap = self.0.make_read_only()?; + Ok(Mmap(mmap)) + } +} + +#[cfg(target_arch = "wasm32")] +impl MmapMut { + #[inline] + pub fn map_anon(len: usize) -> io::Result { + let data = Vec::with_capacity(len); + Ok(MmapMut(data)) + } + + #[inline] + pub fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + + #[inline] + pub fn make_read_only(self) -> std::io::Result { + Ok(Mmap(self.0)) + } +} + +impl Deref for MmapMut { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + &*self.0 + } +} + +impl DerefMut for MmapMut { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + &mut *self.0 + } +} diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index 9aacfba3c8208..1ecbc876c8d8a 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -46,7 +46,6 @@ rustc_query_impl = { path = "../rustc_query_impl" } rustc_resolve = { path = "../rustc_resolve" } rustc_trait_selection = { path = "../rustc_trait_selection" } rustc_ty_utils = { path = "../rustc_ty_utils" } -tempfile = "3.2" [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 502afa493fe66..2504030783815 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -5,19 +5,16 @@ use crate::util; use ast::CRATE_NODE_ID; use rustc_ast::{self as ast, visit}; use rustc_borrowck as mir_borrowck; -use rustc_codegen_ssa::back::link::emit_metadata; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_data_structures::parallel; use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal}; -use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_errors::{Applicability, ErrorGuaranteed, MultiSpan, PResult}; use rustc_expand::base::{ExtCtxt, LintStoreExpand, ResolverExpand}; -use rustc_hir::def_id::{StableCrateId, LOCAL_CRATE}; +use rustc_hir::def_id::StableCrateId; use rustc_hir::definitions::Definitions; use rustc_hir::Crate; use rustc_lint::{EarlyCheckNode, LintStore}; use rustc_metadata::creader::CStore; -use rustc_metadata::{encode_metadata, EncodedMetadata}; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; use rustc_middle::ty::query::{ExternProviders, Providers}; @@ -30,14 +27,13 @@ use rustc_query_impl::{OnDiskCache, Queries as TcxQueries}; use rustc_resolve::{Resolver, ResolverArenas}; use rustc_session::config::{CrateType, Input, OutputFilenames, OutputType}; use rustc_session::cstore::{CrateStoreDyn, MetadataLoader, MetadataLoaderDyn}; -use rustc_session::output::{filename_for_input, filename_for_metadata}; +use rustc_session::output::filename_for_input; use rustc_session::search_paths::PathKind; use rustc_session::{Limit, Session}; use rustc_span::symbol::{sym, Symbol}; use rustc_span::FileName; use rustc_trait_selection::traits; use rustc_typeck as typeck; -use tempfile::Builder as TempFileBuilder; use tracing::{info, warn}; use std::any::Any; @@ -1030,69 +1026,6 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> { Ok(()) } -fn encode_and_write_metadata( - tcx: TyCtxt<'_>, - outputs: &OutputFilenames, -) -> (EncodedMetadata, bool) { - #[derive(PartialEq, Eq, PartialOrd, Ord)] - enum MetadataKind { - None, - Uncompressed, - Compressed, - } - - let metadata_kind = tcx - .sess - .crate_types() - .iter() - .map(|ty| match *ty { - CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None, - - CrateType::Rlib => MetadataKind::Uncompressed, - - CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed, - }) - .max() - .unwrap_or(MetadataKind::None); - - let metadata = match metadata_kind { - MetadataKind::None => EncodedMetadata::new(), - MetadataKind::Uncompressed | MetadataKind::Compressed => encode_metadata(tcx), - }; - - let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); - - let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); - if need_metadata_file { - let crate_name = tcx.crate_name(LOCAL_CRATE); - let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs); - // To avoid races with another rustc process scanning the output directory, - // we need to write the file somewhere else and atomically move it to its - // final destination, with an `fs::rename` call. In order for the rename to - // always succeed, the temporary file needs to be on the same filesystem, - // which is why we create it inside the output directory specifically. - let metadata_tmpdir = TempFileBuilder::new() - .prefix("rmeta") - .tempdir_in(out_filename.parent().unwrap()) - .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); - let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); - let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir); - if let Err(e) = util::non_durable_rename(&metadata_filename, &out_filename) { - tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); - } - if tcx.sess.opts.json_artifact_notifications { - tcx.sess - .parse_sess - .span_diagnostic - .emit_artifact_notification(&out_filename, "metadata"); - } - } - - let need_metadata_module = metadata_kind == MetadataKind::Compressed; - - (metadata, need_metadata_module) -} - /// Runs the codegen backend, after which the AST and analysis can /// be discarded. pub fn start_codegen<'tcx>( @@ -1102,7 +1035,8 @@ pub fn start_codegen<'tcx>( ) -> Box { info!("Pre-codegen\n{:?}", tcx.debug_stats()); - let (metadata, need_metadata_module) = encode_and_write_metadata(tcx, outputs); + let (metadata, need_metadata_module) = + rustc_metadata::fs::encode_and_write_metadata(tcx, outputs); let codegen = tcx.sess.time("codegen_crate", move || { codegen_backend.codegen_crate(tcx, metadata, need_metadata_module) diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index fb9258eb4a938..cbc3ceedc4977 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -655,24 +655,6 @@ pub fn build_output_filenames( } } -#[cfg(not(target_os = "linux"))] -pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> { - std::fs::rename(src, dst) -} - -/// This function attempts to bypass the auto_da_alloc heuristic implemented by some filesystems -/// such as btrfs and ext4. When renaming over a file that already exists then they will "helpfully" -/// write back the source file before committing the rename in case a developer forgot some of -/// the fsyncs in the open/write/fsync(file)/rename/fsync(dir) dance for atomic file updates. -/// -/// To avoid triggering this heuristic we delete the destination first, if it exists. -/// The cost of an extra syscall is much lower than getting descheduled for the sync IO. -#[cfg(target_os = "linux")] -pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> { - let _ = std::fs::remove_file(dst); - std::fs::rename(src, dst) -} - /// Returns a version string such as "1.46.0 (04488afe3 2020-08-24)" pub fn version_str() -> Option<&'static str> { option_env!("CFG_VERSION") diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml index 3ab4a8b7294aa..2c5db9d8b2765 100644 --- a/compiler/rustc_metadata/Cargo.toml +++ b/compiler/rustc_metadata/Cargo.toml @@ -12,6 +12,7 @@ odht = { version = "0.3.1", features = ["nightly"] } snap = "1" tracing = "0.1" smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } +tempfile = "3.2" rustc_middle = { path = "../rustc_middle" } rustc_attr = { path = "../rustc_attr" } rustc_data_structures = { path = "../rustc_data_structures" } diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs new file mode 100644 index 0000000000000..e6072901aaa43 --- /dev/null +++ b/compiler/rustc_metadata/src/fs.rs @@ -0,0 +1,137 @@ +use crate::{encode_metadata, EncodedMetadata}; + +use rustc_data_structures::temp_dir::MaybeTempDir; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{CrateType, OutputFilenames, OutputType}; +use rustc_session::output::filename_for_metadata; +use rustc_session::Session; +use tempfile::Builder as TempFileBuilder; + +use std::fs; +use std::path::{Path, PathBuf}; + +// FIXME(eddyb) maybe include the crate name in this? +pub const METADATA_FILENAME: &str = "lib.rmeta"; + +/// We use a temp directory here to avoid races between concurrent rustc processes, +/// such as builds in the same directory using the same filename for metadata while +/// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a +/// directory being searched for `extern crate` (observing an incomplete file). +/// The returned path is the temporary file containing the complete metadata. +pub fn emit_metadata(sess: &Session, metadata: &[u8], tmpdir: &MaybeTempDir) -> PathBuf { + let out_filename = tmpdir.as_ref().join(METADATA_FILENAME); + let result = fs::write(&out_filename, metadata); + + if let Err(e) = result { + sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); + } + + out_filename +} + +pub fn encode_and_write_metadata( + tcx: TyCtxt<'_>, + outputs: &OutputFilenames, +) -> (EncodedMetadata, bool) { + #[derive(PartialEq, Eq, PartialOrd, Ord)] + enum MetadataKind { + None, + Uncompressed, + Compressed, + } + + let metadata_kind = tcx + .sess + .crate_types() + .iter() + .map(|ty| match *ty { + CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None, + + CrateType::Rlib => MetadataKind::Uncompressed, + + CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed, + }) + .max() + .unwrap_or(MetadataKind::None); + + let crate_name = tcx.crate_name(LOCAL_CRATE); + let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs); + // To avoid races with another rustc process scanning the output directory, + // we need to write the file somewhere else and atomically move it to its + // final destination, with an `fs::rename` call. In order for the rename to + // always succeed, the temporary file needs to be on the same filesystem, + // which is why we create it inside the output directory specifically. + let metadata_tmpdir = TempFileBuilder::new() + .prefix("rmeta") + .tempdir_in(out_filename.parent().unwrap_or_else(|| Path::new(""))) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); + let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); + let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); + + // Always create a file at `metadata_filename`, even if we have nothing to write to it. + // This simplifies the creation of the output `out_filename` when requested. + match metadata_kind { + MetadataKind::None => { + std::fs::File::create(&metadata_filename).unwrap_or_else(|e| { + tcx.sess.fatal(&format!( + "failed to create the file {}: {}", + metadata_filename.display(), + e + )) + }); + } + MetadataKind::Uncompressed | MetadataKind::Compressed => { + encode_metadata(tcx, &metadata_filename); + } + }; + + let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); + + // If the user requests metadata as output, rename `metadata_filename` + // to the expected output `out_filename`. The match above should ensure + // this file always exists. + let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); + let (metadata_filename, metadata_tmpdir) = if need_metadata_file { + if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { + tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); + } + if tcx.sess.opts.json_artifact_notifications { + tcx.sess + .parse_sess + .span_diagnostic + .emit_artifact_notification(&out_filename, "metadata"); + } + (out_filename, None) + } else { + (metadata_filename, Some(metadata_tmpdir)) + }; + + // Load metadata back to memory: codegen may need to include it in object files. + let metadata = + EncodedMetadata::from_path(metadata_filename, metadata_tmpdir).unwrap_or_else(|e| { + tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) + }); + + let need_metadata_module = metadata_kind == MetadataKind::Compressed; + + (metadata, need_metadata_module) +} + +#[cfg(not(target_os = "linux"))] +pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> { + std::fs::rename(src, dst) +} + +/// This function attempts to bypass the auto_da_alloc heuristic implemented by some filesystems +/// such as btrfs and ext4. When renaming over a file that already exists then they will "helpfully" +/// write back the source file before committing the rename in case a developer forgot some of +/// the fsyncs in the open/write/fsync(file)/rename/fsync(dir) dance for atomic file updates. +/// +/// To avoid triggering this heuristic we delete the destination first, if it exists. +/// The cost of an extra syscall is much lower than getting descheduled for the sync IO. +#[cfg(target_os = "linux")] +pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> { + let _ = std::fs::remove_file(dst); + std::fs::rename(src, dst) +} diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs index 5ad16398695b2..6440f3e390cf1 100644 --- a/compiler/rustc_metadata/src/lib.rs +++ b/compiler/rustc_metadata/src/lib.rs @@ -34,6 +34,8 @@ mod native_libs; mod rmeta; pub mod creader; +pub mod fs; pub mod locator; +pub use fs::{emit_metadata, METADATA_FILENAME}; pub use rmeta::{encode_metadata, EncodedMetadata, METADATA_HEADER}; diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 75286b8906871..41de298bcfd8d 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -4,8 +4,10 @@ use crate::rmeta::*; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; +use rustc_data_structures::memmap::{Mmap, MmapMut}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{join, par_iter, Lrc, ParallelIterator}; +use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{ @@ -27,8 +29,7 @@ use rustc_middle::ty::codec::TyEncoder; use rustc_middle::ty::fast_reject::{self, SimplifiedType, TreatParams}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, SymbolName, Ty, TyCtxt}; -use rustc_serialize::opaque::MemEncoder; -use rustc_serialize::{Encodable, Encoder}; +use rustc_serialize::{opaque, Decodable, Decoder, Encodable, Encoder}; use rustc_session::config::CrateType; use rustc_session::cstore::{ForeignModule, LinkagePreference, NativeLib}; use rustc_span::hygiene::{ExpnIndex, HygieneEncodeContext, MacroKind}; @@ -39,12 +40,14 @@ use rustc_span::{ use rustc_target::abi::VariantIdx; use std::borrow::Borrow; use std::hash::Hash; +use std::io::{Read, Seek, Write}; use std::iter; use std::num::NonZeroUsize; +use std::path::{Path, PathBuf}; use tracing::{debug, trace}; pub(super) struct EncodeContext<'a, 'tcx> { - opaque: MemEncoder, + opaque: opaque::FileEncoder, tcx: TyCtxt<'tcx>, feat: &'tcx rustc_feature::Features, @@ -730,12 +733,19 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { assert_eq!(total_bytes, computed_total_bytes); if tcx.sess.meta_stats() { + self.opaque.flush(); + + // Rewind and re-read all the metadata to count the zero bytes we wrote. + let pos_before_rewind = self.opaque.file().stream_position().unwrap(); let mut zero_bytes = 0; - for e in self.opaque.data.iter() { - if *e == 0 { + self.opaque.file().rewind().unwrap(); + let file = std::io::BufReader::new(self.opaque.file()); + for e in file.bytes() { + if e.unwrap() == 0 { zero_bytes += 1; } } + assert_eq!(self.opaque.file().stream_position().unwrap(), pos_before_rewind); let perc = |bytes| (bytes * 100) as f64 / total_bytes as f64; let p = |label, bytes| { @@ -2134,24 +2144,58 @@ fn prefetch_mir(tcx: TyCtxt<'_>) { // will allow us to slice the metadata to the precise length that we just // generated regardless of trailing bytes that end up in it. -#[derive(Encodable, Decodable)] pub struct EncodedMetadata { - raw_data: Vec, + // The declaration order matters because `mmap` should be dropped before `_temp_dir`. + mmap: Option, + // We need to carry MaybeTempDir to avoid deleting the temporary + // directory while accessing the Mmap. + _temp_dir: Option, } impl EncodedMetadata { #[inline] - pub fn new() -> EncodedMetadata { - EncodedMetadata { raw_data: Vec::new() } + pub fn from_path(path: PathBuf, temp_dir: Option) -> std::io::Result { + let file = std::fs::File::open(&path)?; + let file_metadata = file.metadata()?; + if file_metadata.len() == 0 { + return Ok(Self { mmap: None, _temp_dir: None }); + } + let mmap = unsafe { Some(Mmap::map(file)?) }; + Ok(Self { mmap, _temp_dir: temp_dir }) } #[inline] pub fn raw_data(&self) -> &[u8] { - &self.raw_data + self.mmap.as_ref().map(|mmap| mmap.as_ref()).unwrap_or_default() + } +} + +impl Encodable for EncodedMetadata { + fn encode(&self, s: &mut S) { + let slice = self.raw_data(); + slice.encode(s) + } +} + +impl Decodable for EncodedMetadata { + fn decode(d: &mut D) -> Self { + let len = d.read_usize(); + let mmap = if len > 0 { + let mut mmap = MmapMut::map_anon(len).unwrap(); + for _ in 0..len { + (&mut mmap[..]).write(&[d.read_u8()]).unwrap(); + } + mmap.flush().unwrap(); + Some(mmap.make_read_only().unwrap()) + } else { + None + }; + + Self { mmap, _temp_dir: None } } } -pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata { +pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path) { let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata"); // Since encoding metadata is not in a query, and nothing is cached, @@ -2159,7 +2203,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata { tcx.dep_graph.assert_ignored(); join( - || encode_metadata_impl(tcx), + || encode_metadata_impl(tcx, path), || { if tcx.sess.threads() == 1 { return; @@ -2169,12 +2213,12 @@ pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata { // It can be removed if it turns out to cause trouble or be detrimental to performance. join(|| prefetch_mir(tcx), || tcx.exported_symbols(LOCAL_CRATE)); }, - ) - .0 + ); } -fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { - let mut encoder = MemEncoder::new(); +fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) { + let mut encoder = opaque::FileEncoder::new(path) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err))); encoder.emit_raw_bytes(METADATA_HEADER); // Will be filled with the root position after encoding everything. @@ -2209,20 +2253,29 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { // culminating in the `CrateRoot` which points to all of it. let root = ecx.encode_crate_root(); - let mut result = ecx.opaque.finish(); + ecx.opaque.flush(); + + let mut file = ecx.opaque.file(); + // We will return to this position after writing the root position. + let pos_before_seek = file.stream_position().unwrap(); // Encode the root position. let header = METADATA_HEADER.len(); + file.seek(std::io::SeekFrom::Start(header as u64)) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to seek the file: {}", err))); let pos = root.position.get(); - result[header + 0] = (pos >> 24) as u8; - result[header + 1] = (pos >> 16) as u8; - result[header + 2] = (pos >> 8) as u8; - result[header + 3] = (pos >> 0) as u8; + file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8]) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to write to the file: {}", err))); - // Record metadata size for self-profiling - tcx.prof.artifact_size("crate_metadata", "crate_metadata", result.len() as u64); + // Return to the position where we are before writing the root position. + file.seek(std::io::SeekFrom::Start(pos_before_seek)).unwrap(); - EncodedMetadata { raw_data: result } + // Record metadata size for self-profiling + tcx.prof.artifact_size( + "crate_metadata", + "crate_metadata", + file.metadata().unwrap().len() as u64, + ); } pub fn provide(providers: &mut Providers) { @@ -2243,5 +2296,5 @@ pub fn provide(providers: &mut Providers) { }, ..*providers - }; + } } diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index a58c0e68ee38c..5caeec665d267 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -22,7 +22,7 @@ use rustc_middle::ty::fast_reject::SimplifiedType; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, ReprOptions, Ty}; use rustc_middle::ty::{GeneratorDiagnosticData, ParameterizedOverTcx, TyCtxt}; -use rustc_serialize::opaque::MemEncoder; +use rustc_serialize::opaque::FileEncoder; use rustc_session::config::SymbolManglingVersion; use rustc_session::cstore::{CrateDepKind, ForeignModule, LinkagePreference, NativeLib}; use rustc_span::edition::Edition; @@ -323,7 +323,7 @@ macro_rules! define_tables { } impl TableBuilders { - fn encode(&self, buf: &mut MemEncoder) -> LazyTables { + fn encode(&self, buf: &mut FileEncoder) -> LazyTables { LazyTables { $($name: self.$name.encode(buf)),+ } diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs index 5ab4269ae99ad..42759f0a652b3 100644 --- a/compiler/rustc_metadata/src/rmeta/table.rs +++ b/compiler/rustc_metadata/src/rmeta/table.rs @@ -4,8 +4,8 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_hir::def::{CtorKind, CtorOf}; use rustc_index::vec::Idx; use rustc_middle::ty::ParameterizedOverTcx; -use rustc_serialize::opaque::MemEncoder; -use rustc_serialize::Encoder; +use rustc_serialize::opaque::FileEncoder; +use rustc_serialize::Encoder as _; use rustc_span::hygiene::MacroKind; use std::convert::TryInto; use std::marker::PhantomData; @@ -281,7 +281,7 @@ where Some(value).write_to_bytes(&mut self.blocks[i]); } - pub(crate) fn encode(&self, buf: &mut MemEncoder) -> LazyTable + pub(crate) fn encode(&self, buf: &mut FileEncoder) -> LazyTable where Option: FixedSizeEncoding, { diff --git a/compiler/rustc_serialize/src/opaque.rs b/compiler/rustc_serialize/src/opaque.rs index 366efe9cfa519..5c17ef6ace2d5 100644 --- a/compiler/rustc_serialize/src/opaque.rs +++ b/compiler/rustc_serialize/src/opaque.rs @@ -297,6 +297,10 @@ impl FileEncoder { } } + pub fn file(&self) -> &File { + &self.file + } + #[inline] fn capacity(&self) -> usize { self.buf.len() diff --git a/src/test/run-make-fulldeps/issue-26092/Makefile b/src/test/run-make-fulldeps/issue-26092/Makefile index 27631c31c4a06..885f45a022431 100644 --- a/src/test/run-make-fulldeps/issue-26092/Makefile +++ b/src/test/run-make-fulldeps/issue-26092/Makefile @@ -1,4 +1,6 @@ -include ../tools.mk +# This test ensures that rustc does not panic with `-o ""` option. + all: - $(RUSTC) -o "" blank.rs 2>&1 | $(CGREP) -i 'No such file or directory' + $(RUSTC) -o "" blank.rs 2>&1 | $(CGREP) -i 'panic' && exit 1 || exit 0