From 709a78226b12025385976bdafa6d7793ec8a4a5b Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 24 Apr 2022 19:34:35 +0900 Subject: [PATCH 01/32] move emit_metadata to rustc_metadata::fs --- compiler/rustc_codegen_ssa/src/back/link.rs | 29 ++++--------------- .../rustc_codegen_ssa/src/back/metadata.rs | 3 +- compiler/rustc_codegen_ssa/src/lib.rs | 3 -- compiler/rustc_interface/src/passes.rs | 2 +- compiler/rustc_metadata/src/fs.rs | 24 +++++++++++++++ compiler/rustc_metadata/src/lib.rs | 2 ++ 6 files changed, 33 insertions(+), 30 deletions(-) create mode 100644 compiler/rustc_metadata/src/fs.rs diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 72aa790c36357..ab794fd4d274f 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -28,10 +28,7 @@ use super::command::Command; use super::linker::{self, Linker}; use super::metadata::{create_rmeta_file, MetadataPosition}; use super::rpath::{self, RPathConfig}; -use crate::{ - looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib, - METADATA_FILENAME, -}; +use crate::{looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib}; use cc::windows_registry; use regex::Regex; @@ -237,23 +234,7 @@ pub fn each_linked_rlib( Ok(()) } -/// We use a temp directory here to avoid races between concurrent rustc processes, -/// such as builds in the same directory using the same filename for metadata while -/// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a -/// directory being searched for `extern crate` (observing an incomplete file). -/// The returned path is the temporary file containing the complete metadata. -pub fn emit_metadata(sess: &Session, metadata: &[u8], tmpdir: &MaybeTempDir) -> PathBuf { - let out_filename = tmpdir.as_ref().join(METADATA_FILENAME); - let result = fs::write(&out_filename, metadata); - - if let Err(e) = result { - sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); - } - - out_filename -} - -/// Create an 'rlib'. +/// Create an 'arlib'. /// /// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains /// the object file of the crate, but it also contains all of the object files from native @@ -276,7 +257,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( RlibFlavor::Normal => { let (metadata, metadata_position) = create_rmeta_file(sess, codegen_results.metadata.raw_data()); - let metadata = emit_metadata(sess, &metadata, tmpdir); + let metadata = rustc_metadata::fs::emit_metadata(sess, &metadata, tmpdir); match metadata_position { MetadataPosition::First => { // Most of the time metadata in rlib files is wrapped in a "dummy" object @@ -502,7 +483,7 @@ fn link_staticlib<'a, B: ArchiveBuilder<'a>>( ab.add_archive(path, move |fname: &str| { // Ignore metadata files, no matter the name. - if fname == METADATA_FILENAME { + if fname == rustc_metadata::fs::METADATA_FILENAME { return true; } @@ -2474,7 +2455,7 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( let mut archive = ::new(sess, &dst); if let Err(e) = archive.add_archive(cratepath, move |f| { - if f == METADATA_FILENAME { + if f == rustc_metadata::fs::METADATA_FILENAME { return true; } diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs index 3dd607adee501..0302c28815ab8 100644 --- a/compiler/rustc_codegen_ssa/src/back/metadata.rs +++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs @@ -16,14 +16,13 @@ use rustc_data_structures::memmap::Mmap; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; +use rustc_metadata::fs::METADATA_FILENAME; use rustc_metadata::EncodedMetadata; use rustc_session::cstore::MetadataLoader; use rustc_session::Session; use rustc_target::abi::Endian; use rustc_target::spec::{RelocModel, Target}; -use crate::METADATA_FILENAME; - /// The default metadata loader. This is used by cg_llvm and cg_clif. /// /// # Metadata location diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs index 4be3ae11e4e5b..1802eedf193aa 100644 --- a/compiler/rustc_codegen_ssa/src/lib.rs +++ b/compiler/rustc_codegen_ssa/src/lib.rs @@ -64,9 +64,6 @@ pub struct ModuleCodegen { pub kind: ModuleKind, } -// FIXME(eddyb) maybe include the crate name in this? -pub const METADATA_FILENAME: &str = "lib.rmeta"; - impl ModuleCodegen { pub fn into_compiled_module( self, diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 502afa493fe66..6d00ce33d8f46 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -5,7 +5,6 @@ use crate::util; use ast::CRATE_NODE_ID; use rustc_ast::{self as ast, visit}; use rustc_borrowck as mir_borrowck; -use rustc_codegen_ssa::back::link::emit_metadata; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_data_structures::parallel; use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal}; @@ -17,6 +16,7 @@ use rustc_hir::definitions::Definitions; use rustc_hir::Crate; use rustc_lint::{EarlyCheckNode, LintStore}; use rustc_metadata::creader::CStore; +use rustc_metadata::fs::emit_metadata; use rustc_metadata::{encode_metadata, EncodedMetadata}; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs new file mode 100644 index 0000000000000..f57ace6f28a81 --- /dev/null +++ b/compiler/rustc_metadata/src/fs.rs @@ -0,0 +1,24 @@ +use rustc_data_structures::temp_dir::MaybeTempDir; +use rustc_session::Session; + +use std::fs; +use std::path::PathBuf; + +// FIXME(eddyb) maybe include the crate name in this? +pub const METADATA_FILENAME: &str = "lib.rmeta"; + +/// We use a temp directory here to avoid races between concurrent rustc processes, +/// such as builds in the same directory using the same filename for metadata while +/// building an `.rlib` (stomping over one another), or writing an `.rmeta` into a +/// directory being searched for `extern crate` (observing an incomplete file). +/// The returned path is the temporary file containing the complete metadata. +pub fn emit_metadata(sess: &Session, metadata: &[u8], tmpdir: &MaybeTempDir) -> PathBuf { + let out_filename = tmpdir.as_ref().join(METADATA_FILENAME); + let result = fs::write(&out_filename, metadata); + + if let Err(e) = result { + sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); + } + + out_filename +} diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs index 5ad16398695b2..6440f3e390cf1 100644 --- a/compiler/rustc_metadata/src/lib.rs +++ b/compiler/rustc_metadata/src/lib.rs @@ -34,6 +34,8 @@ mod native_libs; mod rmeta; pub mod creader; +pub mod fs; pub mod locator; +pub use fs::{emit_metadata, METADATA_FILENAME}; pub use rmeta::{encode_metadata, EncodedMetadata, METADATA_HEADER}; From 5d9ba49bb9ca0e8c2e563ca499a5e54e90c9fb88 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 24 Apr 2022 23:49:04 +0900 Subject: [PATCH 02/32] move encode_and_write_metadata to rustc_metadata::fs --- Cargo.lock | 2 +- compiler/rustc_interface/Cargo.toml | 1 - compiler/rustc_interface/src/passes.rs | 74 ++------------------- compiler/rustc_metadata/Cargo.toml | 1 + compiler/rustc_metadata/src/fs.rs | 90 +++++++++++++++++++++++++- 5 files changed, 95 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba0098a977534..dd9e4bd9a5f17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3982,7 +3982,6 @@ dependencies = [ "rustc_ty_utils", "rustc_typeck", "smallvec", - "tempfile", "tracing", "winapi", ] @@ -4092,6 +4091,7 @@ dependencies = [ "rustc_type_ir", "smallvec", "snap", + "tempfile", "tracing", ] diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml index 9aacfba3c8208..1ecbc876c8d8a 100644 --- a/compiler/rustc_interface/Cargo.toml +++ b/compiler/rustc_interface/Cargo.toml @@ -46,7 +46,6 @@ rustc_query_impl = { path = "../rustc_query_impl" } rustc_resolve = { path = "../rustc_resolve" } rustc_trait_selection = { path = "../rustc_trait_selection" } rustc_ty_utils = { path = "../rustc_ty_utils" } -tempfile = "3.2" [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 6d00ce33d8f46..2504030783815 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -8,16 +8,13 @@ use rustc_borrowck as mir_borrowck; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_data_structures::parallel; use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal}; -use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_errors::{Applicability, ErrorGuaranteed, MultiSpan, PResult}; use rustc_expand::base::{ExtCtxt, LintStoreExpand, ResolverExpand}; -use rustc_hir::def_id::{StableCrateId, LOCAL_CRATE}; +use rustc_hir::def_id::StableCrateId; use rustc_hir::definitions::Definitions; use rustc_hir::Crate; use rustc_lint::{EarlyCheckNode, LintStore}; use rustc_metadata::creader::CStore; -use rustc_metadata::fs::emit_metadata; -use rustc_metadata::{encode_metadata, EncodedMetadata}; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; use rustc_middle::ty::query::{ExternProviders, Providers}; @@ -30,14 +27,13 @@ use rustc_query_impl::{OnDiskCache, Queries as TcxQueries}; use rustc_resolve::{Resolver, ResolverArenas}; use rustc_session::config::{CrateType, Input, OutputFilenames, OutputType}; use rustc_session::cstore::{CrateStoreDyn, MetadataLoader, MetadataLoaderDyn}; -use rustc_session::output::{filename_for_input, filename_for_metadata}; +use rustc_session::output::filename_for_input; use rustc_session::search_paths::PathKind; use rustc_session::{Limit, Session}; use rustc_span::symbol::{sym, Symbol}; use rustc_span::FileName; use rustc_trait_selection::traits; use rustc_typeck as typeck; -use tempfile::Builder as TempFileBuilder; use tracing::{info, warn}; use std::any::Any; @@ -1030,69 +1026,6 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> { Ok(()) } -fn encode_and_write_metadata( - tcx: TyCtxt<'_>, - outputs: &OutputFilenames, -) -> (EncodedMetadata, bool) { - #[derive(PartialEq, Eq, PartialOrd, Ord)] - enum MetadataKind { - None, - Uncompressed, - Compressed, - } - - let metadata_kind = tcx - .sess - .crate_types() - .iter() - .map(|ty| match *ty { - CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None, - - CrateType::Rlib => MetadataKind::Uncompressed, - - CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed, - }) - .max() - .unwrap_or(MetadataKind::None); - - let metadata = match metadata_kind { - MetadataKind::None => EncodedMetadata::new(), - MetadataKind::Uncompressed | MetadataKind::Compressed => encode_metadata(tcx), - }; - - let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); - - let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); - if need_metadata_file { - let crate_name = tcx.crate_name(LOCAL_CRATE); - let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs); - // To avoid races with another rustc process scanning the output directory, - // we need to write the file somewhere else and atomically move it to its - // final destination, with an `fs::rename` call. In order for the rename to - // always succeed, the temporary file needs to be on the same filesystem, - // which is why we create it inside the output directory specifically. - let metadata_tmpdir = TempFileBuilder::new() - .prefix("rmeta") - .tempdir_in(out_filename.parent().unwrap()) - .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); - let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); - let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir); - if let Err(e) = util::non_durable_rename(&metadata_filename, &out_filename) { - tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); - } - if tcx.sess.opts.json_artifact_notifications { - tcx.sess - .parse_sess - .span_diagnostic - .emit_artifact_notification(&out_filename, "metadata"); - } - } - - let need_metadata_module = metadata_kind == MetadataKind::Compressed; - - (metadata, need_metadata_module) -} - /// Runs the codegen backend, after which the AST and analysis can /// be discarded. pub fn start_codegen<'tcx>( @@ -1102,7 +1035,8 @@ pub fn start_codegen<'tcx>( ) -> Box { info!("Pre-codegen\n{:?}", tcx.debug_stats()); - let (metadata, need_metadata_module) = encode_and_write_metadata(tcx, outputs); + let (metadata, need_metadata_module) = + rustc_metadata::fs::encode_and_write_metadata(tcx, outputs); let codegen = tcx.sess.time("codegen_crate", move || { codegen_backend.codegen_crate(tcx, metadata, need_metadata_module) diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml index 3ab4a8b7294aa..2c5db9d8b2765 100644 --- a/compiler/rustc_metadata/Cargo.toml +++ b/compiler/rustc_metadata/Cargo.toml @@ -12,6 +12,7 @@ odht = { version = "0.3.1", features = ["nightly"] } snap = "1" tracing = "0.1" smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } +tempfile = "3.2" rustc_middle = { path = "../rustc_middle" } rustc_attr = { path = "../rustc_attr" } rustc_data_structures = { path = "../rustc_data_structures" } diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index f57ace6f28a81..b66f37d5d611d 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -1,8 +1,15 @@ +use crate::{encode_metadata, EncodedMetadata}; + use rustc_data_structures::temp_dir::MaybeTempDir; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{CrateType, OutputFilenames, OutputType}; +use rustc_session::output::filename_for_metadata; use rustc_session::Session; +use tempfile::Builder as TempFileBuilder; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; // FIXME(eddyb) maybe include the crate name in this? pub const METADATA_FILENAME: &str = "lib.rmeta"; @@ -22,3 +29,84 @@ pub fn emit_metadata(sess: &Session, metadata: &[u8], tmpdir: &MaybeTempDir) -> out_filename } + +pub fn encode_and_write_metadata( + tcx: TyCtxt<'_>, + outputs: &OutputFilenames, +) -> (EncodedMetadata, bool) { + #[derive(PartialEq, Eq, PartialOrd, Ord)] + enum MetadataKind { + None, + Uncompressed, + Compressed, + } + + let metadata_kind = tcx + .sess + .crate_types() + .iter() + .map(|ty| match *ty { + CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => MetadataKind::None, + + CrateType::Rlib => MetadataKind::Uncompressed, + + CrateType::Dylib | CrateType::ProcMacro => MetadataKind::Compressed, + }) + .max() + .unwrap_or(MetadataKind::None); + + let metadata = match metadata_kind { + MetadataKind::None => EncodedMetadata::new(), + MetadataKind::Uncompressed | MetadataKind::Compressed => encode_metadata(tcx), + }; + + let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); + + let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); + if need_metadata_file { + let crate_name = tcx.crate_name(LOCAL_CRATE); + let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs); + // To avoid races with another rustc process scanning the output directory, + // we need to write the file somewhere else and atomically move it to its + // final destination, with an `fs::rename` call. In order for the rename to + // always succeed, the temporary file needs to be on the same filesystem, + // which is why we create it inside the output directory specifically. + let metadata_tmpdir = TempFileBuilder::new() + .prefix("rmeta") + .tempdir_in(out_filename.parent().unwrap()) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); + let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); + let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir); + if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { + tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); + } + if tcx.sess.opts.json_artifact_notifications { + tcx.sess + .parse_sess + .span_diagnostic + .emit_artifact_notification(&out_filename, "metadata"); + } + } + + let need_metadata_module = metadata_kind == MetadataKind::Compressed; + + (metadata, need_metadata_module) +} + +#[cfg(not(target_os = "linux"))] +pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> { + std::fs::rename(src, dst) +} + +/// This function attempts to bypass the auto_da_alloc heuristic implemented by some filesystems +/// such as btrfs and ext4. When renaming over a file that already exists then they will "helpfully" +/// write back the source file before committing the rename in case a developer forgot some of +/// the fsyncs in the open/write/fsync(file)/rename/fsync(dir) dance for atomic file updates. +/// +/// To avoid triggering this heuristic we delete the destination first, if it exists. +/// The cost of an extra syscall is much lower than getting descheduled for the sync IO. +#[cfg(target_os = "linux")] +pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> { + let _ = std::fs::remove_file(dst); + std::fs::rename(src, dst) +} From e7f95ace08fecfb974cbb2d486e4c7920a6010ca Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 25 Apr 2022 20:58:30 +0900 Subject: [PATCH 03/32] use rustc_serialize::opaque::FileEncoder --- compiler/rustc_metadata/src/fs.rs | 29 +++++++------ compiler/rustc_metadata/src/rmeta/encoder.rs | 45 +++++++++++--------- compiler/rustc_metadata/src/rmeta/mod.rs | 4 +- compiler/rustc_metadata/src/rmeta/table.rs | 6 +-- 4 files changed, 46 insertions(+), 38 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index b66f37d5d611d..e3b6cb99b6373 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -55,27 +55,30 @@ pub fn encode_and_write_metadata( .max() .unwrap_or(MetadataKind::None); + let crate_name = tcx.crate_name(LOCAL_CRATE); + let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs); + // To avoid races with another rustc process scanning the output directory, + // we need to write the file somewhere else and atomically move it to its + // final destination, with an `fs::rename` call. In order for the rename to + // always succeed, the temporary file needs to be on the same filesystem, + // which is why we create it inside the output directory specifically. + let metadata_tmpdir = TempFileBuilder::new() + .prefix("rmeta") + .tempdir_in(out_filename.parent().unwrap_or_else(|| Path::new(""))) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); + let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); + let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); let metadata = match metadata_kind { MetadataKind::None => EncodedMetadata::new(), - MetadataKind::Uncompressed | MetadataKind::Compressed => encode_metadata(tcx), + MetadataKind::Uncompressed | MetadataKind::Compressed => { + encode_metadata(tcx, metadata_filename) + } }; let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); if need_metadata_file { - let crate_name = tcx.crate_name(LOCAL_CRATE); - let out_filename = filename_for_metadata(tcx.sess, crate_name.as_str(), outputs); - // To avoid races with another rustc process scanning the output directory, - // we need to write the file somewhere else and atomically move it to its - // final destination, with an `fs::rename` call. In order for the rename to - // always succeed, the temporary file needs to be on the same filesystem, - // which is why we create it inside the output directory specifically. - let metadata_tmpdir = TempFileBuilder::new() - .prefix("rmeta") - .tempdir_in(out_filename.parent().unwrap()) - .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); - let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir); if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 75286b8906871..9ea383331b6e0 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -27,8 +27,7 @@ use rustc_middle::ty::codec::TyEncoder; use rustc_middle::ty::fast_reject::{self, SimplifiedType, TreatParams}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, SymbolName, Ty, TyCtxt}; -use rustc_serialize::opaque::MemEncoder; -use rustc_serialize::{Encodable, Encoder}; +use rustc_serialize::{opaque, Encodable, Encoder}; use rustc_session::config::CrateType; use rustc_session::cstore::{ForeignModule, LinkagePreference, NativeLib}; use rustc_span::hygiene::{ExpnIndex, HygieneEncodeContext, MacroKind}; @@ -41,10 +40,11 @@ use std::borrow::Borrow; use std::hash::Hash; use std::iter; use std::num::NonZeroUsize; +use std::path::Path; use tracing::{debug, trace}; pub(super) struct EncodeContext<'a, 'tcx> { - opaque: MemEncoder, + opaque: opaque::FileEncoder, tcx: TyCtxt<'tcx>, feat: &'tcx rustc_feature::Features, @@ -730,12 +730,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { assert_eq!(total_bytes, computed_total_bytes); if tcx.sess.meta_stats() { - let mut zero_bytes = 0; - for e in self.opaque.data.iter() { - if *e == 0 { - zero_bytes += 1; - } - } + // let mut zero_bytes = 0; + // for e in self.opaque.data.iter() { + // if *e == 0 { + // zero_bytes += 1; + // } + // } let perc = |bytes| (bytes * 100) as f64 / total_bytes as f64; let p = |label, bytes| { @@ -743,12 +743,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { }; eprintln!(""); - eprintln!( - "{} metadata bytes, of which {} bytes ({:.1}%) are zero", - total_bytes, - zero_bytes, - perc(zero_bytes) - ); + // FIXME print zero bytes + //eprintln!( + // "{} metadata bytes, of which {} bytes ({:.1}%) are zero", + // total_bytes, + // zero_bytes, + // perc(zero_bytes) + //); p("preamble", preamble_bytes); p("dep", dep_bytes); p("lib feature", lib_feature_bytes); @@ -2151,7 +2152,7 @@ impl EncodedMetadata { } } -pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata { +pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef) -> EncodedMetadata { let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata"); // Since encoding metadata is not in a query, and nothing is cached, @@ -2159,7 +2160,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata { tcx.dep_graph.assert_ignored(); join( - || encode_metadata_impl(tcx), + || encode_metadata_impl(tcx, path), || { if tcx.sess.threads() == 1 { return; @@ -2173,8 +2174,9 @@ pub fn encode_metadata(tcx: TyCtxt<'_>) -> EncodedMetadata { .0 } -fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { - let mut encoder = MemEncoder::new(); +fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) -> EncodedMetadata { + let mut encoder = opaque::FileEncoder::new(path.as_ref()) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err))); encoder.emit_raw_bytes(METADATA_HEADER); // Will be filled with the root position after encoding everything. @@ -2209,7 +2211,8 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { // culminating in the `CrateRoot` which points to all of it. let root = ecx.encode_crate_root(); - let mut result = ecx.opaque.finish(); + ecx.opaque.flush(); + let mut result = std::fs::read(path.as_ref()).unwrap(); // Encode the root position. let header = METADATA_HEADER.len(); @@ -2219,6 +2222,8 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>) -> EncodedMetadata { result[header + 2] = (pos >> 8) as u8; result[header + 3] = (pos >> 0) as u8; + std::fs::write(path, &result).unwrap(); + // Record metadata size for self-profiling tcx.prof.artifact_size("crate_metadata", "crate_metadata", result.len() as u64); diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index a58c0e68ee38c..5caeec665d267 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -22,7 +22,7 @@ use rustc_middle::ty::fast_reject::SimplifiedType; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, ReprOptions, Ty}; use rustc_middle::ty::{GeneratorDiagnosticData, ParameterizedOverTcx, TyCtxt}; -use rustc_serialize::opaque::MemEncoder; +use rustc_serialize::opaque::FileEncoder; use rustc_session::config::SymbolManglingVersion; use rustc_session::cstore::{CrateDepKind, ForeignModule, LinkagePreference, NativeLib}; use rustc_span::edition::Edition; @@ -323,7 +323,7 @@ macro_rules! define_tables { } impl TableBuilders { - fn encode(&self, buf: &mut MemEncoder) -> LazyTables { + fn encode(&self, buf: &mut FileEncoder) -> LazyTables { LazyTables { $($name: self.$name.encode(buf)),+ } diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs index 5ab4269ae99ad..42759f0a652b3 100644 --- a/compiler/rustc_metadata/src/rmeta/table.rs +++ b/compiler/rustc_metadata/src/rmeta/table.rs @@ -4,8 +4,8 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_hir::def::{CtorKind, CtorOf}; use rustc_index::vec::Idx; use rustc_middle::ty::ParameterizedOverTcx; -use rustc_serialize::opaque::MemEncoder; -use rustc_serialize::Encoder; +use rustc_serialize::opaque::FileEncoder; +use rustc_serialize::Encoder as _; use rustc_span::hygiene::MacroKind; use std::convert::TryInto; use std::marker::PhantomData; @@ -281,7 +281,7 @@ where Some(value).write_to_bytes(&mut self.blocks[i]); } - pub(crate) fn encode(&self, buf: &mut MemEncoder) -> LazyTable + pub(crate) fn encode(&self, buf: &mut FileEncoder) -> LazyTable where Option: FixedSizeEncoding, { From bb75c4b46aff4a610d2c78949d06e3d9e8110731 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Thu, 28 Apr 2022 10:27:38 +0900 Subject: [PATCH 04/32] call emit_metadata only when metadata_kind is None --- compiler/rustc_metadata/src/fs.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index e3b6cb99b6373..71052195fadc9 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -69,9 +69,13 @@ pub fn encode_and_write_metadata( let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); let metadata = match metadata_kind { - MetadataKind::None => EncodedMetadata::new(), + MetadataKind::None => { + let metadata = EncodedMetadata::new(); + let _ = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir); + metadata + } MetadataKind::Uncompressed | MetadataKind::Compressed => { - encode_metadata(tcx, metadata_filename) + encode_metadata(tcx, &metadata_filename) } }; @@ -79,7 +83,6 @@ pub fn encode_and_write_metadata( let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); if need_metadata_file { - let metadata_filename = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir); if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); } From c26c461c0c0f4ee81616a794b1ac3cb41a3f17a6 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Thu, 28 Apr 2022 10:55:04 +0900 Subject: [PATCH 05/32] construct EncodedMetadata in encode_and_write_metadata --- compiler/rustc_metadata/src/fs.rs | 15 +++++++++------ compiler/rustc_metadata/src/rmeta/encoder.rs | 14 ++++++++------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 71052195fadc9..950009397ceda 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -68,11 +68,9 @@ pub fn encode_and_write_metadata( .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); - let metadata = match metadata_kind { + match metadata_kind { MetadataKind::None => { - let metadata = EncodedMetadata::new(); - let _ = emit_metadata(tcx.sess, metadata.raw_data(), &metadata_tmpdir); - metadata + let _ = emit_metadata(tcx.sess, &[], &metadata_tmpdir); } MetadataKind::Uncompressed | MetadataKind::Compressed => { encode_metadata(tcx, &metadata_filename) @@ -82,7 +80,7 @@ pub fn encode_and_write_metadata( let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); - if need_metadata_file { + let metadata_filename = if need_metadata_file { if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); } @@ -92,7 +90,12 @@ pub fn encode_and_write_metadata( .span_diagnostic .emit_artifact_notification(&out_filename, "metadata"); } - } + out_filename + } else { + metadata_filename + }; + let raw_data = std::fs::read(metadata_filename).unwrap(); + let metadata = EncodedMetadata::from_raw_data(raw_data); let need_metadata_module = metadata_kind == MetadataKind::Compressed; diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 9ea383331b6e0..f18a05fcb1d25 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2146,13 +2146,18 @@ impl EncodedMetadata { EncodedMetadata { raw_data: Vec::new() } } + #[inline] + pub fn from_raw_data(raw_data: Vec) -> Self { + Self { raw_data } + } + #[inline] pub fn raw_data(&self) -> &[u8] { &self.raw_data } } -pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef) -> EncodedMetadata { +pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef) { let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata"); // Since encoding metadata is not in a query, and nothing is cached, @@ -2170,11 +2175,10 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef) -> EncodedMetada // It can be removed if it turns out to cause trouble or be detrimental to performance. join(|| prefetch_mir(tcx), || tcx.exported_symbols(LOCAL_CRATE)); }, - ) - .0 + ); } -fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) -> EncodedMetadata { +fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) { let mut encoder = opaque::FileEncoder::new(path.as_ref()) .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err))); encoder.emit_raw_bytes(METADATA_HEADER); @@ -2226,8 +2230,6 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) -> EncodedMetad // Record metadata size for self-profiling tcx.prof.artifact_size("crate_metadata", "crate_metadata", result.len() as u64); - - EncodedMetadata { raw_data: result } } pub fn provide(providers: &mut Providers) { From 8cfa7caac9bba5b83995040ebf313559e4e2186c Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Fri, 29 Apr 2022 12:24:58 +0900 Subject: [PATCH 06/32] hold Mmap in EncodedMetadata --- compiler/rustc_metadata/src/fs.rs | 6 ++- compiler/rustc_metadata/src/rmeta/encoder.rs | 39 +++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 950009397ceda..fa97cda0aa361 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -94,8 +94,10 @@ pub fn encode_and_write_metadata( } else { metadata_filename }; - let raw_data = std::fs::read(metadata_filename).unwrap(); - let metadata = EncodedMetadata::from_raw_data(raw_data); + let file = std::fs::File::open(metadata_filename).unwrap(); + let metadata = EncodedMetadata::from_file(file).unwrap_or_else(|e| { + tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) + }); let need_metadata_module = metadata_kind == MetadataKind::Compressed; diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index f18a05fcb1d25..6e2044d4bb007 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -4,6 +4,7 @@ use crate::rmeta::*; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; +use rustc_data_structures::memmap::Mmap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{join, par_iter, Lrc, ParallelIterator}; use rustc_hir as hir; @@ -27,7 +28,7 @@ use rustc_middle::ty::codec::TyEncoder; use rustc_middle::ty::fast_reject::{self, SimplifiedType, TreatParams}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, SymbolName, Ty, TyCtxt}; -use rustc_serialize::{opaque, Encodable, Encoder}; +use rustc_serialize::{opaque, Decodable, Decoder, Encodable, Encoder}; use rustc_session::config::CrateType; use rustc_session::cstore::{ForeignModule, LinkagePreference, NativeLib}; use rustc_span::hygiene::{ExpnIndex, HygieneEncodeContext, MacroKind}; @@ -2135,25 +2136,43 @@ fn prefetch_mir(tcx: TyCtxt<'_>) { // will allow us to slice the metadata to the precise length that we just // generated regardless of trailing bytes that end up in it. -#[derive(Encodable, Decodable)] pub struct EncodedMetadata { - raw_data: Vec, + mmap: Option, + decoded: Vec, } impl EncodedMetadata { #[inline] - pub fn new() -> EncodedMetadata { - EncodedMetadata { raw_data: Vec::new() } + pub fn from_file(file: std::fs::File) -> std::io::Result { + let file_metadata = file.metadata()?; + if file_metadata.len() == 0 { + return Ok(Self { mmap: None, decoded: Vec::new() }); + } + let mmap = unsafe { Some(Mmap::map(file)?) }; + Ok(Self { mmap, decoded: Vec::new() }) } #[inline] - pub fn from_raw_data(raw_data: Vec) -> Self { - Self { raw_data } + pub fn raw_data(&self) -> &[u8] { + if !self.decoded.is_empty() { + return &self.decoded; + } + self.mmap.as_ref().map(|mmap| mmap.as_ref()).unwrap_or_default() } +} - #[inline] - pub fn raw_data(&self) -> &[u8] { - &self.raw_data +impl Encodable for EncodedMetadata { + fn encode(&self, s: &mut S) -> Result<(), S::Error> { + let slice = self.raw_data(); + slice.encode(s) + } +} + +impl Decodable for EncodedMetadata { + fn decode(d: &mut D) -> Self { + // FIXME: Write decorded data to a file and map to Mmap. + let decoded = Decodable::decode(d); + EncodedMetadata { mmap: None, decoded } } } From 336af60eae7cd272f702f0420723a733008f0161 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Fri, 29 Apr 2022 13:50:27 +0900 Subject: [PATCH 07/32] write to a temporary file in Decodable for EncodedMetadata --- compiler/rustc_metadata/src/fs.rs | 14 ++++---- compiler/rustc_metadata/src/rmeta/encoder.rs | 37 +++++++++++++------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index fa97cda0aa361..f63734384910f 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -80,7 +80,7 @@ pub fn encode_and_write_metadata( let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); - let metadata_filename = if need_metadata_file { + let (metadata_filename, metadata_tmpdir) = if need_metadata_file { if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); } @@ -90,14 +90,14 @@ pub fn encode_and_write_metadata( .span_diagnostic .emit_artifact_notification(&out_filename, "metadata"); } - out_filename + (out_filename, None) } else { - metadata_filename + (metadata_filename, Some(metadata_tmpdir)) }; - let file = std::fs::File::open(metadata_filename).unwrap(); - let metadata = EncodedMetadata::from_file(file).unwrap_or_else(|e| { - tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) - }); + let metadata = + EncodedMetadata::from_path(metadata_filename, metadata_tmpdir).unwrap_or_else(|e| { + tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) + }); let need_metadata_module = metadata_kind == MetadataKind::Compressed; diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 6e2044d4bb007..68da815dc3065 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -7,6 +7,7 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; use rustc_data_structures::memmap::Mmap; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{join, par_iter, Lrc, ParallelIterator}; +use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{ @@ -39,9 +40,11 @@ use rustc_span::{ use rustc_target::abi::VariantIdx; use std::borrow::Borrow; use std::hash::Hash; +use std::io::Write; use std::iter; use std::num::NonZeroUsize; -use std::path::Path; +use std::path::{Path, PathBuf}; +use tempfile::Builder as TempFileBuilder; use tracing::{debug, trace}; pub(super) struct EncodeContext<'a, 'tcx> { @@ -2138,25 +2141,25 @@ fn prefetch_mir(tcx: TyCtxt<'_>) { pub struct EncodedMetadata { mmap: Option, - decoded: Vec, + // We need to carry MaybeTempDir to avoid deleting the temporary + // directory while accessing the Mmap. + _temp_dir: Option, } impl EncodedMetadata { #[inline] - pub fn from_file(file: std::fs::File) -> std::io::Result { + pub fn from_path(path: PathBuf, temp_dir: Option) -> std::io::Result { + let file = std::fs::File::open(&path)?; let file_metadata = file.metadata()?; if file_metadata.len() == 0 { - return Ok(Self { mmap: None, decoded: Vec::new() }); + return Ok(Self { mmap: None, _temp_dir: temp_dir }); } let mmap = unsafe { Some(Mmap::map(file)?) }; - Ok(Self { mmap, decoded: Vec::new() }) + Ok(Self { mmap, _temp_dir: temp_dir }) } #[inline] pub fn raw_data(&self) -> &[u8] { - if !self.decoded.is_empty() { - return &self.decoded; - } self.mmap.as_ref().map(|mmap| mmap.as_ref()).unwrap_or_default() } } @@ -2170,9 +2173,19 @@ impl Encodable for EncodedMetadata { impl Decodable for EncodedMetadata { fn decode(d: &mut D) -> Self { - // FIXME: Write decorded data to a file and map to Mmap. - let decoded = Decodable::decode(d); - EncodedMetadata { mmap: None, decoded } + let temp_dir = TempFileBuilder::new().prefix("decoded").tempdir().unwrap(); + let temp_dir = MaybeTempDir::new(temp_dir, false); + let filename = temp_dir.as_ref().join("decoded"); + let file = std::fs::File::create(&filename).unwrap(); + let mut file = std::io::BufWriter::new(file); + + let len = d.read_usize(); + for _ in 0..len { + file.write(&[d.read_u8()]).unwrap(); + } + file.flush().unwrap(); + + Self::from_path(filename, Some(temp_dir)).unwrap() } } @@ -2269,5 +2282,5 @@ pub fn provide(providers: &mut Providers) { }, ..*providers - }; + } } From 34f888941ecd7385d2751158dbad0a623a8e29cf Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Fri, 29 Apr 2022 15:56:04 +0900 Subject: [PATCH 08/32] seek and write the root position to the metadata file --- compiler/rustc_metadata/src/rmeta/encoder.rs | 25 ++++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 68da815dc3065..2d038fba17ad9 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -40,7 +40,7 @@ use rustc_span::{ use rustc_target::abi::VariantIdx; use std::borrow::Borrow; use std::hash::Hash; -use std::io::Write; +use std::io::{Seek, Write}; use std::iter; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; @@ -2165,7 +2165,7 @@ impl EncodedMetadata { } impl Encodable for EncodedMetadata { - fn encode(&self, s: &mut S) -> Result<(), S::Error> { + fn encode(&self, s: &mut S) { let slice = self.raw_data(); slice.encode(s) } @@ -2248,20 +2248,25 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) { let root = ecx.encode_crate_root(); ecx.opaque.flush(); - let mut result = std::fs::read(path.as_ref()).unwrap(); + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(path.as_ref()) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to open the file: {}", err))); // Encode the root position. let header = METADATA_HEADER.len(); + file.seek(std::io::SeekFrom::Start(header as u64)) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to seek the file: {}", err))); let pos = root.position.get(); - result[header + 0] = (pos >> 24) as u8; - result[header + 1] = (pos >> 16) as u8; - result[header + 2] = (pos >> 8) as u8; - result[header + 3] = (pos >> 0) as u8; - - std::fs::write(path, &result).unwrap(); + file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8]) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to write to the file: {}", err))); // Record metadata size for self-profiling - tcx.prof.artifact_size("crate_metadata", "crate_metadata", result.len() as u64); + tcx.prof.artifact_size( + "crate_metadata", + "crate_metadata", + file.metadata().unwrap().len() as u64, + ); } pub fn provide(providers: &mut Providers) { From 1be58056e18be6ecc576faa2957536c43156c11f Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Fri, 29 Apr 2022 16:54:27 +0900 Subject: [PATCH 09/32] use BufReader for counting zero bytes --- compiler/rustc_metadata/src/rmeta/encoder.rs | 28 ++++++++++---------- compiler/rustc_serialize/src/opaque.rs | 4 +++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 2d038fba17ad9..3bd7b6115aef5 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -40,7 +40,7 @@ use rustc_span::{ use rustc_target::abi::VariantIdx; use std::borrow::Borrow; use std::hash::Hash; -use std::io::{Seek, Write}; +use std::io::{Read, Seek, Write}; use std::iter; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; @@ -734,12 +734,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { assert_eq!(total_bytes, computed_total_bytes); if tcx.sess.meta_stats() { - // let mut zero_bytes = 0; - // for e in self.opaque.data.iter() { - // if *e == 0 { - // zero_bytes += 1; - // } - // } + let mut zero_bytes = 0; + let file = std::io::BufReader::new(self.opaque.file()); + for e in file.bytes() { + if e.unwrap() == 0 { + zero_bytes += 1; + } + } let perc = |bytes| (bytes * 100) as f64 / total_bytes as f64; let p = |label, bytes| { @@ -747,13 +748,12 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { }; eprintln!(""); - // FIXME print zero bytes - //eprintln!( - // "{} metadata bytes, of which {} bytes ({:.1}%) are zero", - // total_bytes, - // zero_bytes, - // perc(zero_bytes) - //); + eprintln!( + "{} metadata bytes, of which {} bytes ({:.1}%) are zero", + total_bytes, + zero_bytes, + perc(zero_bytes) + ); p("preamble", preamble_bytes); p("dep", dep_bytes); p("lib feature", lib_feature_bytes); diff --git a/compiler/rustc_serialize/src/opaque.rs b/compiler/rustc_serialize/src/opaque.rs index 366efe9cfa519..5c17ef6ace2d5 100644 --- a/compiler/rustc_serialize/src/opaque.rs +++ b/compiler/rustc_serialize/src/opaque.rs @@ -297,6 +297,10 @@ impl FileEncoder { } } + pub fn file(&self) -> &File { + &self.file + } + #[inline] fn capacity(&self) -> usize { self.buf.len() From aa8e06458e07ddfe08085174ccdc2bd83debe876 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Fri, 29 Apr 2022 16:55:54 +0900 Subject: [PATCH 10/32] add Send to the argument type of encode_metadata --- compiler/rustc_metadata/src/rmeta/encoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 3bd7b6115aef5..96bc2b4f8f1de 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2189,7 +2189,7 @@ impl Decodable for EncodedMetadata { } } -pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef) { +pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef + Send) { let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata"); // Since encoding metadata is not in a query, and nothing is cached, From 1c2cc687a7b39e8ca2ed52649185f438a52ced3a Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Fri, 29 Apr 2022 17:35:53 +0900 Subject: [PATCH 11/32] fix a typo --- compiler/rustc_codegen_ssa/src/back/link.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index ab794fd4d274f..8c485813b2db2 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -234,7 +234,7 @@ pub fn each_linked_rlib( Ok(()) } -/// Create an 'arlib'. +/// Create an 'rlib'. /// /// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains /// the object file of the crate, but it also contains all of the object files from native From 5aac75a8da40e66a9a127905e71379908b743d71 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 2 May 2022 11:50:35 +0900 Subject: [PATCH 12/32] import all necessary parts of rustc_metadata::fs --- compiler/rustc_codegen_ssa/src/back/link.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 8c485813b2db2..1051b1eb59389 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -6,6 +6,7 @@ use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_errors::{ErrorGuaranteed, Handler}; use rustc_fs_util::fix_windows_verbatim_for_gcc; use rustc_hir::def_id::CrateNum; +use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME}; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::middle::exported_symbols::SymbolExportKind; use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Strip}; @@ -257,7 +258,7 @@ fn link_rlib<'a, B: ArchiveBuilder<'a>>( RlibFlavor::Normal => { let (metadata, metadata_position) = create_rmeta_file(sess, codegen_results.metadata.raw_data()); - let metadata = rustc_metadata::fs::emit_metadata(sess, &metadata, tmpdir); + let metadata = emit_metadata(sess, &metadata, tmpdir); match metadata_position { MetadataPosition::First => { // Most of the time metadata in rlib files is wrapped in a "dummy" object @@ -483,7 +484,7 @@ fn link_staticlib<'a, B: ArchiveBuilder<'a>>( ab.add_archive(path, move |fname: &str| { // Ignore metadata files, no matter the name. - if fname == rustc_metadata::fs::METADATA_FILENAME { + if fname == METADATA_FILENAME { return true; } @@ -2455,7 +2456,7 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( let mut archive = ::new(sess, &dst); if let Err(e) = archive.add_archive(cratepath, move |f| { - if f == rustc_metadata::fs::METADATA_FILENAME { + if f == METADATA_FILENAME { return true; } From b28b7c90cb5b11d0bba77a86fe119f5b775fe289 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 2 May 2022 14:06:25 +0900 Subject: [PATCH 13/32] remove non_durable_rename in rustc_interface::util --- compiler/rustc_interface/src/util.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index fb9258eb4a938..cbc3ceedc4977 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -655,24 +655,6 @@ pub fn build_output_filenames( } } -#[cfg(not(target_os = "linux"))] -pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> { - std::fs::rename(src, dst) -} - -/// This function attempts to bypass the auto_da_alloc heuristic implemented by some filesystems -/// such as btrfs and ext4. When renaming over a file that already exists then they will "helpfully" -/// write back the source file before committing the rename in case a developer forgot some of -/// the fsyncs in the open/write/fsync(file)/rename/fsync(dir) dance for atomic file updates. -/// -/// To avoid triggering this heuristic we delete the destination first, if it exists. -/// The cost of an extra syscall is much lower than getting descheduled for the sync IO. -#[cfg(target_os = "linux")] -pub fn non_durable_rename(src: &Path, dst: &Path) -> std::io::Result<()> { - let _ = std::fs::remove_file(dst); - std::fs::rename(src, dst) -} - /// Returns a version string such as "1.46.0 (04488afe3 2020-08-24)" pub fn version_str() -> Option<&'static str> { option_env!("CFG_VERSION") From c8e49e4b25a10ebf9333742c3bb0724edf1ef2d9 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 2 May 2022 15:45:36 +0900 Subject: [PATCH 14/32] write the root position at the end --- compiler/rustc_metadata/src/rmeta/decoder.rs | 4 ++-- compiler/rustc_metadata/src/rmeta/encoder.rs | 24 +++++++------------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 3280fd5c3108b..37ccf592aa5a8 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -682,13 +682,13 @@ impl MetadataBlob { } pub(crate) fn get_rustc_version(&self) -> String { - LazyValue::::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4).unwrap()) + LazyValue::::from_position(NonZeroUsize::new(METADATA_HEADER.len()).unwrap()) .decode(self) } pub(crate) fn get_root(&self) -> CrateRoot { let slice = &self.blob()[..]; - let offset = METADATA_HEADER.len(); + let offset = slice.len() - 4; let pos = (((slice[offset + 0] as u32) << 24) | ((slice[offset + 1] as u32) << 16) | ((slice[offset + 2] as u32) << 8) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 96bc2b4f8f1de..c0750abf18873 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -40,7 +40,7 @@ use rustc_span::{ use rustc_target::abi::VariantIdx; use std::borrow::Borrow; use std::hash::Hash; -use std::io::{Read, Seek, Write}; +use std::io::{Read, Write}; use std::iter; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; @@ -2215,9 +2215,6 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) { .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err))); encoder.emit_raw_bytes(METADATA_HEADER); - // Will be filled with the root position after encoding everything. - encoder.emit_raw_bytes(&[0, 0, 0, 0]); - let source_map_files = tcx.sess.source_map().files(); let source_file_cache = (source_map_files[0].clone(), 0); let required_source_files = Some(GrowableBitSet::with_capacity(source_map_files.len())); @@ -2247,25 +2244,20 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) { // culminating in the `CrateRoot` which points to all of it. let root = ecx.encode_crate_root(); - ecx.opaque.flush(); - let mut file = std::fs::OpenOptions::new() - .write(true) - .open(path.as_ref()) - .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to open the file: {}", err))); - // Encode the root position. - let header = METADATA_HEADER.len(); - file.seek(std::io::SeekFrom::Start(header as u64)) - .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to seek the file: {}", err))); let pos = root.position.get(); - file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8]) - .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to write to the file: {}", err))); + ecx.opaque.emit_raw_bytes(&[ + (pos >> 24) as u8, + (pos >> 16) as u8, + (pos >> 8) as u8, + (pos >> 0) as u8, + ]); // Record metadata size for self-profiling tcx.prof.artifact_size( "crate_metadata", "crate_metadata", - file.metadata().unwrap().len() as u64, + ecx.opaque.file().metadata().unwrap().len() as u64, ); } From 1ead92dd64d256ec631fb05eb4d5723c179533cb Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 2 May 2022 16:37:26 +0900 Subject: [PATCH 15/32] call EncodedMetadata::empty in case of MetadataKind::None --- compiler/rustc_metadata/src/fs.rs | 56 +++++++++++--------- compiler/rustc_metadata/src/rmeta/encoder.rs | 5 ++ 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index f63734384910f..ed58d4595a083 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -68,37 +68,43 @@ pub fn encode_and_write_metadata( .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); - match metadata_kind { + let metadata = match metadata_kind { MetadataKind::None => { - let _ = emit_metadata(tcx.sess, &[], &metadata_tmpdir); + if tcx.sess.opts.json_artifact_notifications { + tcx.sess + .parse_sess + .span_diagnostic + .emit_artifact_notification(&out_filename, "metadata"); + } + EncodedMetadata::empty() } MetadataKind::Uncompressed | MetadataKind::Compressed => { - encode_metadata(tcx, &metadata_filename) + encode_metadata(tcx, &metadata_filename); + + let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); + + let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); + let (metadata_filename, metadata_tmpdir) = if need_metadata_file { + if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { + tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); + } + if tcx.sess.opts.json_artifact_notifications { + tcx.sess + .parse_sess + .span_diagnostic + .emit_artifact_notification(&out_filename, "metadata"); + } + (out_filename, None) + } else { + (metadata_filename, Some(metadata_tmpdir)) + }; + + EncodedMetadata::from_path(metadata_filename, metadata_tmpdir).unwrap_or_else(|e| { + tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) + }) } }; - let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); - - let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); - let (metadata_filename, metadata_tmpdir) = if need_metadata_file { - if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { - tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); - } - if tcx.sess.opts.json_artifact_notifications { - tcx.sess - .parse_sess - .span_diagnostic - .emit_artifact_notification(&out_filename, "metadata"); - } - (out_filename, None) - } else { - (metadata_filename, Some(metadata_tmpdir)) - }; - let metadata = - EncodedMetadata::from_path(metadata_filename, metadata_tmpdir).unwrap_or_else(|e| { - tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) - }); - let need_metadata_module = metadata_kind == MetadataKind::Compressed; (metadata, need_metadata_module) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index c0750abf18873..fc069a233f3bc 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2158,6 +2158,11 @@ impl EncodedMetadata { Ok(Self { mmap, _temp_dir: temp_dir }) } + #[inline] + pub fn empty() -> Self { + Self { mmap: None, _temp_dir: None } + } + #[inline] pub fn raw_data(&self) -> &[u8] { self.mmap.as_ref().map(|mmap| mmap.as_ref()).unwrap_or_default() From e11dd802c1aa07ed61de2fdc8a7c79bae1e99f0a Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 2 May 2022 16:44:59 +0900 Subject: [PATCH 16/32] seek before counting zero bytes --- compiler/rustc_metadata/src/rmeta/encoder.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index fc069a233f3bc..7164230550493 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -40,7 +40,7 @@ use rustc_span::{ use rustc_target::abi::VariantIdx; use std::borrow::Borrow; use std::hash::Hash; -use std::io::{Read, Write}; +use std::io::{Read, Seek, Write}; use std::iter; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; @@ -735,6 +735,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { if tcx.sess.meta_stats() { let mut zero_bytes = 0; + self.opaque.file().seek(std::io::SeekFrom::Start(0)).unwrap(); let file = std::io::BufReader::new(self.opaque.file()); for e in file.bytes() { if e.unwrap() == 0 { From 8d35ff16d2c63d9dc9d9d98d9ebcda4840b08d4b Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 2 May 2022 19:59:30 +0900 Subject: [PATCH 17/32] insert dummy 4 bytes to avoid the breaking change --- compiler/rustc_metadata/src/rmeta/decoder.rs | 2 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 37ccf592aa5a8..990ff9e6b40a4 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -682,7 +682,7 @@ impl MetadataBlob { } pub(crate) fn get_rustc_version(&self) -> String { - LazyValue::::from_position(NonZeroUsize::new(METADATA_HEADER.len()).unwrap()) + LazyValue::::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4).unwrap()) .decode(self) } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 7164230550493..d7991e6a660c5 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2221,6 +2221,11 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) { .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err))); encoder.emit_raw_bytes(METADATA_HEADER); + // Though we had holded the root position historically in this place, we moved it to the end + // of all emitted bytes by #96544. Therefore, now these 4 bytes are just a dummy to avoid the + // breaking change. + encoder.emit_raw_bytes(&[0, 0, 0, 0]).unwrap(); + let source_map_files = tcx.sess.source_map().files(); let source_file_cache = (source_map_files[0].clone(), 0); let required_source_files = Some(GrowableBitSet::with_capacity(source_map_files.len())); From dcb599fe6c10a21d18f4f5287ef257ee76448864 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Thu, 5 May 2022 16:40:29 +0900 Subject: [PATCH 18/32] create tmp directory if there is no parent directory --- compiler/rustc_metadata/src/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index ed58d4595a083..ed42efcfd774c 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -64,7 +64,7 @@ pub fn encode_and_write_metadata( // which is why we create it inside the output directory specifically. let metadata_tmpdir = TempFileBuilder::new() .prefix("rmeta") - .tempdir_in(out_filename.parent().unwrap_or_else(|| Path::new(""))) + .tempdir_in(out_filename.parent().unwrap_or_else(|| Path::new("tmp"))) .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); From 47c36893a14b689faadeaa06b56e53d36f526c2f Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 8 May 2022 11:24:48 +0900 Subject: [PATCH 19/32] use &Path instead of AsRef --- compiler/rustc_metadata/src/rmeta/encoder.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index d7991e6a660c5..7980cb6a1324b 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2195,7 +2195,7 @@ impl Decodable for EncodedMetadata { } } -pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef + Send) { +pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path) { let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata"); // Since encoding metadata is not in a query, and nothing is cached, @@ -2216,8 +2216,8 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: impl AsRef + Send) { ); } -fn encode_metadata_impl(tcx: TyCtxt<'_>, path: impl AsRef) { - let mut encoder = opaque::FileEncoder::new(path.as_ref()) +fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) { + let mut encoder = opaque::FileEncoder::new(path) .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err))); encoder.emit_raw_bytes(METADATA_HEADER); From c57d7788729f32e74966f67eea582ec7ec75a133 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 8 May 2022 12:36:12 +0900 Subject: [PATCH 20/32] define MmapMut and use it in Decodable impl --- compiler/rustc_data_structures/src/memmap.rs | 63 +++++++++++++++++++- compiler/rustc_metadata/src/rmeta/encoder.rs | 25 ++++---- 2 files changed, 74 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_data_structures/src/memmap.rs b/compiler/rustc_data_structures/src/memmap.rs index 26b26415eea0f..917416df6b867 100644 --- a/compiler/rustc_data_structures/src/memmap.rs +++ b/compiler/rustc_data_structures/src/memmap.rs @@ -1,6 +1,6 @@ use std::fs::File; use std::io; -use std::ops::Deref; +use std::ops::{Deref, DerefMut}; use crate::owning_ref::StableAddress; @@ -45,3 +45,64 @@ impl Deref for Mmap { // export any function that can cause the `Vec` to be re-allocated. As such the address of the // bytes inside this `Vec` is stable. unsafe impl StableAddress for Mmap {} + +#[cfg(not(target_arch = "wasm32"))] +pub struct MmapMut(memmap2::MmapMut); + +#[cfg(target_arch = "wasm32")] +pub struct MmapMut(Vec); + +#[cfg(not(target_arch = "wasm32"))] +impl MmapMut { + #[inline] + pub fn map_anon(len: usize) -> io::Result { + let mmap = memmap2::MmapMut::map_anon(len)?; + Ok(MmapMut(mmap)) + } + + #[inline] + pub fn flush(&mut self) -> io::Result<()> { + self.0.flush() + } + + #[inline] + pub fn make_read_only(self) -> std::io::Result { + let mmap = self.0.make_read_only()?; + Ok(Mmap(mmap)) + } +} + +#[cfg(target_arch = "wasm32")] +impl MmapMut { + #[inline] + pub fn map_anon(len: usize) -> io::Result { + let data = Vec::with_capacity(len); + Ok(MmapMut(data)) + } + + #[inline] + pub fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + + #[inline] + pub fn make_read_only(self) -> std::io::Result { + Ok(Mmap(self.0)) + } +} + +impl Deref for MmapMut { + type Target = [u8]; + + #[inline] + fn deref(&self) -> &[u8] { + &*self.0 + } +} + +impl DerefMut for MmapMut { + #[inline] + fn deref_mut(&mut self) -> &mut [u8] { + &mut *self.0 + } +} diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 7980cb6a1324b..f3853255f7bdb 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -4,7 +4,7 @@ use crate::rmeta::*; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; -use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::memmap::{Mmap, MmapMut}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{join, par_iter, Lrc, ParallelIterator}; use rustc_data_structures::temp_dir::MaybeTempDir; @@ -44,7 +44,6 @@ use std::io::{Read, Seek, Write}; use std::iter; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; -use tempfile::Builder as TempFileBuilder; use tracing::{debug, trace}; pub(super) struct EncodeContext<'a, 'tcx> { @@ -2179,19 +2178,19 @@ impl Encodable for EncodedMetadata { impl Decodable for EncodedMetadata { fn decode(d: &mut D) -> Self { - let temp_dir = TempFileBuilder::new().prefix("decoded").tempdir().unwrap(); - let temp_dir = MaybeTempDir::new(temp_dir, false); - let filename = temp_dir.as_ref().join("decoded"); - let file = std::fs::File::create(&filename).unwrap(); - let mut file = std::io::BufWriter::new(file); - let len = d.read_usize(); - for _ in 0..len { - file.write(&[d.read_u8()]).unwrap(); - } - file.flush().unwrap(); + let mmap = if len > 0 { + let mut mmap = MmapMut::map_anon(len).unwrap(); + for _ in 0..len { + (&mut mmap[..]).write(&[d.read_u8()]).unwrap(); + } + mmap.flush().unwrap(); + Some(mmap.make_read_only().unwrap()) + } else { + None + }; - Self::from_path(filename, Some(temp_dir)).unwrap() + Self { mmap, _temp_dir: None } } } From 7eb64b4901f43d01aa6bd9a39ef299c0380fd600 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 8 May 2022 14:19:04 +0900 Subject: [PATCH 21/32] flush and assert when counting zero bytes --- compiler/rustc_metadata/src/rmeta/encoder.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index f3853255f7bdb..3ae4fbb70a5eb 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -733,14 +733,18 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { assert_eq!(total_bytes, computed_total_bytes); if tcx.sess.meta_stats() { + self.opaque.flush().unwrap(); + + let pos_before_rewind = self.opaque.file().stream_position().unwrap(); let mut zero_bytes = 0; - self.opaque.file().seek(std::io::SeekFrom::Start(0)).unwrap(); + self.opaque.file().rewind().unwrap(); let file = std::io::BufReader::new(self.opaque.file()); for e in file.bytes() { if e.unwrap() == 0 { zero_bytes += 1; } } + assert_eq!(self.opaque.file().stream_position().unwrap(), pos_before_rewind); let perc = |bytes| (bytes * 100) as f64 / total_bytes as f64; let p = |label, bytes| { From 03de5c4a14e58f45fb411b9369f98508473416ae Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 30 May 2022 00:46:12 +0900 Subject: [PATCH 22/32] create an empty file even in case of MetadataKind::None --- compiler/rustc_metadata/src/fs.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index ed42efcfd774c..b08b1247c6dee 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -68,13 +68,24 @@ pub fn encode_and_write_metadata( .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); + let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); let metadata = match metadata_kind { MetadataKind::None => { - if tcx.sess.opts.json_artifact_notifications { - tcx.sess - .parse_sess - .span_diagnostic - .emit_artifact_notification(&out_filename, "metadata"); + if need_metadata_file { + // Though creating the empty file here seems to be meaningless, cargo expects it. + if let Err(e) = std::fs::File::create(&out_filename) { + tcx.sess.fatal(&format!( + "failed to create the file {}: {}", + out_filename.display(), + e + )); + } + if tcx.sess.opts.json_artifact_notifications { + tcx.sess + .parse_sess + .span_diagnostic + .emit_artifact_notification(&out_filename, "metadata"); + } } EncodedMetadata::empty() } @@ -83,7 +94,6 @@ pub fn encode_and_write_metadata( let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); - let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); let (metadata_filename, metadata_tmpdir) = if need_metadata_file { if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); From ec64b4c90ed7b277b9946e6d10bea25befe13488 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 30 May 2022 00:49:26 +0900 Subject: [PATCH 23/32] add a comment about the drop order for EncodedMetadata --- compiler/rustc_metadata/src/rmeta/encoder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 3ae4fbb70a5eb..b58f95b058b8c 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2144,6 +2144,7 @@ fn prefetch_mir(tcx: TyCtxt<'_>) { // generated regardless of trailing bytes that end up in it. pub struct EncodedMetadata { + // The declaration order matters because `mmap` should be dropped before `_temp_dir`. mmap: Option, // We need to carry MaybeTempDir to avoid deleting the temporary // directory while accessing the Mmap. From 34e44e577485da3f8de4803a1f9248fa6c0089b8 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 30 May 2022 01:12:05 +0900 Subject: [PATCH 24/32] Revert "write the root position at the end" This reverts commit 44f66429e1fdba2cd167b4033f04f462a368b717. --- compiler/rustc_metadata/src/rmeta/decoder.rs | 2 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 27 +++++++++++--------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 990ff9e6b40a4..3280fd5c3108b 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -688,7 +688,7 @@ impl MetadataBlob { pub(crate) fn get_root(&self) -> CrateRoot { let slice = &self.blob()[..]; - let offset = slice.len() - 4; + let offset = METADATA_HEADER.len(); let pos = (((slice[offset + 0] as u32) << 24) | ((slice[offset + 1] as u32) << 16) | ((slice[offset + 2] as u32) << 8) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index b58f95b058b8c..cce59e06bf7b8 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -733,7 +733,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { assert_eq!(total_bytes, computed_total_bytes); if tcx.sess.meta_stats() { - self.opaque.flush().unwrap(); + self.opaque.flush(); let pos_before_rewind = self.opaque.file().stream_position().unwrap(); let mut zero_bytes = 0; @@ -2225,10 +2225,8 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) { .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to create file encoder: {}", err))); encoder.emit_raw_bytes(METADATA_HEADER); - // Though we had holded the root position historically in this place, we moved it to the end - // of all emitted bytes by #96544. Therefore, now these 4 bytes are just a dummy to avoid the - // breaking change. - encoder.emit_raw_bytes(&[0, 0, 0, 0]).unwrap(); + // Will be filled with the root position after encoding everything. + encoder.emit_raw_bytes(&[0, 0, 0, 0]); let source_map_files = tcx.sess.source_map().files(); let source_file_cache = (source_map_files[0].clone(), 0); @@ -2259,20 +2257,25 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) { // culminating in the `CrateRoot` which points to all of it. let root = ecx.encode_crate_root(); + ecx.opaque.flush(); + let mut file = std::fs::OpenOptions::new() + .write(true) + .open(path) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to open the file: {}", err))); + // Encode the root position. + let header = METADATA_HEADER.len(); + file.seek(std::io::SeekFrom::Start(header as u64)) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to seek the file: {}", err))); let pos = root.position.get(); - ecx.opaque.emit_raw_bytes(&[ - (pos >> 24) as u8, - (pos >> 16) as u8, - (pos >> 8) as u8, - (pos >> 0) as u8, - ]); + file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8]) + .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to write to the file: {}", err))); // Record metadata size for self-profiling tcx.prof.artifact_size( "crate_metadata", "crate_metadata", - ecx.opaque.file().metadata().unwrap().len() as u64, + file.metadata().unwrap().len() as u64, ); } From 0928061906bbec3099df7340e1e296b6dfc279a9 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Wed, 1 Jun 2022 00:14:42 +0900 Subject: [PATCH 25/32] refactor encode_and_write_metadata --- compiler/rustc_metadata/src/fs.rs | 70 +++++++++----------- compiler/rustc_metadata/src/rmeta/encoder.rs | 5 -- 2 files changed, 31 insertions(+), 44 deletions(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index b08b1247c6dee..4e00cac466b14 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -68,53 +68,45 @@ pub fn encode_and_write_metadata( .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); - let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); - let metadata = match metadata_kind { + + match metadata_kind { MetadataKind::None => { - if need_metadata_file { - // Though creating the empty file here seems to be meaningless, cargo expects it. - if let Err(e) = std::fs::File::create(&out_filename) { - tcx.sess.fatal(&format!( - "failed to create the file {}: {}", - out_filename.display(), - e - )); - } - if tcx.sess.opts.json_artifact_notifications { - tcx.sess - .parse_sess - .span_diagnostic - .emit_artifact_notification(&out_filename, "metadata"); - } - } - EncodedMetadata::empty() + std::fs::File::create(&metadata_filename).unwrap_or_else(|e| { + tcx.sess.fatal(&format!( + "failed to create the file {}: {}", + out_filename.display(), + e + )) + }); } MetadataKind::Uncompressed | MetadataKind::Compressed => { encode_metadata(tcx, &metadata_filename); + } + }; - let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); - - let (metadata_filename, metadata_tmpdir) = if need_metadata_file { - if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { - tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); - } - if tcx.sess.opts.json_artifact_notifications { - tcx.sess - .parse_sess - .span_diagnostic - .emit_artifact_notification(&out_filename, "metadata"); - } - (out_filename, None) - } else { - (metadata_filename, Some(metadata_tmpdir)) - }; - - EncodedMetadata::from_path(metadata_filename, metadata_tmpdir).unwrap_or_else(|e| { - tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) - }) + let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); + + let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); + let (metadata_filename, metadata_tmpdir) = if need_metadata_file { + if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { + tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e)); } + if tcx.sess.opts.json_artifact_notifications { + tcx.sess + .parse_sess + .span_diagnostic + .emit_artifact_notification(&out_filename, "metadata"); + } + (out_filename, None) + } else { + (metadata_filename, Some(metadata_tmpdir)) }; + let metadata = + EncodedMetadata::from_path(metadata_filename, metadata_tmpdir).unwrap_or_else(|e| { + tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) + }); + let need_metadata_module = metadata_kind == MetadataKind::Compressed; (metadata, need_metadata_module) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index cce59e06bf7b8..0da87abbc487a 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2163,11 +2163,6 @@ impl EncodedMetadata { Ok(Self { mmap, _temp_dir: temp_dir }) } - #[inline] - pub fn empty() -> Self { - Self { mmap: None, _temp_dir: None } - } - #[inline] pub fn raw_data(&self) -> &[u8] { self.mmap.as_ref().map(|mmap| mmap.as_ref()).unwrap_or_default() From 63dec941e1fb1a70e7dba9043e92a856cf7cd01a Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 5 Jun 2022 22:19:36 +0900 Subject: [PATCH 26/32] fix an incorrect filename for an error message --- compiler/rustc_metadata/src/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 4e00cac466b14..1db620420572c 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -74,7 +74,7 @@ pub fn encode_and_write_metadata( std::fs::File::create(&metadata_filename).unwrap_or_else(|e| { tcx.sess.fatal(&format!( "failed to create the file {}: {}", - out_filename.display(), + metadata_filename.display(), e )) }); From ad55481043d780d341ee526a0902e080646aae4f Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 5 Jun 2022 22:23:06 +0900 Subject: [PATCH 27/32] add some comments for encode_and_write_metadata --- compiler/rustc_metadata/src/fs.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 1db620420572c..7661eec7006b3 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -69,6 +69,8 @@ pub fn encode_and_write_metadata( let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); + // Always create a file at `metadata_filename`, even if we have nothing to write to it. + // This simplifies the creation of the output `out_filename` when requested. match metadata_kind { MetadataKind::None => { std::fs::File::create(&metadata_filename).unwrap_or_else(|e| { @@ -86,6 +88,9 @@ pub fn encode_and_write_metadata( let _prof_timer = tcx.sess.prof.generic_activity("write_crate_metadata"); + // If the user requests metadata as output, rename `metadata_filename` + // to the expected output `out_filename`. The match above should ensure + // this file always exists. let need_metadata_file = tcx.sess.opts.output_types.contains_key(&OutputType::Metadata); let (metadata_filename, metadata_tmpdir) = if need_metadata_file { if let Err(e) = non_durable_rename(&metadata_filename, &out_filename) { @@ -102,6 +107,7 @@ pub fn encode_and_write_metadata( (metadata_filename, Some(metadata_tmpdir)) }; + // Load metadata back to memory: codegen may need to include it in object files. let metadata = EncodedMetadata::from_path(metadata_filename, metadata_tmpdir).unwrap_or_else(|e| { tcx.sess.fatal(&format!("failed to create encoded metadata from file: {}", e)) From 3e309350d2dcac7f6e28754008fc899dea88bae9 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 5 Jun 2022 22:26:20 +0900 Subject: [PATCH 28/32] add a comment about counting zero bytes --- compiler/rustc_metadata/src/rmeta/encoder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 0da87abbc487a..d2157ebe7dca9 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -735,6 +735,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { if tcx.sess.meta_stats() { self.opaque.flush(); + // Rewind and re-read all the metadata to count the zero bytes we wrote. let pos_before_rewind = self.opaque.file().stream_position().unwrap(); let mut zero_bytes = 0; self.opaque.file().rewind().unwrap(); From ea0e0f4e13a576dc0a31a0156281fbffceffa379 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 6 Jun 2022 23:13:54 +0900 Subject: [PATCH 29/32] avoid holding the temp_dir for empty metadata file --- compiler/rustc_metadata/src/rmeta/encoder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index d2157ebe7dca9..2b5bafb8b9cc3 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2158,7 +2158,7 @@ impl EncodedMetadata { let file = std::fs::File::open(&path)?; let file_metadata = file.metadata()?; if file_metadata.len() == 0 { - return Ok(Self { mmap: None, _temp_dir: temp_dir }); + return Ok(Self { mmap: None, _temp_dir: None }); } let mmap = unsafe { Some(Mmap::map(file)?) }; Ok(Self { mmap, _temp_dir: temp_dir }) From a206121388524c6b0d5b8c5f97fc732e4ad16709 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 6 Jun 2022 23:43:44 +0900 Subject: [PATCH 30/32] seek instead of opening a new file handle --- compiler/rustc_metadata/src/rmeta/encoder.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 2b5bafb8b9cc3..41de298bcfd8d 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2254,10 +2254,10 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) { let root = ecx.encode_crate_root(); ecx.opaque.flush(); - let mut file = std::fs::OpenOptions::new() - .write(true) - .open(path) - .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to open the file: {}", err))); + + let mut file = ecx.opaque.file(); + // We will return to this position after writing the root position. + let pos_before_seek = file.stream_position().unwrap(); // Encode the root position. let header = METADATA_HEADER.len(); @@ -2267,6 +2267,9 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) { file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8]) .unwrap_or_else(|err| tcx.sess.fatal(&format!("failed to write to the file: {}", err))); + // Return to the position where we are before writing the root position. + file.seek(std::io::SeekFrom::Start(pos_before_seek)).unwrap(); + // Record metadata size for self-profiling tcx.prof.artifact_size( "crate_metadata", From 7a5e773dc23bb485df1655ca01d59cc55e435b6a Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sat, 25 Jun 2022 01:12:08 +0900 Subject: [PATCH 31/32] fall back on the blank path if out_filename is blank --- compiler/rustc_metadata/src/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs index 7661eec7006b3..e6072901aaa43 100644 --- a/compiler/rustc_metadata/src/fs.rs +++ b/compiler/rustc_metadata/src/fs.rs @@ -64,7 +64,7 @@ pub fn encode_and_write_metadata( // which is why we create it inside the output directory specifically. let metadata_tmpdir = TempFileBuilder::new() .prefix("rmeta") - .tempdir_in(out_filename.parent().unwrap_or_else(|| Path::new("tmp"))) + .tempdir_in(out_filename.parent().unwrap_or_else(|| Path::new(""))) .unwrap_or_else(|err| tcx.sess.fatal(&format!("couldn't create a temp dir: {}", err))); let metadata_tmpdir = MaybeTempDir::new(metadata_tmpdir, tcx.sess.opts.cg.save_temps); let metadata_filename = metadata_tmpdir.as_ref().join(METADATA_FILENAME); From 1147d50050788851438e80c57279b99b9a457924 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Sun, 3 Jul 2022 23:02:55 +0900 Subject: [PATCH 32/32] ensure rustc does not panic by the test for issue-26092 --- src/test/run-make-fulldeps/issue-26092/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/run-make-fulldeps/issue-26092/Makefile b/src/test/run-make-fulldeps/issue-26092/Makefile index 27631c31c4a06..885f45a022431 100644 --- a/src/test/run-make-fulldeps/issue-26092/Makefile +++ b/src/test/run-make-fulldeps/issue-26092/Makefile @@ -1,4 +1,6 @@ -include ../tools.mk +# This test ensures that rustc does not panic with `-o ""` option. + all: - $(RUSTC) -o "" blank.rs 2>&1 | $(CGREP) -i 'No such file or directory' + $(RUSTC) -o "" blank.rs 2>&1 | $(CGREP) -i 'panic' && exit 1 || exit 0