Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions compiler/rustc_incremental/src/persist/file_format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ const FILE_MAGIC: &[u8] = b"RSIC";
/// Change this if the header format changes.
// CGUs now encode a placement key separately from their versioned artifact identity.
// Local built MIR now fingerprints the type-checking root's region annotation.
// OwnerNodes now records whether it contains loop-control attributes.
const HEADER_FORMAT_VERSION: u16 = 61340;
// OwnerNodes records loop-control attributes and params_in_repr fingerprints the full bitset.
const HEADER_FORMAT_VERSION: u16 = 61341;

pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) {
stream.emit_raw_bytes(FILE_MAGIC);
Expand Down
13 changes: 13 additions & 0 deletions compiler/rustc_metadata/src/rmeta/decoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1183,6 +1183,19 @@ impl CrateRoot {
self.stable_crate_id
}

/// Return the typed dependencies with the positions of their fixed-width SVHs.
pub(super) fn crate_deps_with_hash_positions<'a>(
&self,
metadata: &'a MetadataBlob,
) -> impl ExactSizeIterator<Item = (CrateDep, usize)> + 'a {
let mut decoder = metadata.decoder(self.crate_deps.position.get());
(0..self.crate_deps.num_elems).map(move |_| {
let mut name = metadata.decoder(decoder.opaque.position());
let _ = name.decode_symbol();
(CrateDep::decode(&mut decoder), name.opaque.position())
})
}

pub(crate) fn decode_crate_deps<'a>(
&self,
metadata: &'a MetadataBlob,
Expand Down
76 changes: 69 additions & 7 deletions compiler/rustc_metadata/src/rmeta/encoder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2405,7 +2405,9 @@ impl<'a, 'tcx, M: MetadataEncoder<'tcx>> EncodeContext<'a, 'tcx, M> {
}
if should_encode_visibility(def_kind) {
let vis = tcx.local_visibility(local_id);
record!(self.tables.visibility[def_id] <- vis.map_id(|mod_id| mod_id.to_local_def_id().local_def_index), hcx, vis);
// Accessibility depends on the restricted module's identity. Hash that identity
// and write the usual local index without pulling the module's contents into it.
record!(self.tables.visibility[def_id] <- vis.map_id(|mod_id| mod_id.to_local_def_id().local_def_index), hcx, vis, |_| RecordMode::None);
}
if should_encode_stability(def_kind) {
self.encode_stability(def_id, hcx);
Expand Down Expand Up @@ -3243,7 +3245,13 @@ impl<'a, 'tcx, M: MetadataEncoder<'tcx>> EncodeContext<'a, 'tcx, M> {
) -> Hashed<LazyArray<CrateDep>> {
empty_proc_macro!(self);

let deps = crate_deps(self.tcx).collect::<Vec<_>>();
let refreshes_dependency_hashes = dependency_hash_refresh_allowed(self.tcx)
&& self.metadata_prefix_manifest.is_some()
&& MetadataPrefixManifest::clear_for_dependency_hashes(self.tcx.incr_comp_session);
if let Some(manifest) = &mut self.metadata_prefix_manifest {
manifest.refreshes_dependency_hashes = refreshes_dependency_hashes;
}
let deps = crate_deps(self.tcx, refreshes_dependency_hashes).collect::<Vec<_>>();

{
// Sanity-check the crate numbers
Expand Down Expand Up @@ -3824,7 +3832,8 @@ fn try_reuse_metadata_prefix(
&changed_dep_node,
DepKind::metadata_source_file_hash,
)?;
let manifest = MetadataPrefixManifest::load(&manifest_path)?;
let (manifest, refreshes_dependency_hashes) =
MetadataPrefixManifest::load(&manifest_path, dependency_hash_refresh_allowed(tcx))?;
let file = File::open(metadata_path).ok()?;
// SAFETY: The previous work-product remains immutable; replacement metadata is written to a
// freshly created inode and atomically renamed only after encoding completes.
Expand Down Expand Up @@ -3882,6 +3891,33 @@ fn try_reuse_metadata_prefix(
{
return None;
}
let mut updated_hashes = Vec::new();
if previous_root.crate_deps.num_elems != tcx.crates(()).len() {
return None;
}
let mut checked_end = preamble;
for ((mut previous, pos), (_, current, _)) in
previous_root.crate_deps_with_hash_positions(&metadata).zip(crate_deps(tcx, false))
{
let end = pos.checked_add(size_of::<u128>())?;
if pos < checked_end || end > manifest.prefix_position {
return None;
}
let old_svh = previous.hash;
if metadata.get(pos..end)? != old_svh.as_u128().to_le_bytes().as_slice() {
return None;
}
checked_end = end;
// A legacy v3 checksum covers only its payload. Verify its complete rows even if
// the previous graph appears green; only an authenticated v4 may rewrite a hash.
if old_svh != current.hash && refreshes_dependency_hashes {
previous.hash = current.hash;
updated_hashes.push((pos, current.hash.as_u128().to_le_bytes()));
}
if previous != current {
return None;
}
}
let previous_rdr_hashes = previous_root.rdr_hashes.take()?;
let local_hashes_range = previous_rdr_hashes.local.encoded_range(&metadata)?;
let expn_hashes_range = previous_rdr_hashes.expn.encoded_range(&metadata)?;
Expand All @@ -3907,10 +3943,17 @@ fn try_reuse_metadata_prefix(
let mut refreshed_source_files = FxIndexSet::default();
with_encode_metadata_header::<DefPathHashDefIdEncoder<'_>>(tcx, temporary_file.path(), |ecx| {
let preamble = ecx.position();
ecx.emit_raw_bytes(&metadata[preamble..prefix_position]);
let mut start = preamble;
for (pos, bytes) in updated_hashes {
ecx.emit_raw_bytes(&metadata[start..pos]);
ecx.emit_raw_bytes(&bytes);
start = pos + bytes.len();
}
ecx.emit_raw_bytes(&metadata[start..prefix_position]);
ecx.required_source_files = Some(required_source_files);
ecx.metadata_prefix_manifest = Some(MetadataPrefixManifestBuilder {
manifest,
refreshes_dependency_hashes,
protected_span_keys: FxIndexSet::default(),
definition_span_hasher: StableHasher::new(),
});
Expand Down Expand Up @@ -4238,7 +4281,9 @@ fn with_encode_metadata_header<'tcx, M: MetadataEncoder<'tcx>>(
{
builder.manifest.metadata_len = metadata.len();
builder.manifest.metadata_fingerprint = metadata_fingerprint(&metadata);
if let Err(error) = builder.manifest.save(tcx.incr_comp_session) {
if let Err(error) =
builder.manifest.save(tcx.incr_comp_session, builder.refreshes_dependency_hashes)
{
debug!(target: "ohm_metadata_reuse", ?error, "could not persist metadata prefix manifest");
}
}
Expand Down Expand Up @@ -4378,13 +4423,30 @@ pub fn rendered_const<'tcx>(tcx: TyCtxt<'tcx>, body: &hir::Body<'_>, def_id: Loc
}
}

fn crate_deps(tcx: TyCtxt<'_>) -> impl Iterator<Item = (CrateNum, CrateDep, Fingerprint)> + '_ {
fn dependency_hash_refresh_allowed(tcx: TyCtxt<'_>) -> bool {
tcx.dep_graph.is_fully_enabled()
&& tcx.sess.opts.unstable_opts.public_api_hash
&& !tcx.sess.opts.unstable_opts.incremental_ignore_spans
&& !tcx.sess.opts.unstable_opts.meta_stats
&& tcx.sess.opts.unstable_opts.embed_metadata
&& !tcx.sess.opts.output_types.should_codegen()
&& !tcx.crate_types().contains(&CrateType::ProcMacro)
}

fn crate_deps(
tcx: TyCtxt<'_>,
refreshes_dependency_hashes: bool,
) -> impl Iterator<Item = (CrateNum, CrateDep, Fingerprint)> + '_ {
tcx.crates(()).iter().map(move |&cnum| {
let dep = CrateDep {
name: tcx.crate_name(cnum),
// Ordinary metadata encodes raw foreign DefIndices and must bind to the
// exact producer. RDR metadata resolves foreign DefIds by DefPathHash.
hash: if tcx.sess.opts.unstable_opts.public_api_hash {
hash: if refreshes_dependency_hashes {
// This serialized field is separately refreshed. The metadata input hash covers all upstream
// identities and public SVHs, including changes to dependencies with no item reads.
tcx.dep_graph.with_ignore(|| tcx.public_api_hash(cnum))
} else if tcx.sess.opts.unstable_opts.public_api_hash {
tcx.public_api_hash(cnum)
} else {
tcx.crate_hash(cnum)
Expand Down
65 changes: 56 additions & 9 deletions compiler/rustc_metadata/src/rmeta/encoder/metadata_prefix.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,13 @@ use super::{DefPathHashDefIdEncoder, EncodeContext, EncodedMetadata};
use crate::rmeta::LazyTable;
use crate::rmeta::table::FixedSizeEncoding;

const METADATA_PREFIX_MAGIC: &[u8] = b"ohm-metadata-prefix-v3\0";
fn manifest_magic(refreshes_dependency_hashes: bool) -> &'static [u8] {
if refreshes_dependency_hashes {
b"ohm-metadata-prefix-v4\0"
} else {
b"ohm-metadata-prefix-v3\0"
}
}

#[derive(Encodable, Decodable)]
pub(super) struct MetadataProtectedSpan {
Expand All @@ -48,6 +54,7 @@ pub(super) struct MetadataPrefixManifest {
}

pub(super) struct MetadataPrefixManifestBuilder {
pub(super) refreshes_dependency_hashes: bool,
pub(super) manifest: MetadataPrefixManifest,
pub(super) protected_span_keys: FxIndexSet<(StableSourceFileId, u32, u32)>,
pub(super) definition_span_hasher: StableHasher,
Expand All @@ -59,6 +66,17 @@ pub(super) fn metadata_fingerprint(bytes: &[u8]) -> Fingerprint {
hasher.finish()
}

// A v4 prefix may omit dependency-hash reads from the graph. Protect this capability
// together with the payload. Released v3 manifests use the original payload-only hash.
fn manifest_fingerprint(payload: &[u8], refreshes_dependency_hashes: bool) -> Fingerprint {
if !refreshes_dependency_hashes {
return metadata_fingerprint(payload);
}
let mut hasher = StableHasher::new();
(manifest_magic(true), payload).hash(&mut hasher);
hasher.finish()
}

/// Copies a suffix table whose complete encoding was validated before opening the output file.
pub(super) fn relocate_metadata_table<'tcx, I, T: FixedSizeEncoding>(
ecx: &mut EncodeContext<'_, 'tcx, DefPathHashDefIdEncoder<'tcx>>,
Expand Down Expand Up @@ -246,6 +264,7 @@ impl MetadataPrefixManifestBuilder {
definition_span_count: 0,
definition_span_fingerprint: Fingerprint::ZERO,
},
refreshes_dependency_hashes: false,
protected_span_keys: FxIndexSet::default(),
definition_span_hasher: StableHasher::new(),
}
Expand Down Expand Up @@ -375,7 +394,7 @@ impl MetadataPrefixManifest {
bytes.is_empty().then_some(())
}

pub(super) fn load(path: &Path) -> Option<Self> {
pub(super) fn load(path: &Path, allow_dependency_hash_refresh: bool) -> Option<(Self, bool)> {
const MAX_MANIFEST_SIZE: u64 = 16 * 1024 * 1024;

let file = File::open(path).ok()?;
Expand All @@ -387,18 +406,46 @@ impl MetadataPrefixManifest {
if bytes.len() > MAX_MANIFEST_SIZE as usize {
return None;
}
let payload = bytes.strip_prefix(METADATA_PREFIX_MAGIC)?;
// A v3 prefix keeps its original tracked dependency hashes. Only v4
// delegates those fields to the refresh below the immutable item data.
let (payload, refreshes_dependency_hashes) = if allow_dependency_hash_refresh
&& let Some(payload) = bytes.strip_prefix(manifest_magic(true))
{
(payload, true)
} else {
(bytes.strip_prefix(manifest_magic(false))?, false)
};
let (expected_fingerprint, payload) = payload.split_first_chunk::<16>()?;
if metadata_fingerprint(payload) != Fingerprint::from_le_bytes(*expected_fingerprint) {
if manifest_fingerprint(payload, refreshes_dependency_hashes)
!= Fingerprint::from_le_bytes(*expected_fingerprint)
{
return None;
}
Self::validate_payload(payload)?;
let mut decoder = opaque::MemDecoder::new(payload, 0).ok()?;
let manifest = Self::decode(&mut decoder);
(decoder.position() + opaque::MAGIC_END_BYTES.len() == payload.len()).then_some(manifest)
(decoder.position() + opaque::MAGIC_END_BYTES.len() == payload.len())
.then_some((manifest, refreshes_dependency_hashes))
}

/// A fresh encoder which delegates dependency SVHs must never leave behind a usable v3
/// manifest if the new optional manifest cannot subsequently be written.
pub(super) fn clear_for_dependency_hashes(incr_comp_session: Option<&IncrCompSession>) -> bool {
let Some(path) = EncodedMetadata::prefix_manifest_path(incr_comp_session) else {
return false;
};
match std::fs::remove_file(path) {
Ok(()) => true,
Err(err) => err.kind() == std::io::ErrorKind::NotFound,
}
}

pub(super) fn save(&self, incr_comp_session: Option<&IncrCompSession>) -> std::io::Result<()> {
pub(super) fn save(
&self,
incr_comp_session: Option<&IncrCompSession>,
refreshes_dependency_hashes: bool,
) -> std::io::Result<()> {
let magic = manifest_magic(refreshes_dependency_hashes);
let Some(path) = EncodedMetadata::prefix_manifest_path(incr_comp_session) else {
return Ok(());
};
Expand All @@ -410,7 +457,7 @@ impl MetadataPrefixManifest {
let temporary_file =
tempfile::Builder::new().prefix(".ohm-metadata-prefix-").tempfile_in(directory)?;
let mut encoder = opaque::FileEncoder::new(temporary_file.path())?;
encoder.emit_raw_bytes(METADATA_PREFIX_MAGIC);
encoder.emit_raw_bytes(magic);
encoder.emit_raw_bytes(&[0; 16]);
self.encode(&mut encoder);
if let Err((_, error)) = encoder.finish() {
Expand All @@ -422,10 +469,10 @@ impl MetadataPrefixManifest {
// SAFETY: This private temporary file has finished encoding and is not mutated while
// its read-only mapping is alive.
let manifest = unsafe { Mmap::map(file)? };
metadata_fingerprint(&manifest[METADATA_PREFIX_MAGIC.len() + 16..])
manifest_fingerprint(&manifest[magic.len() + 16..], refreshes_dependency_hashes)
};
let mut file = File::options().write(true).open(temporary_file.path())?;
file.seek(std::io::SeekFrom::Start(METADATA_PREFIX_MAGIC.len() as u64))?;
file.seek(std::io::SeekFrom::Start(magic.len() as u64))?;
file.write_all(&fingerprint.to_le_bytes())?;
file.flush()?;
temporary_file.persist(path).map_err(|error| error.error)?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -615,8 +615,12 @@ pub(super) fn crate_hashes<'tcx, 'h>(
let mut hasher = StableHasher::default();
// A legacy intermediate can retain positional expansion IDs into a public dependency.
// Prevent it from accepting a new-format producer under the old equivalence relation.
let mut public_global_hash =
stable_hash(hcx.hcx_mut(), &(crate::rmeta::OHM_METADATA_EPOCH, root));
// Prior visibility tables also recorded the full restricted module as a dependency.
const VISIBILITY_REACHABILITY_VERSION: u8 = 1;
let mut public_global_hash = stable_hash(
hcx.hcx_mut(),
&(crate::rmeta::OHM_METADATA_EPOCH, VISIBILITY_REACHABILITY_VERSION, root),
);
if tcx.sess.opts.unstable_opts.incremental_ignore_spans {
// Per-item hashes omit spans in this mode. Invalidate readers on local source edits.
let mut files: Vec<_> = tcx
Expand Down
2 changes: 1 addition & 1 deletion compiler/rustc_metadata/src/rmeta/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ impl RawDefId {
///
/// Note the absence of the dependency's `-C extra-filename`: it lives in
/// [`CrateRootUnhashed::dep_extra_filenames`] instead, deliberately outside the hash.
#[derive(Encodable, BlobDecodable)]
#[derive(Encodable, BlobDecodable, PartialEq)]
pub(crate) struct CrateDep {
pub name: Symbol,
/// The dependency hash required by the writer's DefId encoding: public for RDR, full for raw indices.
Expand Down
1 change: 0 additions & 1 deletion compiler/rustc_middle/src/queries.rs
Original file line number Diff line number Diff line change
Expand Up @@ -644,7 +644,6 @@ rustc_queries! {
query params_in_repr(key: DefId) -> &'tcx rustc_index::bit_set::DenseBitSet<u32> {
desc { "finding type parameters in the representation" }
arena_cache
no_hash
separate_provide_extern
}

Expand Down
Loading
Loading