diff --git a/Cargo.lock b/Cargo.lock index 5a40c73601340..246274313af00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4387,6 +4387,7 @@ dependencies = [ "rustc_target", "tempfile", "tracing", + "twox-hash 2.1.2", ] [[package]] diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 1265bae778601..852c079fa21aa 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -917,7 +917,7 @@ impl<'hir> LoweringContext<'_, 'hir> { let nodes = hir::OwnerNodes { opt_hash: bodies_hash, nodes, bodies }; let attrs = hir::AttributeMap { map: attrs, opt_hash: attrs_hash, define_opaque }; - let opt_hash = self.tcx.needs_hir_hash().then(|| { + let opt_hash = self.tcx.needs_owner_info_hash().then(|| { self.tcx.with_stable_hashing_context(|mut hcx| { let mut stable_hasher = StableHasher::new(); bodies_hash.unwrap().stable_hash(&mut hcx, &mut stable_hasher); diff --git a/compiler/rustc_crate_store/src/cstore.rs b/compiler/rustc_crate_store/src/cstore.rs index fba2fc288f5a9..c8fe5562bdd59 100644 --- a/compiler/rustc_crate_store/src/cstore.rs +++ b/compiler/rustc_crate_store/src/cstore.rs @@ -4,9 +4,11 @@ use std::any::Any; use std::path::PathBuf; +use std::sync::OnceLock; use rustc_abi::ExternAbi; use rustc_attr_ir::{CfgEntry, PeImportNameType}; +use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::{self, AppendOnlyIndexVec, FreezeLock}; use rustc_hir_id::definitions::{DefKey, DefPath, Definitions}; use rustc_macros::{BlobDecodable, Decodable, Encodable, StableHash}; @@ -220,6 +222,8 @@ pub struct Untracked { pub definitions: FreezeLock, /// The interned [StableCrateId]s. pub stable_crate_ids: FreezeLock, + /// The hash of the local crate as computed in metadata encoding. + pub local_crate_hash: OnceLock, } impl Untracked { diff --git a/compiler/rustc_data_structures/src/svh.rs b/compiler/rustc_data_structures/src/svh.rs index 67594f6dae79d..0b80c655d8a5a 100644 --- a/compiler/rustc_data_structures/src/svh.rs +++ b/compiler/rustc_data_structures/src/svh.rs @@ -32,6 +32,10 @@ impl Svh { pub fn to_hex(self) -> String { format!("{:032x}", self.hash.as_u128()) } + + pub fn as_fingerprint(self) -> Fingerprint { + self.hash + } } impl fmt::Display for Svh { diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 54a1babbaae72..ae212fc83980f 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -315,10 +315,6 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send)) tcx.ensure_ok().analysis(()); - if let Some(metrics_dir) = &sess.opts.unstable_opts.metrics_dir { - dump_feature_usage_metrics(tcx, metrics_dir); - } - if callbacks.after_analysis(compiler, tcx) == Compilation::Stop { return None; } @@ -331,6 +327,10 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send)) let linker = Linker::codegen_and_build_linker(tcx, codegen_backend); + if let Some(metrics_dir) = &sess.opts.unstable_opts.metrics_dir { + dump_feature_usage_metrics(tcx, metrics_dir); + } + tcx.report_unused_features(); Some(linker) diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index 37b2ca7718498..df1da6715a002 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -1405,6 +1405,16 @@ impl<'tcx> OwnerInfo<'tcx> { pub fn node(&self) -> OwnerNode<'tcx> { self.nodes.node() } + + // A fingerprint that identifies the contents of the OwnerInfo. + // It only depends on `nodes` and `attrs` because `parenting` and `trait_map` are + // deterministically calculated from `nodes` and `attrs`. + #[inline] + pub fn fingerprint(&self) -> Fingerprint { + let body = self.nodes.opt_hash.expect("HIR hash requested without needs_hir_hash"); + let attrs = self.attrs.opt_hash.expect("HIR hash requested without needs_hir_hash"); + body.combine(attrs) + } } #[derive(Copy, Clone, Debug, StableHash)] diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index c829864b02288..478b6cd308c2b 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -966,8 +966,13 @@ pub fn create_and_enter_global_ctxt FnOnce(TyCtxt<'tcx>) -> T>( let definitions = FreezeLock::new(Definitions::new(stable_crate_id)); let stable_crate_ids = FreezeLock::new(StableCrateIdMap::default()); - let untracked = - Untracked { cstore, source_span: AppendOnlyIndexVec::new(), definitions, stable_crate_ids }; + let untracked = Untracked { + cstore, + source_span: AppendOnlyIndexVec::new(), + definitions, + stable_crate_ids, + local_crate_hash: OnceLock::new(), + }; // We're constructing the HIR here; we don't care what we will // read, since we haven't even constructed the *input* to diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml index a2ef8454ed6a6..583c146c736e6 100644 --- a/compiler/rustc_metadata/Cargo.toml +++ b/compiler/rustc_metadata/Cargo.toml @@ -33,6 +33,7 @@ rustc_structures = { path = "../rustc_structures" } rustc_target = { path = "../rustc_target" } tempfile = "3.7.1" tracing = "0.1" +twox-hash = { version = "2", default-features = false, features = ["xxhash3_128", "std"] } # tidy-alphabetical-end [target.'cfg(target_os = "aix")'.dependencies] diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs index 60de9d179cb98..509ba327bdcfa 100644 --- a/compiler/rustc_metadata/src/creader.rs +++ b/compiler/rustc_metadata/src/creader.rs @@ -153,6 +153,10 @@ enum CrateOrigin<'a> { parent_private: bool, /// Dependency info about this crate. dep: &'a CrateDep, + /// The dependency's `-C extra-filename`, used to narrow the search for it on disk. Stored + /// separately from `dep` because it is encoded outside the hashed crate root; see + /// `CrateRootUnhashed::dep_extra_filenames`. + dep_extra_filename: &'a str, }, /// Injected by `rustc`. Injected, @@ -179,6 +183,14 @@ impl<'a> CrateOrigin<'a> { } } + /// Return the dependency's `-C extra-filename`, if any. + fn dep_extra_filename(&self) -> Option<&'a str> { + match self { + CrateOrigin::IndirectDependency { dep_extra_filename, .. } => Some(dep_extra_filename), + _ => None, + } + } + /// `Some(true)` if the dependency is private or its parent is private, `Some(false)` if the /// dependency is not private, `None` if it could not be determined. fn private_dep(&self) -> Option { @@ -592,7 +604,8 @@ impl CStore { let Library { source, metadata } = lib; let crate_root = metadata.get_root(); - let host_hash = host_lib.as_ref().map(|lib| lib.metadata.get_root().hash()); + let unhashed = metadata.get_root_unhashed(); + let host_hash = host_lib.as_ref().map(|lib| lib.metadata.get_crate_hash()); let private_dep = self.is_private_dep(&tcx.sess.opts.externs, name, private_dep); // Claim this crate number and cache it @@ -645,6 +658,7 @@ impl CStore { tcx, metadata, crate_root, + unhashed, raw_proc_macros, cnum, cnum_map, @@ -787,7 +801,7 @@ impl CStore { let dep = origin.dep(); let hash = dep.map(|d| d.hash); let host_hash = dep.map(|d| d.host_hash).flatten(); - let extra_filename = dep.map(|d| &d.extra_filename[..]); + let extra_filename = origin.dep_extra_filename(); let path_kind = if dep.is_some() { PathKind::Dependency } else { PathKind::Crate }; let private_dep = origin.private_dep(); @@ -868,10 +882,11 @@ impl CStore { // against a hash, we could load a crate which has the same hash // as an already loaded crate. If this is the case prevent // duplicates by just using the first crate. - let root = library.metadata.get_root(); + let root_name = library.metadata.get_root().name(); + let root_hash = library.metadata.get_crate_hash(); let mut result = LoadResult::Loaded(library); for (cnum, data) in self.iter_crate_data() { - if data.name() == root.name() && root.hash() == data.hash() { + if data.name() == root_name && data.hash() == root_hash { assert!(locator.hash.is_none()); info!("load success, going to previous cnum: {}", cnum); result = LoadResult::Previous(cnum); @@ -905,15 +920,26 @@ impl CStore { // We map 0 and all other holes in the map to our parent crate. The "additional" // self-dependencies should be harmless. let deps = crate_root.decode_crate_deps(metadata); + // Encoded outside the hashed crate root; see `CrateRootUnhashed::dep_extra_filenames`. + // Holds one entry per dep after the unused `LOCAL_CRATE` slot, so it lines up with `deps` + // once that slot is skipped. + let dep_extra_filenames = metadata.get_dep_extra_filenames(); + assert_eq!( + dep_extra_filenames.len(), + deps.len() + 1, + "expected one dep_extra_filename per crate dep, plus the unused LOCAL_CRATE slot", + ); let mut crate_num_map = CrateNumMap::with_capacity(1 + deps.len()); crate_num_map.push(krate); - for dep in deps { + for (dep, dep_extra_filename) in + deps.zip(dep_extra_filenames.iter().skip(1).map(String::as_str)) + { info!( "resolving dep `{}`->`{}` hash: `{}` extra filename: `{}` private {}", crate_root.name(), dep.name, dep.hash, - dep.extra_filename, + dep_extra_filename, dep.is_private, ); let dep_kind = match dep_kind { @@ -928,6 +954,7 @@ impl CStore { dep_root_for_errors, parent_private: parent_is_private, dep: &dep, + dep_extra_filename, }, )?; crate_num_map.push(cnum); diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs index 23b51dfbf62c6..38da6561ecb38 100644 --- a/compiler/rustc_metadata/src/locator.rs +++ b/compiler/rustc_metadata/src/locator.rs @@ -724,7 +724,7 @@ impl<'a> CrateLocator<'a> { return None; } - let hash = header.hash; + let hash = metadata.get_crate_hash(); if let Some(expected_hash) = self.hash { if hash != expected_hash { info!("Rejecting via hash: expected {} got {}", expected_hash, hash); diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index 8a565369d7610..5d6744246e2ad 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -97,6 +97,8 @@ pub(crate) struct CrateMetadata { // --- Some data pre-decoded from the metadata blob, usually for performance --- /// Data about the top-level items in a crate, as well as various crate-level metadata. root: CrateRoot, + /// Crate-level metadata that is not in the SVH. + unhashed: CrateRootUnhashed, /// Trait impl data. /// FIXME: Used only from queries and can use query cache, /// so pre-decoding can probably be avoided. @@ -717,8 +719,7 @@ impl MetadataBlob { } let found_version = - LazyValue::::from_position(NonZero::new(METADATA_HEADER.len() + 8).unwrap()) - .decode(self); + LazyValue::::from_position(NonZero::new(VERSION_OFFSET).unwrap()).decode(self); if rustc_version(cfg_version) != found_version { return Err(Some(found_version)); } @@ -727,8 +728,13 @@ impl MetadataBlob { } fn root_pos(&self) -> NonZero { - let offset = METADATA_HEADER.len(); - let pos_bytes = self[offset..][..8].try_into().unwrap(); + let pos_bytes = self[ROOT_POS_OFFSET..][..8].try_into().unwrap(); + let pos = u64::from_le_bytes(pos_bytes); + NonZero::new(pos as usize).unwrap() + } + + fn unhashed_pos(&self) -> NonZero { + let pos_bytes = self[UNHASHED_POS_OFFSET..][..8].try_into().unwrap(); let pos = u64::from_le_bytes(pos_bytes); NonZero::new(pos as usize).unwrap() } @@ -743,12 +749,28 @@ impl MetadataBlob { LazyValue::::from_position(pos).decode(self) } + pub(crate) fn get_root_unhashed(&self) -> CrateRootUnhashed { + let pos = self.unhashed_pos(); + LazyValue::::from_position(pos).decode(self) + } + + pub(crate) fn get_dep_extra_filenames(&self) -> IndexVec { + self.get_root_unhashed().dep_extra_filenames + } + + pub(crate) fn get_crate_hash(&self) -> Svh { + let bytes: [u8; CRATE_HASH_LEN] = + self[CRATE_HASH_OFFSET..][..CRATE_HASH_LEN].try_into().unwrap(); + Svh::new(Fingerprint::from_le_bytes(bytes)) + } + pub(crate) fn list_crate_metadata( &self, out: &mut dyn io::Write, ls_kinds: &[String], ) -> io::Result<()> { let root = self.get_root(); + let extra_filename = self.get_root_unhashed().extra_filename; let all_ls_kinds = vec![ "root".to_owned(), @@ -763,11 +785,11 @@ impl MetadataBlob { match &**kind { "root" => { writeln!(out, "Crate info:")?; - writeln!(out, "name {}{}", root.name(), root.extra_filename)?; + writeln!(out, "name {}{}", root.name(), extra_filename)?; writeln!( out, "hash {} stable_crate_id {:?}", - root.hash(), + self.get_crate_hash(), root.stable_crate_id )?; writeln!(out, "proc_macro {:?}", root.proc_macro_data.is_some())?; @@ -801,10 +823,15 @@ impl MetadataBlob { writeln!(out, "=External Dependencies=")?; let dylib_dependency_formats = root.dylib_dependency_formats.decode(self).collect::>(); + // `extra_filename` is stored outside the hashed root; see + // `CrateRootUnhashed::dep_extra_filenames`. + let dep_extra_filenames = self.get_dep_extra_filenames(); for (i, dep) in root.crate_deps.decode(self).enumerate() { - let CrateDep { name, extra_filename, hash, host_hash, kind, is_private } = - dep; + let CrateDep { name, hash, host_hash, kind, is_private } = dep; let number = i + 1; + let extra_filename = dep_extra_filenames + .get(CrateNum::new(number)) + .map_or("", |name| name.as_str()); writeln!( out, @@ -975,10 +1002,6 @@ impl CrateRoot { self.header.name } - pub(crate) fn hash(&self) -> Svh { - self.header.hash - } - pub(crate) fn stable_crate_id(&self) -> StableCrateId { self.stable_crate_id } @@ -1944,6 +1967,7 @@ impl CrateMetadata { tcx: TyCtxt<'_>, blob: MetadataBlob, root: CrateRoot, + unhashed: CrateRootUnhashed, raw_proc_macros: Option<&'static [ProcMacroClient]>, cnum: CrateNum, cnum_map: CrateNumMap, @@ -1967,6 +1991,7 @@ impl CrateMetadata { let mut cdata = CrateMetadata { blob, root, + unhashed, trait_impls, incoherent_impls: Default::default(), raw_proc_macros, @@ -2108,7 +2133,7 @@ impl CrateMetadata { } pub(crate) fn hash(&self) -> Svh { - self.root.header.hash + self.blob.get_crate_hash() } pub(crate) fn has_async_drops(&self) -> bool { diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 8fe1d6561d135..8c3694ae231af 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -377,12 +377,12 @@ provide! { tcx, def_id, other, cdata, } native_libraries => { cdata.get_native_libraries(tcx).collect() } foreign_modules => { cdata.get_foreign_modules(tcx).map(|m| (m.def_id, m)).collect() } - crate_hash => { cdata.root.header.hash } + crate_hash => { cdata.hash() } crate_host_hash => { cdata.host_hash } crate_name => { cdata.root.header.name } num_extern_def_ids => { cdata.num_def_ids() } - extra_filename => { cdata.root.extra_filename.clone() } + extra_filename => { cdata.unhashed.extra_filename.clone() } traits => { tcx.arena.alloc_from_iter(cdata.get_traits(tcx)) } trait_impls_in_crate => { tcx.arena.alloc_from_iter(cdata.get_trait_impls(tcx)) } diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index cc7da00fcec5c..7c975294287c5 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -1,12 +1,16 @@ use std::borrow::Borrow; use std::collections::hash_map::Entry; use std::fs::File; -use std::io::{Read, Seek, Write}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::memmap::{Mmap, MmapMut}; +use rustc_data_structures::owned_slice::slice_owned; +use rustc_data_structures::stable_hash::{StableHash, StableHasher}; use rustc_data_structures::sync::{par_for_each_in, par_join}; use rustc_data_structures::temp_dir::MaybeTempDir; use rustc_data_structures::thousands::usize_with_underscores; @@ -16,7 +20,9 @@ use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId, LocalDefIdSet}; use rustc_hir::definitions::DefPathData; use rustc_hir::find_attr; use rustc_hir_pretty::id_to_string; +use rustc_index::IndexVec; use rustc_middle::dep_graph::WorkProductId; +use rustc_middle::hir::map::compute_hir_hash; use rustc_middle::middle::dependency_format::Linkage; use rustc_middle::mir::interpret; use rustc_middle::query::Providers; @@ -25,7 +31,8 @@ use rustc_middle::ty::AssocContainer; use rustc_middle::ty::codec::TyEncoder; use rustc_middle::ty::fast_reject::{self, TreatParams}; use rustc_middle::{bug, span_bug}; -use rustc_serialize::{Decodable, Decoder, Encodable, Encoder, opaque}; +use rustc_serialize::opaque::FileEncoder; +use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use rustc_session::config::mitigation_coverage::DeniedPartialMitigation; use rustc_session::config::{OptLevel, TargetModifier}; use rustc_span::def_id::CRATE_MOD_ID; @@ -36,13 +43,15 @@ use rustc_span::{ }; use rustc_structures::CrateType; use tracing::{debug, instrument, trace}; +use twox_hash::XxHash3_128; use crate::diagnostics::{FailCreateFileEncoder, FailWriteFile}; use crate::eii::EiiMapEncodedKeyValue; use crate::rmeta::*; pub(super) struct EncodeContext<'a, 'tcx> { - opaque: opaque::FileEncoder<'a>, + opaque: FileEncoder<'a>, + metadata_hasher: Arc>, tcx: TyCtxt<'tcx>, feat: &'tcx rustc_feature::Features, tables: TableBuilders, @@ -605,7 +614,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { adapted.encode(&mut self.opaque) } - fn encode_crate_root(&mut self) -> LazyValue { + fn encode_crate_root(&mut self) -> (LazyValue, LazyValue) { let tcx = self.tcx; let mut stats: Vec<(&'static str, usize)> = Vec::with_capacity(32); @@ -724,17 +733,16 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { let denied_partial_mitigations = stat!("denied-partial-mitigations", || self .encode_enabled_denied_partial_mitigations()); - let root = stat!("final", || { + let root = stat!("crate-root", || { let attrs = tcx.hir_krate_attrs(); self.lazy(CrateRoot { header: CrateHeader { name: tcx.crate_name(LOCAL_CRATE), triple: tcx.sess.opts.target_triple.clone(), - hash: tcx.crate_hash(LOCAL_CRATE), is_proc_macro_crate: proc_macro_data.is_some(), is_stub: false, }, - extra_filename: tcx.sess.opts.cg.extra_filename.clone(), + stable_crate_id: tcx.stable_crate_id(LOCAL_CRATE), required_panic_strategy: tcx.required_panic_strategy(LOCAL_CRATE), panic_in_drop_strategy: tcx.sess.opts.unstable_opts.panic_in_drop, @@ -785,6 +793,36 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { }) }); + // By default the crate hash (SVH) is derived from the encoded metadata bytes. With + // `-Z metadata-crate-hash=no` we instead store the legacy HIR-based hash, which the + // `crate_hash` query computes directly without consulting `local_crate_hash`. + let hash = if tcx.sess.opts.unstable_opts.metadata_crate_hash { + self.opaque.flush(); + Svh::new(Fingerprint::from_le_bytes( + self.metadata_hasher.lock().unwrap().finish_128().to_le_bytes(), + )) + } else { + tcx.crate_hash(LOCAL_CRATE) + }; + tcx.untracked().local_crate_hash.set(hash).expect("local_crate_hash set twice"); + + let unhashed = stat!("final", || { + // Indexed by dependency `CrateNum`, matching the numbering `encode_crate_deps` uses. + // Slot 0 (`LOCAL_CRATE`) is filler; this crate's own value is `extra_filename`. + let mut dep_extra_filenames = IndexVec::from_elem_n(String::new(), 1); + if !self.is_proc_macro { + for &cnum in self.tcx.crates(()).iter() { + let idx = dep_extra_filenames.push(self.tcx.extra_filename(cnum).clone()); + assert_eq!(idx, cnum, "dep_extra_filenames must be indexed by CrateNum"); + } + } + + self.lazy(CrateRootUnhashed { + extra_filename: self.tcx.sess.opts.cg.extra_filename.clone(), + dep_extra_filenames, + }) + }); + let total_bytes = self.position(); let computed_total_bytes: usize = stats.iter().map(|(_, size)| size).sum(); @@ -848,7 +886,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { eprint!("{s}"); } - root + (root, unhashed) } } @@ -2095,7 +2133,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { hash: self.tcx.crate_hash(cnum), host_hash: self.tcx.crate_host_hash(cnum), kind: self.tcx.crate_dep_kind(cnum), - extra_filename: self.tcx.extra_filename(cnum).clone(), is_private: self.tcx.is_private_dep(cnum), }; (cnum, dep) @@ -2458,22 +2495,6 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { // there's no need to do dep-graph tracking for any of it. tcx.dep_graph.assert_ignored(); - // Generate the metadata stub manually, as that is a small file compared to full metadata. - if let Some(ref_path) = ref_path { - let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata_stub"); - - with_encode_metadata_header(tcx, ref_path, |ecx| { - let header: LazyValue = ecx.lazy(CrateHeader { - name: tcx.crate_name(LOCAL_CRATE), - triple: tcx.sess.opts.target_triple.clone(), - hash: tcx.crate_hash(LOCAL_CRATE), - is_proc_macro_crate: false, - is_stub: true, - }); - header.position.get() - }) - } - let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata"); let dep_node = tcx.metadata_dep_node(); @@ -2492,6 +2513,15 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { Ok(_) => {} Err(err) => tcx.dcx().emit_fatal(FailCreateFileEncoder { err }), }; + + // Read the SVH from the old metadata header. + let file = std::fs::File::open(&source_file_in_incr_dir).unwrap(); + let mmap = unsafe { Mmap::map(file) }.unwrap(); + let owned = slice_owned(mmap, Deref::deref); + let blob = MetadataBlob::new(owned); + let hash = blob.expect("file already created").get_crate_hash(); + tcx.untracked().local_crate_hash.set(hash).expect("local_crate_hash set twice"); + return; }; @@ -2517,7 +2547,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { with_encode_metadata_header(tcx, path, |ecx| { // Encode all the entries and extra information in the crate, // culminating in the `CrateRoot` which points to all of it. - let root = ecx.encode_crate_root(); + let (root, unhashed) = ecx.encode_crate_root(); // Flush buffer to ensure backing file has the correct size. ecx.opaque.flush(); @@ -2528,25 +2558,80 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) { ecx.opaque.file().metadata().unwrap().len(), ); - root.position.get() + (root.position.get(), unhashed.position.get()) }) }, None, ); + + // Generate the metadata stub manually, as that is a small file compared to full metadata. + if let Some(ref_path) = ref_path { + let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata_stub"); + + with_encode_metadata_header(tcx, ref_path, |ecx| { + let header: LazyValue = ecx.lazy(CrateHeader { + name: tcx.crate_name(LOCAL_CRATE), + triple: tcx.sess.opts.target_triple.clone(), + is_proc_macro_crate: false, + is_stub: true, + }); + (header.position.get(), 0) + }) + } } fn with_encode_metadata_header( tcx: TyCtxt<'_>, path: &Path, - f: impl FnOnce(&mut EncodeContext<'_, '_>) -> usize, + f: impl FnOnce(&mut EncodeContext<'_, '_>) -> (usize, usize), ) { - let mut encoder = opaque::FileEncoder::new(path) - .unwrap_or_else(|err| tcx.dcx().emit_fatal(FailCreateFileEncoder { err })); + // By default the crate hash (SVH) is computed from the bytes of the encoded metadata, + // Under `-Z metadata-crate-hash=no` the SVH comes from the legacy `crate_hash` query instead and + // this hasher is never consulted, so we skip seeding it and feeding metadata bytes into it. + let metadata_crate_hash = tcx.sess.opts.unstable_opts.metadata_crate_hash; + // The SVH is an XXH3-128 digest of the encoded metadata bytes. XXH3 is a fast, non-cryptographic + // hash; that is sufficient here because the SVH is only a change detector, is never keyed + // secretly (it must be reproducible), and compiling a crate already runs its build scripts and + // proc-macros, so the crate author is trusted regardless. + let metadata_hasher = Arc::new(Mutex::new(XxHash3_128::new())); + if metadata_crate_hash { + // Fold in inputs that are not part of the encoded metadata bytes, reduced to a single + // fingerprint via the stable hasher and then mixed into the byte digest. + let hir_body_hash = compute_hir_hash(tcx); + let supplement: Fingerprint = tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + // Add dep_tracking_hash to ensure the SVH changes when any tracked flag changes. + tcx.sess.opts.dep_tracking_hash(true).stable_hash(&mut hcx, &mut hasher); + // Add HIR hash for untracked elements, e.g. DefKind::GlobalAsm. + hir_body_hash.stable_hash(&mut hcx, &mut hasher); + hasher.finish() + }); + metadata_hasher.lock().unwrap().write(&supplement.to_le_bytes()); + } + + // Feed every flushed byte into `metadata_hasher` so the SVH covers the entire encoded metadata. + let mut flush_strategy = { + let metadata_hasher = Arc::clone(&metadata_hasher); + move |bytes: &[u8]| metadata_hasher.lock().unwrap().write(bytes) + }; + + let mut encoder = if metadata_crate_hash { + FileEncoder::with_flush_strategy(path, &mut flush_strategy) + } else { + FileEncoder::new(path) + } + .unwrap_or_else(|err| tcx.dcx().emit_fatal(FailCreateFileEncoder { err })); encoder.emit_raw_bytes(METADATA_HEADER); // Will be filled with the root position after encoding everything. encoder.emit_raw_bytes(&0u64.to_le_bytes()); + // Same with unhashed_position. + encoder.emit_raw_bytes(&0u64.to_le_bytes()); + + // Same with crate_hash. + encoder.emit_raw_bytes(&Fingerprint::ZERO.to_le_bytes()); + let source_map_files = tcx.sess.source_map().files(); let source_file_cache = (Arc::clone(&source_map_files[0]), 0); let required_source_files = Some(FxIndexSet::default()); @@ -2556,6 +2641,7 @@ fn with_encode_metadata_header( let mut ecx = EncodeContext { opaque: encoder, + metadata_hasher: Arc::clone(&metadata_hasher), tcx, feat: tcx.features(), tables: Default::default(), @@ -2574,7 +2660,7 @@ fn with_encode_metadata_header( // Encode the rustc version string in a predictable location. rustc_version(tcx.sess.cfg_version).encode(&mut ecx); - let root_position = f(&mut ecx); + let (root_position, unhashed_position) = f(&mut ecx); // Make sure we report any errors from writing to the file. // If we forget this, compilation can succeed with an incomplete rmeta file, @@ -2583,23 +2669,48 @@ fn with_encode_metadata_header( tcx.dcx().emit_fatal(FailWriteFile { path: &path, err }); } - let file = ecx.opaque.file(); + let mut file = ecx.opaque.file(); + // We will return to this position after writing the root position and crate hash. + let pos_before_seek = file.stream_position().unwrap(); + if let Err(err) = encode_root_position(file, root_position) { tcx.dcx().emit_fatal(FailWriteFile { path: ecx.opaque.path(), err }); } + + if let Err(err) = encode_unhashed_position(file, unhashed_position) { + tcx.dcx().emit_fatal(FailWriteFile { path: ecx.opaque.path(), err }); + } + + let hash = tcx + .untracked() + .local_crate_hash + .get() + .copied() + .expect("local_crate_hash set during encoding"); + if let Err(err) = encode_crate_hash(file, hash) { + tcx.dcx().emit_fatal(FailWriteFile { path: ecx.opaque.path(), err }); + } + + if let Err(err) = file.seek(SeekFrom::Start(pos_before_seek)) { + tcx.dcx().emit_fatal(FailWriteFile { path: ecx.opaque.path(), err }); + } } fn encode_root_position(mut file: &File, pos: usize) -> Result<(), std::io::Error> { - // We will return to this position after writing the root position. - let pos_before_seek = file.stream_position().unwrap(); + file.seek(SeekFrom::Start(ROOT_POS_OFFSET as u64))?; + file.write_all(&pos.to_le_bytes())?; + Ok(()) +} - // Encode the root position. - let header = METADATA_HEADER.len(); - file.seek(std::io::SeekFrom::Start(header as u64))?; +fn encode_unhashed_position(mut file: &File, pos: usize) -> Result<(), std::io::Error> { + file.seek(SeekFrom::Start(UNHASHED_POS_OFFSET as u64))?; file.write_all(&pos.to_le_bytes())?; + Ok(()) +} - // Return to the position where we are before writing the root position. - file.seek(std::io::SeekFrom::Start(pos_before_seek))?; +fn encode_crate_hash(mut file: &File, hash: Svh) -> Result<(), std::io::Error> { + file.seek(SeekFrom::Start(CRATE_HASH_OFFSET as u64))?; + file.write_all(&hash.as_fingerprint().to_le_bytes())?; Ok(()) } diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs index 064d906293ae8..35d88e07eb6f9 100644 --- a/compiler/rustc_metadata/src/rmeta/mod.rs +++ b/compiler/rustc_metadata/src/rmeta/mod.rs @@ -1,4 +1,5 @@ use std::marker::PhantomData; +use std::mem::size_of; use std::num::NonZero; use decoder::LazyDecoder; @@ -10,6 +11,7 @@ pub(crate) use parameterized::ParameterizedOverTcx; use rustc_abi::{FieldIdx, ReprOptions, VariantIdx}; use rustc_ast as ast; use rustc_crate_store::{CrateDepKind, ForeignModule, LinkagePreference, NativeLib}; +use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::svh::Svh; use rustc_hir as hir; @@ -60,15 +62,27 @@ pub(crate) fn rustc_version(cfg_version: &'static str) -> String { /// Metadata encoding version. /// N.B., increment this if you change the format of metadata such that /// the rustc version can't be found to compare with `rustc_version()`. -const METADATA_VERSION: u8 = 10; +const METADATA_VERSION: u8 = 11; /// Metadata header which includes `METADATA_VERSION`. /// -/// This header is followed by the length of the compressed data, then -/// the position of the `CrateRoot`, which is encoded as a 64-bit little-endian -/// unsigned integer, and further followed by the rustc version string. +/// This header is followed by the `CrateRoot` and `CrateRootUnhashed` positions +/// which represent the hashed and unhashed metadata contents respectively, the +/// crate hash (SVH), and the rustc version string. See the offset constants +/// below for the exact layout. pub const METADATA_HEADER: &[u8] = &[b'r', b'u', b's', b't', 0, 0, 0, METADATA_VERSION]; +/// Fixed-size fields encoded immediately after `METADATA_HEADER`, in order: +/// `CrateRoot` position (u64), `CrateRootUnhashed` position (u64), crate hash +/// (`Fingerprint`/SVH), then the variable-length rustc version string. +const ROOT_POS_OFFSET: usize = METADATA_HEADER.len(); +const ROOT_POS_LEN: usize = size_of::(); +const UNHASHED_POS_OFFSET: usize = ROOT_POS_OFFSET + ROOT_POS_LEN; +const UNHASHED_POS_LEN: usize = size_of::(); +const CRATE_HASH_OFFSET: usize = UNHASHED_POS_OFFSET + UNHASHED_POS_LEN; +const CRATE_HASH_LEN: usize = size_of::(); +const VERSION_OFFSET: usize = CRATE_HASH_OFFSET + CRATE_HASH_LEN; + /// A value of type T referred to by its absolute position /// in the metadata, and which can be decoded lazily. /// @@ -213,7 +227,6 @@ pub enum ProcMacroKind { #[derive(MetadataEncodable, BlobDecodable)] pub(crate) struct CrateHeader { pub(crate) triple: TargetTuple, - pub(crate) hash: Svh, pub(crate) name: Symbol, /// Whether this is the header for a proc-macro crate. /// @@ -250,7 +263,6 @@ pub(crate) struct CrateRoot { /// A header used to detect if this is the right crate to load. header: CrateHeader, - extra_filename: String, stable_crate_id: StableCrateId, required_panic_strategy: Option, panic_in_drop_strategy: PanicStrategy, @@ -307,6 +319,18 @@ pub(crate) struct CrateRoot { specialization_enabled_in: bool, } +/// A separate struct for extra metadata that must be encoded *after* +/// the main crate hash is finalized. +#[derive(MetadataEncodable, LazyDecodable)] +pub(crate) struct CrateRootUnhashed { + extra_filename: String, + + /// The `-C extra-filename` of each dependency, indexed by the `CrateNum` they had in *this* + /// crate's encoding. The `LOCAL_CRATE` slot is unused filler so that dependency `CrateNum`s + /// can index this directly; this crate's own value is `extra_filename` above. + dep_extra_filenames: IndexVec, +} + /// On-disk representation of `DefId`. /// This creates a type-safe way to enforce that we remap the CrateNum between the on-disk /// representation and the compilation session. @@ -331,13 +355,16 @@ impl RawDefId { } } +/// A dependency record, as stored in the hashed [`CrateRoot`]. +/// +/// Note the absence of the dependency's `-C extra-filename`: it lives in +/// [`CrateRootUnhashed::dep_extra_filenames`] instead, deliberately outside the hash. #[derive(Encodable, BlobDecodable)] pub(crate) struct CrateDep { pub name: Symbol, pub hash: Svh, pub host_hash: Option, pub kind: CrateDepKind, - pub extra_filename: String, pub is_private: bool, } diff --git a/compiler/rustc_metadata/src/rmeta/parameterized.rs b/compiler/rustc_metadata/src/rmeta/parameterized.rs index f19737bb936be..0f0c5dfc7441f 100644 --- a/compiler/rustc_metadata/src/rmeta/parameterized.rs +++ b/compiler/rustc_metadata/src/rmeta/parameterized.rs @@ -78,6 +78,7 @@ trivially_parameterized_over_tcx! { crate::rmeta::CrateDep, crate::rmeta::CrateHeader, crate::rmeta::CrateRoot, + crate::rmeta::CrateRootUnhashed, crate::rmeta::IncoherentImpls, crate::rmeta::ProcMacroKind, crate::rmeta::RawDefId, diff --git a/compiler/rustc_middle/src/hir/map.rs b/compiler/rustc_middle/src/hir/map.rs index 8ec27921a5787..92cc3d75a4965 100644 --- a/compiler/rustc_middle/src/hir/map.rs +++ b/compiler/rustc_middle/src/hir/map.rs @@ -10,7 +10,7 @@ use rustc_data_structures::steal::Steal; use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::{DynSend, DynSync, par_for_each_in, try_par_for_each_in}; use rustc_hir::def::{DefKind, Res}; -use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId, LocalModId}; +use rustc_hir::def_id::{DefId, LocalDefId, LocalModId}; use rustc_hir::definitions::{DefKey, DefPath, DefPathHash}; use rustc_hir::intravisit::Visitor; use rustc_hir::lints::DelayedLints; @@ -18,6 +18,7 @@ use rustc_hir::*; use rustc_span::def_id::{CRATE_MOD_ID, StableCrateId}; use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol, kw, with_metavar_spans}; +use crate::hir::def_id::LOCAL_CRATE; use crate::hir::{ModuleItems, ProjectedMaybeOwner, nested_filter}; use crate::middle::debugger_visualizer::DebuggerVisualizerFile; use crate::query::{IntoQueryKey, LocalCrate}; @@ -1156,7 +1157,47 @@ impl<'tcx> intravisit::HirTyCtxt<'tcx> for TyCtxt<'tcx> { } pub(super) fn crate_hash(tcx: TyCtxt<'_>, _: LocalCrate) -> Svh { + // `-Z metadata-crate-hash=no` opts back into computing the SVH from the HIR rather than + // from the encoded crate metadata. This is a safety fallback for the metadata-based hashing. + if !tcx.sess.opts.unstable_opts.metadata_crate_hash { + return legacy_crate_hash(tcx); + } + + // If metadata is being encoded, the crate hash has already been computed as part of the + // metadata encoding. + if tcx.needs_metadata() { + *tcx.untracked() + .local_crate_hash + .get() + .expect("crate_hash(LOCAL_CRATE) called before metadata encoding") + } else { + // When metadata isn't encoded, use an HIR based approximation. Encoding metadata for a + // dylib/binary is expensive and fragile. These fields are enough to identify the session. + let hir_body_hash = compute_hir_hash(tcx); + + let upstream_crates = upstream_crates(tcx); + + let crate_hash: Fingerprint = tcx.with_stable_hashing_context(|mut hcx| { + let mut stable_hasher = StableHasher::new(); + hir_body_hash.stable_hash(&mut hcx, &mut stable_hasher); + upstream_crates.stable_hash(&mut hcx, &mut stable_hasher); + tcx.sess.opts.dep_tracking_hash(true).stable_hash(&mut hcx, &mut stable_hasher); + tcx.stable_crate_id(LOCAL_CRATE).stable_hash(&mut hcx, &mut stable_hasher); + + stable_hasher.finish() + }); + + Svh::new(crate_hash) + } +} + +/// Only reached when `-Z metadata-crate-hash=no` reverts to the pre-metadata-hashing behavior. +/// The HIR-based hashing scheme here matches the `crate_hash` query as it existed before this PR. +/// (The resulting SVH still differs from a pre-PR compiler, because `metadata_crate_hash` is a +/// `[TRACKED]` option and therefore contributes to the dep-tracking hash hashed in below.) +fn legacy_crate_hash(tcx: TyCtxt<'_>) -> Svh { let krate = tcx.hir_crate_items(()); + let upstream_crates = upstream_crates(tcx); let resolutions = tcx.resolutions(()); @@ -1232,6 +1273,19 @@ pub(super) fn crate_hash(tcx: TyCtxt<'_>, _: LocalCrate) -> Svh { Svh::new(crate_hash) } +/// Compute the new, metadata-oriented HIR hash for the full crate. +/// +/// Unlike the legacy hash (see [`legacy_crate_hash`]), this is an order-independent combine of each +/// owner's fingerprint, so it does not depend on the iteration order of the owners. It becomes part +/// of the `crate_hash` which is stored in the crate metadata. +pub fn compute_hir_hash(tcx: TyCtxt<'_>) -> Fingerprint { + tcx.hir_crate_items(()) + .owners() + .filter_map(|owner| Some(tcx.lower_to_hir(owner.def_id).as_owner()?.fingerprint())) + .reduce(Fingerprint::combine_commutative) + .expect("HIR hash requested without any content") +} + fn upstream_crates(tcx: TyCtxt<'_>) -> Vec<(StableCrateId, Svh)> { let mut upstream_crates: Vec<_> = tcx .crates(()) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 7a3b4c7fbbeb8..f604298268f58 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1156,6 +1156,27 @@ impl<'tcx> TyCtxt<'tcx> { || self.sess.opts.unstable_opts.metrics_dir.is_some() } + /// Whether the combined per-owner HIR hash (`OwnerInfo::opt_hash`, which folds `parenting`, + /// `trait_map` and `children` on top of the node/attr hashes) needs to be computed during + /// lowering. + /// + /// This is a strict subset of [`Self::needs_hir_hash`]: notably it drops the plain + /// `needs_metadata` case. With metadata-based crate hashing (the default) the crate hash is + /// built from the encoded metadata plus each owner's cheaper `OwnerInfo::fingerprint` (just the + /// node and attr sub-hashes), so the combined hash is never read and computing it is wasted + /// work. It is still required for: + /// - `-Z metadata-crate-hash=no`, where `crate_hash` falls back to hashing each `OwnerInfo`; + /// - incremental, where the `lower_to_hir` result is fingerprinted for red/green tracking; + /// - debug assertions, where every query result is fingerprinted to catch nondeterminism. + /// + /// The `needs_hir_hash()` conjunct guarantees the node/attr sub-hashes it folds in are present. + pub fn needs_owner_info_hash(self) -> bool { + self.needs_hir_hash() + && (!self.sess.opts.unstable_opts.metadata_crate_hash + || self.sess.opts.incremental.is_some() + || cfg!(debug_assertions)) + } + #[inline] pub fn stable_crate_id(self, crate_num: CrateNum) -> StableCrateId { if crate_num == LOCAL_CRATE { diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 68e0fec59f06e..618e982dbc0bd 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2642,6 +2642,9 @@ options! { the same values as the target option of the same name"), meta_stats: bool = (false, parse_bool, [UNTRACKED], "gather metadata statistics (default: no)"), + metadata_crate_hash: bool = (true, parse_bool, [TRACKED], + "compute the crate hash (SVH) from the encoded crate metadata; set to `no` to \ + revert to computing it from the HIR (default: yes)"), metrics_dir: Option = (None, parse_opt_pathbuf, [UNTRACKED], "the directory metrics emitted by rustc are dumped into (implicitly enables default set of metrics)"), min_function_alignment: Option = (None, parse_align, [TRACKED], diff --git a/tests/run-make/crate-hash-dep-extra-filename/a.rs b/tests/run-make/crate-hash-dep-extra-filename/a.rs new file mode 100644 index 0000000000000..7da56d12511aa --- /dev/null +++ b/tests/run-make/crate-hash-dep-extra-filename/a.rs @@ -0,0 +1,7 @@ +pub fn f() -> u32 { + 1 +} + +pub struct S { + pub field: u32, +} diff --git a/tests/run-make/crate-hash-dep-extra-filename/b.rs b/tests/run-make/crate-hash-dep-extra-filename/b.rs new file mode 100644 index 0000000000000..2e8656738e61f --- /dev/null +++ b/tests/run-make/crate-hash-dep-extra-filename/b.rs @@ -0,0 +1,9 @@ +extern crate a; + +pub fn g() -> u32 { + a::f() +} + +pub fn make() -> a::S { + a::S { field: 2 } +} diff --git a/tests/run-make/crate-hash-dep-extra-filename/rmake.rs b/tests/run-make/crate-hash-dep-extra-filename/rmake.rs new file mode 100644 index 0000000000000..e5e4cfe56c31c --- /dev/null +++ b/tests/run-make/crate-hash-dep-extra-filename/rmake.rs @@ -0,0 +1,89 @@ +//@ needs-target-std +// +// A dependency's `extra_filename` is only a hint for locating it on disk, not +// part of its identity: the crate loader treats two libraries with equal SVHs +// as the same crate. Verifies that changing the `extra_filename` of a +// dependency does not change the crate hash of its dependents. +// See https://github.com/rust-lang/rust/issues/94878 and PR #154724. + +use run_make_support::{diff, rfs, rustc}; + +fn build_a(dir: &str, extra_filename: &str) -> String { + // Build `a` into `dir` under the given `-C extra-filename`, and return the + // path of the rlib it produced. + let mut cmd = rustc(); + cmd.input("a.rs").crate_name("a").crate_type("rlib").out_dir(dir); + if !extra_filename.is_empty() { + cmd.arg(format!("-Cextra-filename={extra_filename}")); + } + cmd.run(); + format!("{dir}/liba{extra_filename}.rlib") +} + +fn build_b(dir: &str, a_rlib: &str) -> String { + // Build `b` against the given build of `a`, then dump `b`'s SVH. Every + // build of `b` is invoked identically; only which `a` it links differs. + rustc() + .input("b.rs") + .crate_name("b") + .crate_type("rlib") + .extern_("a", a_rlib) + .out_dir(dir) + .run(); + svh(&format!("{dir}/libb.rlib")) +} + +fn svh(rlib: &str) -> String { + // Only the `hash` line of the `-Zls=root` dump. The rest of the dump lists + // the dependencies, and their `extra_filename`s are exactly what varies + // here, so comparing it whole would differ for uninteresting reasons. + let dump = rustc().arg("-Zls=root").input(rlib).run().stdout_utf8(); + dump.lines() + .find(|line| line.starts_with("hash ")) + .expect("`-Zls=root` printed no `hash` line") + .to_owned() +} + +fn main() { + rfs::create_dir("a-one"); + rfs::create_dir("a-two"); + rfs::create_dir("a-here"); + rfs::create_dir("a-there"); + rfs::create_dir("b-one"); + rfs::create_dir("b-two"); + rfs::create_dir("b-here"); + rfs::create_dir("b-there"); + rfs::create_dir("b-one-again"); + + // Two builds of `a` from byte-identical sources, differing only in + // `-C extra-filename`. + let a_one = build_a("a-one", "-one"); + let a_two = build_a("a-two", "-two"); + + // A crate's own `extra_filename` is already excluded from its own SVH, so + // the two builds of `a` must hash identically. Without this the rest of + // the test would prove nothing, because `b` would be entitled to change + // on account of `a` having changed. + let a_one_svh = svh(&a_one); + diff().expected_text("a-one", &a_one_svh).actual_text("a-two", svh(&a_two)).run(); + + // The property under test: `a`'s `extra_filename` must not reach `b`'s SVH. + let b_one = build_b("b-one", &a_one); + let b_two = build_b("b-two", &a_two); + diff().expected_text("b-one", &b_one).actual_text("b-two", b_two).run(); + + // Those two builds also read `a` from different directories, so pin that a + // dependency's *path* does not move `b`'s SVH either. This is what makes + // the comparison above attributable to `extra_filename`. + let a_here = build_a("a-here", ""); + let a_there = build_a("a-there", ""); + let b_here = build_b("b-here", &a_here); + let b_there = build_b("b-there", &a_there); + diff().expected_text("b-here", &b_here).actual_text("b-there", b_there).run(); + + // Sanity: rebuilding `b` against the original `a` reproduces its SVH, so + // the comparisons above are not passing by accident of metadata encoding + // being non-deterministic. + let b_one_again = build_b("b-one-again", &a_one); + diff().expected_text("b-one", &b_one).actual_text("b-one-again", b_one_again).run(); +} diff --git a/tests/run-make/crate-hash-metadata-flag/foo.rs b/tests/run-make/crate-hash-metadata-flag/foo.rs new file mode 100644 index 0000000000000..08fd8d55b09d9 --- /dev/null +++ b/tests/run-make/crate-hash-metadata-flag/foo.rs @@ -0,0 +1,7 @@ +pub fn foo() -> u32 { + 42 +} + +pub struct Bar { + pub x: u32, +} diff --git a/tests/run-make/crate-hash-metadata-flag/rmake.rs b/tests/run-make/crate-hash-metadata-flag/rmake.rs new file mode 100644 index 0000000000000..342a2c565e8c3 --- /dev/null +++ b/tests/run-make/crate-hash-metadata-flag/rmake.rs @@ -0,0 +1,41 @@ +// `-Z metadata-crate-hash=no` is the safety fallback that reverts the crate hash (SVH) +// computation from the encoded crate metadata back to the legacy HIR-based scheme. This test +// checks that the flag actually changes the SVH for an otherwise identical crate, and that each +// mode is deterministic. + +//@ ignore-cross-compile + +use run_make_support::{diff, rfs, rustc}; + +/// Build `foo.rs` into `dir` (optionally with the legacy hashing flag) and return the +/// `-Zls=root` metadata dump, which includes the crate hash (SVH). +fn build_in(dir: &str, metadata_crate_hash: bool) -> String { + let mut cmd = rustc(); + cmd.input("foo.rs").crate_type("rlib").out_dir(dir); + if !metadata_crate_hash { + cmd.arg("-Zmetadata-crate-hash=no"); + } + cmd.run(); + rustc().arg("-Zls=root").input(format!("{dir}/libfoo.rlib")).run().stdout_utf8() +} + +fn main() { + rfs::create_dir("default"); + rfs::create_dir("legacy"); + rfs::create_dir("default_again"); + rfs::create_dir("legacy_again"); + + let default = build_in("default", true); + let legacy = build_in("legacy", false); + + // The SVH (printed by `-Zls=root`) must differ between the metadata-based default and the + // legacy HIR-based scheme. + diff().expected_text("default", &default).actual_text("legacy", &legacy).run_fail(); + + // Each mode must be deterministic. + let default_again = build_in("default_again", true); + diff().expected_text("default", &default).actual_text("default_again", default_again).run(); + + let legacy_again = build_in("legacy_again", false); + diff().expected_text("legacy", &legacy).actual_text("legacy_again", legacy_again).run(); +} diff --git a/tests/run-make/proc-macro-dep-source-changes-crate-hash/foo.rs b/tests/run-make/proc-macro-dep-source-changes-crate-hash/foo.rs new file mode 100644 index 0000000000000..d645ab8680949 --- /dev/null +++ b/tests/run-make/proc-macro-dep-source-changes-crate-hash/foo.rs @@ -0,0 +1,16 @@ +// Consumer crate. Byte-identical across all invocations of the test; +// only the tokens spliced in by `#[derive(ChangingDerive)]` change between +// builds, driven by which version of the proc-macro is on disk. + +#![crate_type = "rlib"] + +extern crate changing_macro; + +use changing_macro::ChangingDerive; + +#[derive(ChangingDerive)] +pub struct Foo; + +pub fn answer() -> u32 { + ANSWER +} diff --git a/tests/run-make/proc-macro-dep-source-changes-crate-hash/rmake.rs b/tests/run-make/proc-macro-dep-source-changes-crate-hash/rmake.rs new file mode 100644 index 0000000000000..1d6f285464043 --- /dev/null +++ b/tests/run-make/proc-macro-dep-source-changes-crate-hash/rmake.rs @@ -0,0 +1,41 @@ +// Verifies that when the *source* of a proc-macro dependency changes (so the +// tokens it emits in the consumer crate change), the consumer crate's +// crate_hash / SVH changes. +// See https://github.com/rust-lang/rust/issues/94878 and PR #154724. + +//@ needs-crate-type: proc-macro + +use run_make_support::{diff, rfs, rustc}; + +fn build_in(dir: &str, macro_src: &str) -> String { + // Build the proc-macro and the consumer into `dir`, then dump the + // consumer's crate metadata root (which includes the SVH). + rustc() + .input(macro_src) + .crate_name("changing_macro") + .crate_type("proc-macro") + .out_dir(dir) + .run(); + rustc().input("foo.rs").library_search_path(dir).out_dir(dir).run(); + rustc().arg("-Zls=root").input(format!("{dir}/libfoo.rlib")).run().stdout_utf8() +} + +fn main() { + rfs::create_dir("v1"); + rfs::create_dir("v2"); + rfs::create_dir("v1_again"); + + // Build the consumer against proc-macro v1, then against v2. foo.rs is + // byte-identical across builds; only the tokens spliced in by the derive + // differ. + let v1 = build_in("v1", "v1.rs"); + let v2 = build_in("v2", "v2.rs"); + // The SVH (printed by `-Zls=root`) must differ between the two builds. + diff().expected_text("v1", &v1).actual_text("v2", v2).run_fail(); + + // Sanity: rebuilding against v1 reproduces the original dump, so the + // difference above is genuinely caused by the proc-macro source change + // and not by non-determinism in metadata encoding. + let v1_again = build_in("v1_again", "v1.rs"); + diff().expected_text("v1", &v1).actual_text("v1_again", v1_again).run(); +} diff --git a/tests/run-make/proc-macro-dep-source-changes-crate-hash/v1.rs b/tests/run-make/proc-macro-dep-source-changes-crate-hash/v1.rs new file mode 100644 index 0000000000000..056e3f142212c --- /dev/null +++ b/tests/run-make/proc-macro-dep-source-changes-crate-hash/v1.rs @@ -0,0 +1,12 @@ +// First version of the proc-macro. Emits `pub const ANSWER: u32 = 1;`. + +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro_derive(ChangingDerive)] +pub fn changing_derive(_input: TokenStream) -> TokenStream { + "pub const ANSWER: u32 = 1;".parse().unwrap() +} diff --git a/tests/run-make/proc-macro-dep-source-changes-crate-hash/v2.rs b/tests/run-make/proc-macro-dep-source-changes-crate-hash/v2.rs new file mode 100644 index 0000000000000..c739934013967 --- /dev/null +++ b/tests/run-make/proc-macro-dep-source-changes-crate-hash/v2.rs @@ -0,0 +1,13 @@ +// Second version of the proc-macro. Source has changed: it now emits +// `pub const ANSWER: u32 = 2;`. Crate name and exported macro name match v1. + +#![crate_type = "proc-macro"] + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro_derive(ChangingDerive)] +pub fn changing_derive(_input: TokenStream) -> TokenStream { + "pub const ANSWER: u32 = 2;".parse().unwrap() +} diff --git a/tests/run-make/proc-macro-env-changes-crate-hash/changing_macro.rs b/tests/run-make/proc-macro-env-changes-crate-hash/changing_macro.rs new file mode 100644 index 0000000000000..53c6e17bafad2 --- /dev/null +++ b/tests/run-make/proc-macro-env-changes-crate-hash/changing_macro.rs @@ -0,0 +1,15 @@ +// A proc-macro whose output depends on the value of `PROC_MACRO_DEP_TOKEN` +// at the time the *consumer* crate is compiled. The source of this crate is +// stable across the test; only the env var differs between runs. + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro] +pub fn emit_token(_input: TokenStream) -> TokenStream { + let value = std::env::var("PROC_MACRO_DEP_TOKEN").unwrap(); + // Emit a constant whose value embeds the env var, so the tokens the + // consumer ends up with depend on the env var. + format!("pub const TOKEN: &str = {value:?};").parse().unwrap() +} diff --git a/tests/run-make/proc-macro-env-changes-crate-hash/foo.rs b/tests/run-make/proc-macro-env-changes-crate-hash/foo.rs new file mode 100644 index 0000000000000..04279a659397e --- /dev/null +++ b/tests/run-make/proc-macro-env-changes-crate-hash/foo.rs @@ -0,0 +1,9 @@ +#![crate_type = "rlib"] + +extern crate changing_macro; + +changing_macro::emit_token!(); + +pub fn get() -> &'static str { + TOKEN +} diff --git a/tests/run-make/proc-macro-env-changes-crate-hash/rmake.rs b/tests/run-make/proc-macro-env-changes-crate-hash/rmake.rs new file mode 100644 index 0000000000000..410bbce0562a9 --- /dev/null +++ b/tests/run-make/proc-macro-env-changes-crate-hash/rmake.rs @@ -0,0 +1,54 @@ +// Verifies that when a proc-macro's *output* changes without its source +// changing — here, because the proc-macro reads an environment variable at +// expansion time and we vary that variable between consumer builds — the +// consumer crate's crate_hash / SVH changes. +// +// Companion to `proc-macro-dep-source-changes-crate-hash`: that test covers +// the case where the proc-macro source (and therefore its compiled metadata) +// changes; this test covers the case where only the tokens produced during +// expansion change. Both must invalidate the consumer's crate_hash. +// +// Note: the env var is read at consumer-compile time (when the proc-macro +// runs), so we set it on the `rustc` invocation that builds `foo.rs`, not on +// the one that builds the proc-macro itself. +// See https://github.com/rust-lang/rust/issues/94878 and PR #154724. + +//@ needs-crate-type: proc-macro + +use run_make_support::{diff, rfs, rustc}; + +const ENV_VAR: &str = "PROC_MACRO_DEP_TOKEN"; + +fn build_in(dir: &str, value: &str) -> String { + // The proc-macro is built once per build, but its source is identical; + // what differs is the value of ENV_VAR seen during expansion in the + // consumer build. + rustc() + .input("changing_macro.rs") + .crate_name("changing_macro") + .crate_type("proc-macro") + .out_dir(dir) + .run(); + rustc().input("foo.rs").library_search_path(dir).out_dir(dir).env(ENV_VAR, value).run(); + rustc().arg("-Zls=root").input(format!("{dir}/libfoo.rlib")).run().stdout_utf8() +} + +fn main() { + rfs::create_dir("a"); + rfs::create_dir("b"); + rfs::create_dir("a_again"); + + // Build the consumer twice with the same proc-macro source but different + // values of ENV_VAR. foo.rs is byte-identical; only the tokens spliced in + // by the derive differ. + let a = build_in("a", "first"); + let b = build_in("b", "second"); + // The SVH (printed by `-Zls=root`) must differ between the two builds. + diff().expected_text("a", &a).actual_text("b", b).run_fail(); + + // Sanity: rebuilding with the original env value reproduces the original + // dump, so the difference above is genuinely caused by the env change and + // not by non-determinism in metadata encoding. + let a_again = build_in("a_again", "first"); + diff().expected_text("a", &a).actual_text("a_again", a_again).run(); +} diff --git a/tests/run-make/proc-macro-global-asm-changes-crate-hash/changing_macro.rs b/tests/run-make/proc-macro-global-asm-changes-crate-hash/changing_macro.rs new file mode 100644 index 0000000000000..7949ffb7d189d --- /dev/null +++ b/tests/run-make/proc-macro-global-asm-changes-crate-hash/changing_macro.rs @@ -0,0 +1,18 @@ +// A proc-macro that emits a `core::arch::global_asm!` block whose template +// depends on `PROC_MACRO_ASM_TOKEN`, read at expansion time. The source of +// this crate is stable across the test; only the env var differs between +// runs, so the only thing that changes in the consumer is the asm template +// spliced in by the macro. +// +// The body is a pure assembler comment, so it assembles to nothing on every +// target we test on. + +extern crate proc_macro; + +use proc_macro::TokenStream; + +#[proc_macro] +pub fn emit_global_asm(_input: TokenStream) -> TokenStream { + let value = std::env::var("PROC_MACRO_ASM_TOKEN").unwrap(); + format!(r##"core::arch::global_asm!("# {}");"##, value).parse().unwrap() +} diff --git a/tests/run-make/proc-macro-global-asm-changes-crate-hash/foo.rs b/tests/run-make/proc-macro-global-asm-changes-crate-hash/foo.rs new file mode 100644 index 0000000000000..f2c6a4905e182 --- /dev/null +++ b/tests/run-make/proc-macro-global-asm-changes-crate-hash/foo.rs @@ -0,0 +1,9 @@ +// Consumer crate. Byte-identical across all invocations of the test; +// only the asm template inside the `global_asm!` block spliced in by +// `changing_macro::emit_global_asm!` differs between builds. + +#![crate_type = "rlib"] + +extern crate changing_macro; + +changing_macro::emit_global_asm!(); diff --git a/tests/run-make/proc-macro-global-asm-changes-crate-hash/rmake.rs b/tests/run-make/proc-macro-global-asm-changes-crate-hash/rmake.rs new file mode 100644 index 0000000000000..805ceb31df096 --- /dev/null +++ b/tests/run-make/proc-macro-global-asm-changes-crate-hash/rmake.rs @@ -0,0 +1,62 @@ +// Verifies that when a proc-macro emits a `global_asm!` block whose template +// changes between builds (here driven by an env var read at expansion time), +// the consumer crate's crate_hash / SVH changes. +// +// This exercises an item kind (`DefKind::GlobalAsm`) whose body lives only +// in HIR and is *not* recorded in any way by the metadata encoder: +// `should_encode_span`, `should_encode_attrs`, `should_encode_visibility`, +// `should_encode_generics`, `should_encode_type` and `should_encode_mir` are +// all false for `GlobalAsm`, and `def_kind.has_codegen_attrs()` is false too +// (see `compiler/rustc_metadata/src/rmeta/encoder.rs`). All that ends up in +// the rmeta byte stream for a `global_asm!` invocation is the fixed-size +// `DefKind::GlobalAsm` enum discriminant in the def_kind table, which is +// identical regardless of the asm template's contents. The asm template +// itself is only read out of HIR later by `MonoItem::GlobalAsm` codegen in +// `rustc_monomorphize::collector`. +// +// Companion to `proc-macro-dep-source-changes-crate-hash` and +// `proc-macro-env-changes-crate-hash`. +// See https://github.com/rust-lang/rust/issues/94878 and PR #154724. + +//@ needs-crate-type: proc-macro + +use run_make_support::{diff, rfs, rustc}; + +const ENV_VAR: &str = "PROC_MACRO_ASM_TOKEN"; + +fn build_in(dir: &str, value: &str) -> String { + // The proc-macro is built once per build, but its source is identical; + // what differs is the value of ENV_VAR seen during expansion in the + // consumer build. + rustc() + .input("changing_macro.rs") + .crate_name("changing_macro") + .crate_type("proc-macro") + .out_dir(dir) + .run(); + rustc().input("foo.rs").library_search_path(dir).out_dir(dir).env(ENV_VAR, value).run(); + rustc().arg("-Zls=root").input(format!("{dir}/libfoo.rlib")).run().stdout_utf8() +} + +fn main() { + rfs::create_dir("a"); + rfs::create_dir("b"); + rfs::create_dir("a_again"); + + // Build the consumer twice with the same proc-macro source but different + // values of ENV_VAR. foo.rs is byte-identical; only the asm template + // spliced in by `emit_global_asm!` differs. + let a = build_in("a", "first"); + let b = build_in("b", "second"); + // The SVH (printed by `-Zls=root`) must differ between the two builds. + // Under PR #154724 without an HIR-hash contribution, the rmeta encoder + // writes no bytes that depend on the asm template, so the two SVHs are + // identical and this `run_fail` fails (the dumps match). + diff().expected_text("a", &a).actual_text("b", b).run_fail(); + + // Sanity: rebuilding with the original env value reproduces the original + // dump, so the difference above is genuinely caused by the env change + // and not by non-determinism in metadata encoding. + let a_again = build_in("a_again", "first"); + diff().expected_text("a", &a).actual_text("a_again", a_again).run(); +}