Auto merge of #154724 - Daniel-B-Smith:smithdb3/fix-94878, r=<try>

Computing crate_hash from metadata encoding instead of HIR (implements #94878) (very draft)

Author: bors

compiler/rustc_ast_lowering/src/lib.rs

@@ -562,8 +562,11 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> mid_hir::Crate<'_> {
     }
 
     // Don't hash unless necessary, because it's expensive.
-    let opt_hir_hash =
-        if tcx.needs_crate_hash() { Some(compute_hir_hash(tcx, &owners)) } else { None };
+    let opt_hir_hash = if tcx.needs_crate_hash() && !tcx.needs_metadata() {
+        Some(compute_hir_hash(tcx, &owners))
+    } else {
+        None
+    };
 
     let delayed_resolver = Steal::new((resolver, krate));
     mid_hir::Crate::new(owners, delayed_ids, delayed_resolver, opt_hir_hash)

compiler/rustc_driver_impl/src/lib.rs

@@ -324,10 +324,6 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
 
             tcx.ensure_ok().analysis(());
 
-            if let Some(metrics_dir) = &sess.opts.unstable_opts.metrics_dir {
-                dump_feature_usage_metrics(tcx, metrics_dir);
-            }
-
             if callbacks.after_analysis(compiler, tcx) == Compilation::Stop {
                 return early_exit();
             }
@@ -340,6 +336,10 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
 
             let linker = Linker::codegen_and_build_linker(tcx, &*compiler.codegen_backend);
 
+            if let Some(metrics_dir) = &sess.opts.unstable_opts.metrics_dir {
+                dump_feature_usage_metrics(tcx, metrics_dir);
+            }
+
             tcx.report_unused_features();
 
             Some(linker)

compiler/rustc_interface/src/passes.rs

@@ -948,8 +948,13 @@ pub fn create_and_enter_global_ctxt<T, F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> T>(
     let definitions = FreezeLock::new(Definitions::new(stable_crate_id));
 
     let stable_crate_ids = FreezeLock::new(StableCrateIdMap::default());
-    let untracked =
-        Untracked { cstore, source_span: AppendOnlyIndexVec::new(), definitions, stable_crate_ids };
+    let untracked = Untracked {
+        cstore,
+        source_span: AppendOnlyIndexVec::new(),
+        definitions,
+        stable_crate_ids,
+        local_crate_hash: OnceLock::new(),
+    };
 
     // We're constructing the HIR here; we don't care what we will
     // read, since we haven't even constructed the *input* to

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -2,13 +2,17 @@ use std::any::Any;
 use std::mem;
 use std::sync::Arc;
 
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_data_structures::svh::Svh;
 use rustc_hir::attrs::Deprecation;
 use rustc_hir::def::{CtorKind, DefKind};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathHash};
 use rustc_middle::arena::ArenaAllocatable;
 use rustc_middle::bug;
 use rustc_middle::metadata::{AmbigModChild, ModChild};
+use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
 use rustc_middle::middle::exported_symbols::ExportedSymbol;
 use rustc_middle::middle::stability::DeprecationEntry;
 use rustc_middle::queries::ExternProviders;
@@ -20,7 +24,7 @@ use rustc_serialize::Decoder;
 use rustc_session::StableCrateId;
 use rustc_session::cstore::{CrateStore, ExternCrate};
 use rustc_span::hygiene::ExpnId;
-use rustc_span::{Span, Symbol, kw};
+use rustc_span::{Span, Symbol, kw, with_metavar_spans};
 
 use super::{Decodable, DecodeIterator};
 use crate::creader::{CStore, LoadedMacro};
@@ -750,4 +754,104 @@ fn provide_cstore_hooks(providers: &mut Providers) {
             cdata.imported_source_file(tcx, file_index as u32);
         }
     };
+
+    providers.queries.crate_hash = |tcx: TyCtxt<'_>, _: LocalCrate| {
+        if tcx.needs_metadata() {
+            *tcx.untracked()
+                .local_crate_hash
+                .get()
+                .expect("crate_hash(LOCAL_CRATE) called before metadata encoding")
+        } else {
+            crate_hash(tcx)
+        }
+    };
+}
+
+pub(super) fn crate_hash(tcx: TyCtxt<'_>) -> Svh {
+    let krate = tcx.hir_crate(());
+    let hir_body_hash = krate.opt_hir_hash.expect("HIR hash missing while computing crate hash");
+
+    let upstream_crates = upstream_crates(tcx);
+
+    let resolutions = tcx.resolutions(());
+
+    // We hash the final, remapped names of all local source files so we
+    // don't have to include the path prefix remapping commandline args.
+    // If we included the full mapping in the SVH, we could only have
+    // reproducible builds by compiling from the same directory. So we just
+    // hash the result of the mapping instead of the mapping itself.
+    let mut source_file_names: Vec<_> = tcx
+        .sess
+        .source_map()
+        .files()
+        .iter()
+        .filter(|source_file| source_file.cnum == LOCAL_CRATE)
+        .map(|source_file| source_file.stable_id)
+        .collect();
+
+    source_file_names.sort_unstable();
+
+    // We have to take care of debugger visualizers explicitly. The HIR (and
+    // thus `hir_body_hash`) contains the #[debugger_visualizer] attributes but
+    // these attributes only store the file path to the visualizer file, not
+    // their content. Yet that content is exported into crate metadata, so any
+    // changes to it need to be reflected in the crate hash.
+    let debugger_visualizers: Vec<_> = tcx
+        .debugger_visualizers(LOCAL_CRATE)
+        .iter()
+        // We ignore the path to the visualizer file since it's not going to be
+        // encoded in crate metadata and we already hash the full contents of
+        // the file.
+        .map(DebuggerVisualizerFile::path_erased)
+        .collect();
+
+    let crate_hash: Fingerprint = tcx.with_stable_hashing_context(|mut hcx| {
+        let mut stable_hasher = StableHasher::new();
+        hir_body_hash.hash_stable(&mut hcx, &mut stable_hasher);
+        upstream_crates.hash_stable(&mut hcx, &mut stable_hasher);
+        source_file_names.hash_stable(&mut hcx, &mut stable_hasher);
+        debugger_visualizers.hash_stable(&mut hcx, &mut stable_hasher);
+        if tcx.sess.opts.incremental.is_some() {
+            let definitions = tcx.untracked().definitions.freeze();
+            let mut owner_spans: Vec<_> = tcx
+                .hir_crate_items(())
+                .definitions()
+                .map(|def_id| {
+                    let def_path_hash = definitions.def_path_hash(def_id);
+                    let span = tcx.source_span(def_id);
+                    debug_assert_eq!(span.parent(), None);
+                    (def_path_hash, span)
+                })
+                .collect();
+            owner_spans.sort_unstable_by_key(|bn| bn.0);
+            owner_spans.hash_stable(&mut hcx, &mut stable_hasher);
+        }
+        tcx.sess.opts.dep_tracking_hash(true).hash_stable(&mut hcx, &mut stable_hasher);
+        tcx.stable_crate_id(LOCAL_CRATE).hash_stable(&mut hcx, &mut stable_hasher);
+        // Hash visibility information since it does not appear in HIR.
+        // FIXME: Figure out how to remove `visibilities_for_hashing` by hashing visibilities on
+        // the fly in the resolver, storing only their accumulated hash in `ResolverGlobalCtxt`,
+        // and combining it with other hashes here.
+        resolutions.visibilities_for_hashing.hash_stable(&mut hcx, &mut stable_hasher);
+        with_metavar_spans(|mspans| {
+            mspans.freeze_and_get_read_spans().hash_stable(&mut hcx, &mut stable_hasher);
+        });
+        stable_hasher.finish()
+    });
+
+    Svh::new(crate_hash)
+}
+
+fn upstream_crates(tcx: TyCtxt<'_>) -> Vec<(StableCrateId, Svh)> {
+    let mut upstream_crates: Vec<_> = tcx
+        .crates(())
+        .iter()
+        .map(|&cnum| {
+            let stable_crate_id = tcx.stable_crate_id(cnum);
+            let hash = tcx.crate_hash(cnum);
+            (stable_crate_id, hash)
+        })
+        .collect();
+    upstream_crates.sort_unstable_by_key(|&(stable_crate_id, _)| stable_crate_id);
+    upstream_crates
 }