Remove fewer Storage calls in CopyProp and GVN

compiler/rustc_mir_dataflow/src/impls/initialized.rs

@@ -558,6 +558,80 @@ impl<'tcx> Analysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
     }
 }
 
+/// A dataflow analysis that tracks locals that are maybe uninitialized.
+///
+/// This is a simpler analysis than `MaybeUninitializedPlaces`, because it does not track
+/// individual fields.
+pub struct MaybeUninitializedLocals;
+
+impl MaybeUninitializedLocals {
+    pub fn new() -> Self {
+        Self {}
+    }
+}
+
+impl<'tcx> Analysis<'tcx> for MaybeUninitializedLocals {
+    type Domain = DenseBitSet<mir::Local>;
+
+    const NAME: &'static str = "maybe_uninit_locals";
+
+    fn bottom_value(&self, body: &Body<'tcx>) -> Self::Domain {
+        // bottom = all locals are initialized.
+        DenseBitSet::new_empty(body.local_decls.len())
+    }
+
+    fn initialize_start_block(&self, body: &Body<'tcx>, state: &mut Self::Domain) {
+        // All locals start as uninitialized...
+        state.insert_all();
+        // ...except for arguments, which are definitely initialized.
+        for arg in body.args_iter() {
+            state.remove(arg);
+        }
+    }
+
+    fn apply_primary_statement_effect(
+        &mut self,
+        state: &mut Self::Domain,
+        statement: &mir::Statement<'tcx>,
+        _location: Location,
+    ) {
+        match statement.kind {
+            // An assignment makes a local initialized.
+            mir::StatementKind::Assign(box (place, _)) => {
+                if let Some(local) = place.as_local() {
+                    state.remove(local);
+                }
+            }
+            // Deinit makes the local uninitialized.
+            mir::StatementKind::Deinit(box place) => {
+                // A deinit makes a local uninitialized.
+                if let Some(local) = place.as_local() {
+                    state.insert(local);
+                }
+            }
+            // Storage{Live,Dead} makes a local uninitialized.
+            mir::StatementKind::StorageLive(local) | mir::StatementKind::StorageDead(local) => {
+                state.insert(local);
+            }
+            _ => {}
+        }
+    }
+
+    fn apply_call_return_effect(
+        &mut self,
+        state: &mut Self::Domain,
+        _block: mir::BasicBlock,
+        return_places: CallReturnPlaces<'_, 'tcx>,
+    ) {
+        // The return place of a call is initialized.
+        return_places.for_each(|place| {
+            if let Some(local) = place.as_local() {
+                state.remove(local);
+            }
+        });
+    }
+}
+
 /// There can be many more `InitIndex` than there are locals in a MIR body.
 /// We use a mixed bitset to avoid paying too high a memory footprint.
 pub type EverInitializedPlacesDomain = MixedBitSet<InitIndex>;

compiler/rustc_mir_dataflow/src/impls/mod.rs

@@ -6,7 +6,7 @@ mod storage_liveness;
 pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals};
 pub use self::initialized::{
     EverInitializedPlaces, EverInitializedPlacesDomain, MaybeInitializedPlaces,
-    MaybeUninitializedPlaces, MaybeUninitializedPlacesDomain,
+    MaybeUninitializedLocals, MaybeUninitializedPlaces, MaybeUninitializedPlacesDomain,
 };
 pub use self::liveness::{
     MaybeLiveLocals, MaybeTransitiveLiveLocals, TransferFunction as LivenessTransferFunction,

compiler/rustc_mir_transform/src/copy_prop.rs

@@ -3,6 +3,8 @@ use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
+use rustc_mir_dataflow::impls::MaybeUninitializedLocals;
+use rustc_mir_dataflow::{Analysis, ResultsCursor};
 use tracing::{debug, instrument};
 
 use crate::ssa::SsaLocals;
@@ -16,7 +18,7 @@ use crate::ssa::SsaLocals;
 ///   _d = move? _c
 /// where each of the locals is only assigned once.
 ///
-/// We want to replace all those locals by `_a`, either copied or moved.
+/// We want to replace all those locals by `_a` (the "head"), either copied or moved.
 pub(super) struct CopyProp;
 
 impl<'tcx> crate::MirPass<'tcx> for CopyProp {
@@ -30,21 +32,56 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp {
 
         let typing_env = body.typing_env(tcx);
         let ssa = SsaLocals::new(tcx, body, typing_env);
-        debug!(borrowed_locals = ?ssa.borrowed_locals());
+        let borrowed_locals = ssa.borrowed_locals().clone();
+
+        debug!(?borrowed_locals);
         debug!(copy_classes = ?ssa.copy_classes());
 
         let fully_moved = fully_moved_locals(&ssa, body);
         debug!(?fully_moved);
 
+        let mut head_storage_to_check = DenseBitSet::new_empty(fully_moved.domain_size());
         let mut storage_to_remove = DenseBitSet::new_empty(fully_moved.domain_size());
+
         for (local, &head) in ssa.copy_classes().iter_enumerated() {
             if local != head {
-                storage_to_remove.insert(head);
+                // We need to determine if we can keep the head's storage statements (which enables better optimizations).
+                // For every local's usage location, if the head is maybe-uninitialized, we'll need to remove it's storage statements.
+                head_storage_to_check.insert(head);
+
+                if borrowed_locals.contains(local) {
+                    // To keep the storage of a head, we require that none of the locals in it's copy class are borrowed,
+                    // since otherwise we cannot easily identify when it is used.
+                    storage_to_remove.insert(head);
+                }
             }
         }
 
         let any_replacement = ssa.copy_classes().iter_enumerated().any(|(l, &h)| l != h);
 
+        // Debug builds have no use for the storage statements, so avoid extra work.
+        let storage_to_remove = if any_replacement && tcx.sess.emit_lifetime_markers() {
+            let maybe_uninit = MaybeUninitializedLocals::new()
+                .iterate_to_fixpoint(tcx, body, Some("mir_opt::copy_prop"))
+                .into_results_cursor(body);
+
+            let mut storage_checker = StorageChecker {
+                maybe_uninit,
+                copy_classes: ssa.copy_classes(),
+                head_storage_to_check,
+                storage_to_remove,
+            };
+
+            storage_checker.visit_body(body);
+
+            storage_checker.storage_to_remove
+        } else {
+            // Conservatively remove all storage statements for the head locals.
+            head_storage_to_check
+        };
+
+        debug!(?storage_to_remove);
+
         Replacer { tcx, copy_classes: ssa.copy_classes(), fully_moved, storage_to_remove }
             .visit_body_preserves_cfg(body);
 
@@ -152,3 +189,51 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
         }
     }
 }
+
+// Marks heads of copy classes that are maybe uninitialized at the location of a local
+// as needing storage statement removal.
+struct StorageChecker<'a, 'tcx> {
+    maybe_uninit: ResultsCursor<'a, 'tcx, MaybeUninitializedLocals>,
+    copy_classes: &'a IndexSlice<Local, Local>,
+    head_storage_to_check: DenseBitSet<Local>,
+    storage_to_remove: DenseBitSet<Local>,
+}
+
+impl<'a, 'tcx> Visitor<'tcx> for StorageChecker<'a, 'tcx> {
+    fn visit_local(&mut self, local: Local, context: PlaceContext, loc: Location) {
+        // We don't need to check storage statements and statements for which the local doesn't need to be initialized.
+        match context {
+            PlaceContext::MutatingUse(
+                MutatingUseContext::Store
+                | MutatingUseContext::Call
+                | MutatingUseContext::Yield
+                | MutatingUseContext::AsmOutput,
+            )
+            | PlaceContext::NonUse(_) => {
+                return;
+            }
+            _ => {}
+        };
+
+        let head = self.copy_classes[local];
+
+        // The head must be initialized at the location of the local, otherwise we must remove it's storage statements.
+        if self.head_storage_to_check.contains(head) {
+            self.maybe_uninit.seek_before_primary_effect(loc);
+
+            if self.maybe_uninit.get().contains(head) {
+                debug!(
+                    ?loc,
+                    ?context,
+                    ?local,
+                    ?head,
+                    "found a head at a location in which it is maybe uninit, marking head for storage statement removal"
+                );
+                self.storage_to_remove.insert(head);
+
+                // Once we found a use of the head that is maybe uninit, we do not need to check it again.
+                self.head_storage_to_check.remove(head);
+            }
+        }
+    }
+}

compiler/rustc_mir_transform/src/gvn.rs

@@ -104,6 +104,8 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
 use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_mir_dataflow::impls::MaybeUninitializedLocals;
+use rustc_mir_dataflow::{Analysis, ResultsCursor};
 use rustc_span::DUMMY_SP;
 use rustc_span::def_id::DefId;
 use smallvec::SmallVec;
@@ -140,10 +142,31 @@ impl<'tcx> crate::MirPass<'tcx> for GVN {
             state.visit_basic_block_data(bb, data);
         }
 
-        // For each local that is reused (`y` above), we remove its storage statements do avoid any
-        // difficulty. Those locals are SSA, so should be easy to optimize by LLVM without storage
-        // statements.
-        StorageRemover { tcx, reused_locals: state.reused_locals }.visit_body_preserves_cfg(body);
+        // If we emit storage annotations, use `MaybeStorageDead` to check which reused locals
+        // require storage removal (making them alive for the duration of the function).
+        let storage_to_remove = if tcx.sess.emit_lifetime_markers() {
+            let maybe_uninit = MaybeUninitializedLocals::new()
+                .iterate_to_fixpoint(tcx, body, Some("mir_opt::gvn"))
+                .into_results_cursor(body);
+
+            let mut storage_checker = StorageChecker {
+                storage_to_check: state.reused_locals.clone(),
+                storage_to_remove: DenseBitSet::new_empty(body.local_decls.len()),
+                maybe_uninit,
+            };
+
+            storage_checker.visit_body(body);
+
+            storage_checker.storage_to_remove
+        } else {
+            // Conservatively remove all storage statements for reused locals.
+            state.reused_locals.clone()
+        };
+
+        debug!(?storage_to_remove);
+
+        StorageRemover { tcx, reused_locals: state.reused_locals, storage_to_remove }
+            .visit_body_preserves_cfg(body);
     }
 
     fn is_required(&self) -> bool {
@@ -1824,6 +1847,7 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
 struct StorageRemover<'tcx> {
     tcx: TyCtxt<'tcx>,
     reused_locals: DenseBitSet<Local>,
+    storage_to_remove: DenseBitSet<Local>,
 }
 
 impl<'tcx> MutVisitor<'tcx> for StorageRemover<'tcx> {
@@ -1844,11 +1868,48 @@ impl<'tcx> MutVisitor<'tcx> for StorageRemover<'tcx> {
         match stmt.kind {
             // When removing storage statements, we need to remove both (#107511).
             StatementKind::StorageLive(l) | StatementKind::StorageDead(l)
-                if self.reused_locals.contains(l) =>
+                if self.storage_to_remove.contains(l) =>
             {
                 stmt.make_nop()
             }
             _ => self.super_statement(stmt, loc),
         }
     }
 }
+
+struct StorageChecker<'a, 'tcx> {
+    storage_to_check: DenseBitSet<Local>,
+    storage_to_remove: DenseBitSet<Local>,
+    maybe_uninit: ResultsCursor<'a, 'tcx, MaybeUninitializedLocals>,
+}
+
+impl<'a, 'tcx> Visitor<'tcx> for StorageChecker<'a, 'tcx> {
+    fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) {
+        match context {
+            // These mutating uses do not require the local to be initialized.
+            PlaceContext::MutatingUse(MutatingUseContext::AsmOutput)
+            | PlaceContext::MutatingUse(MutatingUseContext::Call)
+            | PlaceContext::MutatingUse(MutatingUseContext::Store)
+            | PlaceContext::MutatingUse(MutatingUseContext::Yield)
+            | PlaceContext::NonUse(_) => {
+                return;
+            }
+            // Must check validity for other mutating usages and all non-mutating uses.
+            PlaceContext::MutatingUse(_) | PlaceContext::NonMutatingUse(_) => {}
+        }
+
+        if self.storage_to_check.contains(local) {
+            self.maybe_uninit.seek_before_primary_effect(location);
+
+            if self.maybe_uninit.get().contains(local) {
+                debug!(
+                    ?location,
+                    ?local,
+                    "local is maybe uninit in this location, removing storage"
+                );
+                self.storage_to_remove.insert(local);
+                self.storage_to_check.remove(local);
+            }
+        }
+    }
+}

tests/mir-opt/const_debuginfo.main.SingleUseConsts.diff

@@ -55,14 +55,14 @@
       }
 
       bb0: {
-          nop;
+          StorageLive(_1);
 -         _1 = const 1_u8;
-          nop;
--         _2 = const 2_u8;
-          nop;
--         _3 = const 3_u8;
 +         nop;
+          StorageLive(_2);
+-         _2 = const 2_u8;
 +         nop;
+          StorageLive(_3);
+-         _3 = const 3_u8;
 +         nop;
           StorageLive(_4);
           StorageLive(_5);
@@ -95,7 +95,7 @@
 -         _12 = const Point {{ x: 32_u32, y: 32_u32 }};
 +         nop;
           StorageLive(_13);
-          nop;
+          StorageLive(_14);
 -         _14 = const 32_u32;
 +         nop;
           StorageLive(_15);
@@ -104,17 +104,17 @@
 +         nop;
 +         nop;
           StorageDead(_15);
-          nop;
+          StorageDead(_14);
           _0 = const ();
           StorageDead(_13);
           StorageDead(_12);
           StorageDead(_11);
           StorageDead(_10);
           StorageDead(_9);
           StorageDead(_4);
-          nop;
-          nop;
-          nop;
+          StorageDead(_3);
+          StorageDead(_2);
+          StorageDead(_1);
           return;
       }
   }

tests/mir-opt/const_prop/aggregate.main.GVN.panic-abort.diff

@@ -13,8 +13,7 @@
       }
 
       bb0: {
--         StorageLive(_1);
-+         nop;
+          StorageLive(_1);
           StorageLive(_2);
           StorageLive(_3);
           _3 = (const 0_i32, const 1_u8, const 2_i32);
@@ -36,8 +35,7 @@
           StorageDead(_5);
           StorageDead(_4);
           _0 = const ();
--         StorageDead(_1);
-+         nop;
+          StorageDead(_1);
           return;
       }
   }