Auto merge of rust-lang#66394 - wesleywiser:fix_oom, r=oli-obk

Fix two OOM issues related to `ConstProp`

Fixes rust-lang#66342
Fixes rust-lang#66397

r? @oli-obk

Author: bors

src/librustc_mir/transform/const_prop.rs

@@ -22,7 +22,7 @@ use rustc::ty::subst::InternalSubsts;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_index::vec::IndexVec;
 use rustc::ty::layout::{
-    LayoutOf, TyLayout, LayoutError, HasTyCtxt, TargetDataLayout, HasDataLayout,
+    LayoutOf, TyLayout, LayoutError, HasTyCtxt, TargetDataLayout, HasDataLayout, Size,
 };
 
 use crate::rustc::ty::subst::Subst;
@@ -35,6 +35,9 @@ use crate::interpret::{
 use crate::const_eval::error_to_const_error;
 use crate::transform::{MirPass, MirSource};
 
+/// The maximum number of bytes that we'll allocate space for a return value.
+const MAX_ALLOC_LIMIT: u64 = 1024;
+
 pub struct ConstProp;
 
 impl<'tcx> MirPass<'tcx> for ConstProp {
@@ -313,8 +316,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             ecx
                 .layout_of(body.return_ty().subst(tcx, substs))
                 .ok()
-                // Don't bother allocating memory for ZST types which have no values.
-                .filter(|ret_layout| !ret_layout.is_zst())
+                // Don't bother allocating memory for ZST types which have no values
+                // or for large values.
+                .filter(|ret_layout| !ret_layout.is_zst() &&
+                                     ret_layout.size < Size::from_bytes(MAX_ALLOC_LIMIT))
                 .map(|ret_layout| ecx.allocate(ret_layout, MemoryKind::Stack));
 
         ecx.push_stack_frame(
@@ -453,6 +458,11 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
     ) -> Option<()> {
         let span = source_info.span;
 
+        // #66397: Don't try to eval into large places as that can cause an OOM
+        if place_layout.size >= Size::from_bytes(MAX_ALLOC_LIMIT) {
+            return None;
+        }
+
         let overflow_check = self.tcx.sess.overflow_checks();
 
         // Perform any special handling for specific Rvalue types.

src/test/ui/consts/issue-66342.rs

@@ -0,0 +1,12 @@
+// check-pass
+// only-x86_64
+
+// Checks that the compiler does not actually try to allocate 4 TB during compilation and OOM crash.
+
+fn foo() -> [u8; 4 * 1024 * 1024 * 1024 * 1024] {
+    unimplemented!()
+}
+
+fn main() {
+    foo();
+}

src/test/ui/consts/issue-66397.rs

@@ -0,0 +1,8 @@
+// check-pass
+// only-x86_64
+
+// Checks that the compiler does not actually try to allocate 4 TB during compilation and OOM crash.
+
+fn main() {
+    [0; 4 * 1024 * 1024 * 1024 * 1024];
+}