Auto merge of #23849 - bcoopers:master, r=pnkfelix

Right now, if the user requests to increase the vector size via reserve() or push_back() and the request brings the attempted memory above usize::MAX, we panic.

With this change there is only a panic if the minimum requested memory that could meet the requirement is above usize::MAX- otherwise it simply requests its largest capacity possible, usize::MAX.

Author: bors

src/libcollections/vec.rs

@@ -65,10 +65,14 @@ use core::ops;
 use core::ptr;
 use core::ptr::Unique;
 use core::slice;
+use core::isize;
 use core::usize;
 
 use borrow::{Cow, IntoCow};
 
+// FIXME- fix places which assume the max vector allowed has memory usize::MAX.
+static MAX_MEMORY_SIZE: usize = isize::MAX as usize;
+
 /// A growable list type, written `Vec<T>` but pronounced 'vector.'
 ///
 /// # Examples
@@ -307,10 +311,15 @@ impl<T> Vec<T> {
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn reserve(&mut self, additional: usize) {
         if self.cap - self.len < additional {
-            let err_msg = "Vec::reserve: `usize` overflow";
-            let new_cap = self.len.checked_add(additional).expect(err_msg)
-                .checked_next_power_of_two().expect(err_msg);
-            self.grow_capacity(new_cap);
+            const ERR_MSG: &'static str  = "Vec::reserve: `isize` overflow";
+
+            let new_min_cap = self.len.checked_add(additional).expect(ERR_MSG);
+            if new_min_cap > MAX_MEMORY_SIZE { panic!(ERR_MSG) }
+            self.grow_capacity(match new_min_cap.checked_next_power_of_two() {
+                Some(x) if x > MAX_MEMORY_SIZE => MAX_MEMORY_SIZE,
+                None => MAX_MEMORY_SIZE,
+                Some(x) => x,
+            });
         }
     }
 
@@ -647,8 +656,11 @@ impl<T> Vec<T> {
         #[inline(never)]
         fn resize<T>(vec: &mut Vec<T>) {
             let old_size = vec.cap * mem::size_of::<T>();
-            let size = max(old_size, 2 * mem::size_of::<T>()) * 2;
-            if old_size > size { panic!("capacity overflow") }
+            if old_size >= MAX_MEMORY_SIZE { panic!("capacity overflow") }
+            let mut size = max(old_size, 2 * mem::size_of::<T>()) * 2;
+            if old_size > size || size > MAX_MEMORY_SIZE {
+                size = MAX_MEMORY_SIZE;
+            }
             unsafe {
                 let ptr = alloc_or_realloc(*vec.ptr, old_size, size);
                 if ptr.is_null() { ::alloc::oom() }