Fix allocation bug if compiled with modern clang or gcc

* Add overflow checked intrinsic abstractions that check on overflow.
* Use them for memory allocation code.
* Use size_t type for memory allocation code to support full platform dependent width.

Fixes #3756.

1 files changed, 26 insertions, 9 deletions

diff --git a/core/vector.h b/core/vector.h
index 641721f401..398d7f1bd5 100644
--- a/core/vector.h
+++ b/core/vector.h
@@ -68,11 +68,26 @@ class Vector {
 		return reinterpret_cast<T*>(_ptr);
  		
  	}
- 	
-	_FORCE_INLINE_ int _get_alloc_size(int p_elements) const {
- 	
- 		return  nearest_power_of_2(p_elements*sizeof(T)+sizeof(SafeRefCount)+sizeof(int));
- 	}
+
+	_FORCE_INLINE_ size_t _get_alloc_size(size_t p_elements) const {
+		return nearest_power_of_2_templated(p_elements*sizeof(T)+sizeof(SafeRefCount)+sizeof(int));
+	}
+
+	_FORCE_INLINE_ bool _get_alloc_size_checked(size_t p_elements, size_t *out) const {
+#if defined(_add_overflow) && defined(_mul_overflow)
+		size_t o;
+		size_t p;
+		if (_mul_overflow(p_elements, sizeof(T), &o)) return false;
+		if (_add_overflow(o, sizeof(SafeRefCount)+sizeof(int), &p)) return false;
+		*out = nearest_power_of_2_templated(p);
+		return true;
+#else
+		// Speed is more important than correctness here, do the operations unchecked
+		// and hope the best
+		*out = _get_alloc_size(p_elements);
+		return true;
+#endif
+	}
  	
 	void _unref(void *p_data);
 	
@@ -257,19 +272,21 @@ Error Vector<T>::resize(int p_size) {
 	// possibly changing size, copy on write
 	_copy_on_write();
 	
+	size_t alloc_size;
+	ERR_FAIL_COND_V(!_get_alloc_size_checked(p_size, &alloc_size), ERR_OUT_OF_MEMORY);
+
 	if (p_size>size()) {
 
 		if (size()==0) {
 			// alloc from scratch
-			void* ptr=memalloc(_get_alloc_size(p_size));
+			void* ptr=memalloc(alloc_size);
 			ERR_FAIL_COND_V( !ptr ,ERR_OUT_OF_MEMORY);
 			_ptr=(T*)((uint8_t*)ptr+sizeof(int)+sizeof(SafeRefCount));
 			_get_refcount()->init(); // init refcount
 			*_get_size()=0; // init size (currently, none)
 
 		} else {
-			
-			void *_ptrnew = (T*)memrealloc((uint8_t*)_ptr-sizeof(int)-sizeof(SafeRefCount),_get_alloc_size(p_size));
+			void *_ptrnew = (T*)memrealloc((uint8_t*)_ptr-sizeof(int)-sizeof(SafeRefCount), alloc_size);
 			ERR_FAIL_COND_V( !_ptrnew ,ERR_OUT_OF_MEMORY);
 			_ptr=(T*)((uint8_t*)_ptrnew+sizeof(int)+sizeof(SafeRefCount));
 		}
@@ -293,7 +310,7 @@ Error Vector<T>::resize(int p_size) {
 			t->~T();
 		}
 
-		void *_ptrnew = (T*)memrealloc((uint8_t*)_ptr-sizeof(int)-sizeof(SafeRefCount),_get_alloc_size(p_size));
+		void *_ptrnew = (T*)memrealloc((uint8_t*)_ptr-sizeof(int)-sizeof(SafeRefCount), alloc_size);
 		ERR_FAIL_COND_V( !_ptrnew ,ERR_OUT_OF_MEMORY);
 		
 		_ptr=(T*)((uint8_t*)_ptrnew+sizeof(int)+sizeof(SafeRefCount));