diff options
| author | Rémi Verschelde <rverschelde@gmail.com> | 2023-09-01 11:11:12 +0200 |
|---|---|---|
| committer | Rémi Verschelde <rverschelde@gmail.com> | 2023-09-01 11:23:48 +0200 |
| commit | 728dbeab69c21e7bead0f07604e7d044a67e9a14 (patch) | |
| tree | e864a8a23e107ef9c50152bd54be8befc1068304 /thirdparty/vulkan/vk_mem_alloc.h | |
| parent | 549fcce5f8f7beace3e5c90e9bbe4335d4fd1476 (diff) | |
| download | redot-engine-728dbeab69c21e7bead0f07604e7d044a67e9a14.tar.gz | |
vulkan: Update all components to Vulkan SDK 1.3.261.1
Updates to volk, vulkan headers, `vk_enum_string_helper.h`, VMA,
glslang, spirv-reflect.
VMA doesn't tag SDK releases specifically, and still hasn't had a tagged
release since 3.0.1, but the Vulkan SDK now seems to ship a recent master
commit, so we do the same.
Diffstat (limited to 'thirdparty/vulkan/vk_mem_alloc.h')
| -rw-r--r-- | thirdparty/vulkan/vk_mem_alloc.h | 755 |
1 files changed, 442 insertions, 313 deletions
diff --git a/thirdparty/vulkan/vk_mem_alloc.h b/thirdparty/vulkan/vk_mem_alloc.h index ea30060649..42fa3da29e 100644 --- a/thirdparty/vulkan/vk_mem_alloc.h +++ b/thirdparty/vulkan/vk_mem_alloc.h @@ -25,7 +25,7 @@ /** \mainpage Vulkan Memory Allocator -<b>Version 3.0.1 (2022-05-26)</b> +<b>Version 3.1.0-development</b> Copyright (c) 2017-2022 Advanced Micro Devices, Inc. All rights reserved. \n License: MIT @@ -126,17 +126,12 @@ See documentation chapter: \ref statistics. extern "C" { #endif -#ifndef VULKAN_H_ - #ifdef USE_VOLK - #include <volk.h> - #else - #include <vulkan/vulkan.h> - #endif +#ifdef USE_VOLK + #include <volk.h> +#else + #include <vulkan/vulkan.h> #endif -// Define this macro to declare maximum supported Vulkan version in format AAABBBCCC, -// where AAA = major, BBB = minor, CCC = patch. -// If you want to use version > 1.0, it still needs to be enabled via VmaAllocatorCreateInfo::vulkanApiVersion. #if !defined(VMA_VULKAN_VERSION) #if defined(VK_VERSION_1_3) #define VMA_VULKAN_VERSION 1003000 @@ -241,6 +236,12 @@ extern "C" { #define VMA_CALL_POST #endif +// Define this macro to decorate pNext pointers with an attribute specifying the Vulkan +// structure that will be extended via the pNext chain. +#ifndef VMA_EXTENDS_VK_STRUCT + #define VMA_EXTENDS_VK_STRUCT(vkStruct) +#endif + // Define this macro to decorate pointers with an attribute specifying the // length of the array they point to if they are not null. // @@ -635,7 +636,7 @@ typedef enum VmaAllocationCreateFlagBits VMA_ALLOCATION_CREATE_STRATEGY_MIN_TIME_BIT = 0x00020000, /** Allocation strategy that chooses always the lowest offset in available space. This is not the most efficient strategy but achieves highly packed data. - Used internally by defragmentation, not recomended in typical usage. + Used internally by defragmentation, not recommended in typical usage. */ VMA_ALLOCATION_CREATE_STRATEGY_MIN_OFFSET_BIT = 0x00040000, /** Alias to #VMA_ALLOCATION_CREATE_STRATEGY_MIN_MEMORY_BIT. @@ -891,7 +892,7 @@ Use it as a unique identifier to virtual allocation within the single block. Use value `VK_NULL_HANDLE` to represent a null/invalid allocation. */ -VK_DEFINE_NON_DISPATCHABLE_HANDLE(VmaVirtualAllocation); +VK_DEFINE_NON_DISPATCHABLE_HANDLE(VmaVirtualAllocation) /** @} */ @@ -1326,7 +1327,7 @@ typedef struct VmaPoolCreateInfo Please note that some structures, e.g. `VkMemoryPriorityAllocateInfoEXT`, `VkMemoryDedicatedAllocateInfoKHR`, can be attached automatically by this library when using other, more convenient of its features. */ - void* VMA_NULLABLE pMemoryAllocateNext; + void* VMA_NULLABLE VMA_EXTENDS_VK_STRUCT(VkMemoryAllocateInfo) pMemoryAllocateNext; } VmaPoolCreateInfo; /** @} */ @@ -1396,6 +1397,12 @@ typedef struct VmaAllocationInfo const char* VMA_NULLABLE pName; } VmaAllocationInfo; +/** Callback function called during vmaBeginDefragmentation() to check custom criterion about ending current defragmentation pass. + +Should return true if the defragmentation needs to stop current pass. +*/ +typedef VkBool32 (VKAPI_PTR* PFN_vmaCheckDefragmentationBreakFunction)(void* VMA_NULLABLE pUserData); + /** \brief Parameters for defragmentation. To be used with function vmaBeginDefragmentation(). @@ -1419,6 +1426,13 @@ typedef struct VmaDefragmentationInfo `0` means no limit. */ uint32_t maxAllocationsPerPass; + /** \brief Optional custom callback for stopping vmaBeginDefragmentation(). + + Have to return true for breaking current defragmentation pass. + */ + PFN_vmaCheckDefragmentationBreakFunction VMA_NULLABLE pfnBreakCallback; + /// \brief Optional data to pass to custom callback for stopping pass of defragmentation. + void* VMA_NULLABLE pBreakCallbackUserData; } VmaDefragmentationInfo; /// Single move of an allocation to be done for defragmentation. @@ -1912,7 +1926,7 @@ VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages( /** \brief Returns current information about specified allocation. -Current paramteres of given allocation are returned in `pAllocationInfo`. +Current parameters of given allocation are returned in `pAllocationInfo`. Although this function doesn't lock any mutex, so it should be quite efficient, you should avoid calling it too often. @@ -2165,7 +2179,7 @@ VMA_CALL_PRE void VMA_CALL_POST vmaEndDefragmentation( \param allocator Allocator object. \param context Context object that has been created by vmaBeginDefragmentation(). -\param[out] pPassInfo Computed informations for current pass. +\param[out] pPassInfo Computed information for current pass. \returns - `VK_SUCCESS` if no more moves are possible. Then you can omit call to vmaEndDefragmentationPass() and simply end whole defragmentation. - `VK_INCOMPLETE` if there are pending moves returned in `pPassInfo`. You need to perform them, call vmaEndDefragmentationPass(), @@ -2180,7 +2194,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaBeginDefragmentationPass( \param allocator Allocator object. \param context Context object that has been created by vmaBeginDefragmentation(). -\param pPassInfo Computed informations for current pass filled by vmaBeginDefragmentationPass() and possibly modified by you. +\param pPassInfo Computed information for current pass filled by vmaBeginDefragmentationPass() and possibly modified by you. Returns `VK_SUCCESS` if no more moves are possible or `VK_INCOMPLETE` if more defragmentations are possible. @@ -2234,7 +2248,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory2( VmaAllocation VMA_NOT_NULL allocation, VkDeviceSize allocationLocalOffset, VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer, - const void* VMA_NULLABLE pNext); + const void* VMA_NULLABLE VMA_EXTENDS_VK_STRUCT(VkBindBufferMemoryInfoKHR) pNext); /** \brief Binds image to allocation. @@ -2271,7 +2285,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory2( VmaAllocation VMA_NOT_NULL allocation, VkDeviceSize allocationLocalOffset, VkImage VMA_NOT_NULL_NON_DISPATCHABLE image, - const void* VMA_NULLABLE pNext); + const void* VMA_NULLABLE VMA_EXTENDS_VK_STRUCT(VkBindImageMemoryInfoKHR) pNext); /** \brief Creates a new `VkBuffer`, allocates and binds memory for it. @@ -2348,6 +2362,8 @@ returned value is negative error code and `*pBuffer` is null. If the function succeeded, you must destroy the buffer when you no longer need it using `vkDestroyBuffer()`. If you want to also destroy the corresponding allocation you can use convenience function vmaDestroyBuffer(). + +\note There is a new version of this function augmented with parameter `allocationLocalOffset` - see vmaCreateAliasingBuffer2(). */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingBuffer( VmaAllocator VMA_NOT_NULL allocator, @@ -2355,6 +2371,35 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingBuffer( const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo, VkBuffer VMA_NULLABLE_NON_DISPATCHABLE* VMA_NOT_NULL pBuffer); +/** \brief Creates a new `VkBuffer`, binds already created memory for it. + +\param allocator +\param allocation Allocation that provides memory to be used for binding new buffer to it. +\param allocationLocalOffset Additional offset to be added while binding, relative to the beginning of the allocation. Normally it should be 0. +\param pBufferCreateInfo +\param[out] pBuffer Buffer that was created. + +This function automatically: + +-# Creates buffer. +-# Binds the buffer with the supplied memory. + +If any of these operations fail, buffer is not created, +returned value is negative error code and `*pBuffer` is null. + +If the function succeeded, you must destroy the buffer when you +no longer need it using `vkDestroyBuffer()`. If you want to also destroy the corresponding +allocation you can use convenience function vmaDestroyBuffer(). + +\note This is a new version of the function augmented with parameter `allocationLocalOffset`. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingBuffer2( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize allocationLocalOffset, + const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo, + VkBuffer VMA_NULLABLE_NON_DISPATCHABLE* VMA_NOT_NULL pBuffer); + /** \brief Destroys Vulkan buffer and frees allocated memory. This is just a convenience function equivalent to: @@ -2380,13 +2425,21 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateImage( VmaAllocation VMA_NULLABLE* VMA_NOT_NULL pAllocation, VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); -/// Function similar to vmaCreateAliasingBuffer(). +/// Function similar to vmaCreateAliasingBuffer() but for images. VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingImage( VmaAllocator VMA_NOT_NULL allocator, VmaAllocation VMA_NOT_NULL allocation, const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo, VkImage VMA_NULLABLE_NON_DISPATCHABLE* VMA_NOT_NULL pImage); +/// Function similar to vmaCreateAliasingBuffer2() but for images. +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingImage2( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize allocationLocalOffset, + const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo, + VkImage VMA_NULLABLE_NON_DISPATCHABLE* VMA_NOT_NULL pImage); + /** \brief Destroys Vulkan image and frees allocated memory. This is just a convenience function equivalent to: @@ -2619,8 +2672,7 @@ VmaAllocatorCreateInfo::pVulkanFunctions. Other members can be null. #endif #ifndef VMA_USE_STL_SHARED_MUTEX - // Compiler conforms to C++17. - #if __cplusplus >= 201703L + #if __cplusplus >= 201703L || _MSVC_LANG >= 201703L // C++17 #define VMA_USE_STL_SHARED_MUTEX 1 // Visual studio defines __cplusplus properly only when passed additional parameter: /Zc:__cplusplus // Otherwise it is always 199711L, despite shared_mutex works since Visual Studio 2015 Update 2. @@ -2665,6 +2717,38 @@ remove them if not needed. #define VMA_NULL nullptr #endif +#ifndef VMA_FALLTHROUGH + #if __cplusplus >= 201703L || _MSVC_LANG >= 201703L // C++17 + #define VMA_FALLTHROUGH [[fallthrough]] + #else + #define VMA_FALLTHROUGH + #endif +#endif + +// Normal assert to check for programmer's errors, especially in Debug configuration. +#ifndef VMA_ASSERT + #ifdef NDEBUG + #define VMA_ASSERT(expr) + #else + #define VMA_ASSERT(expr) assert(expr) + #endif +#endif + +// Assert that will be called very often, like inside data structures e.g. operator[]. +// Making it non-empty can make program slow. +#ifndef VMA_HEAVY_ASSERT + #ifdef NDEBUG + #define VMA_HEAVY_ASSERT(expr) + #else + #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr) + #endif +#endif + +// If your compiler is not compatible with C++17 and definition of +// aligned_alloc() function is missing, uncommenting following line may help: + +//#include <malloc.h> + #if defined(__ANDROID_API__) && (__ANDROID_API__ < 16) #include <cstdlib> static void* vma_aligned_alloc(size_t alignment, size_t size) @@ -2693,7 +2777,7 @@ static void* vma_aligned_alloc(size_t alignment, size_t size) // // For C++14, usr/include/malloc/_malloc.h declares aligned_alloc()) only // // with the MacOSX11.0 SDK in Xcode 12 (which is what adds // // MAC_OS_X_VERSION_10_16), even though the function is marked - // // availabe for 10.15. That is why the preprocessor checks for 10.16 but + // // available for 10.15. That is why the preprocessor checks for 10.16 but // // the __builtin_available checks for 10.15. // // People who use C++17 could call aligned_alloc with the 10.15 SDK already. // if (__builtin_available(macOS 10.15, iOS 13, *)) @@ -2717,11 +2801,17 @@ static void* vma_aligned_alloc(size_t alignment, size_t size) { return _aligned_malloc(size, alignment); } -#else +#elif __cplusplus >= 201703L || _MSVC_LANG >= 201703L // C++17 static void* vma_aligned_alloc(size_t alignment, size_t size) { return aligned_alloc(alignment, size); } +#else +static void* vma_aligned_alloc(size_t alignment, size_t size) +{ + VMA_ASSERT(0 && "Could not implement aligned_alloc automatically. Please enable C++17 or later in your compiler or provide custom implementation of macro VMA_SYSTEM_ALIGNED_MALLOC (and VMA_SYSTEM_ALIGNED_FREE if needed) using the API of your system."); + return VMA_NULL; +} #endif #if defined(_WIN32) @@ -2736,32 +2826,8 @@ static void vma_aligned_free(void* VMA_NULLABLE ptr) } #endif -// If your compiler is not compatible with C++11 and definition of -// aligned_alloc() function is missing, uncommeting following line may help: - -//#include <malloc.h> - -// Normal assert to check for programmer's errors, especially in Debug configuration. -#ifndef VMA_ASSERT - #ifdef NDEBUG - #define VMA_ASSERT(expr) - #else - #define VMA_ASSERT(expr) assert(expr) - #endif -#endif - -// Assert that will be called very often, like inside data structures e.g. operator[]. -// Making it non-empty can make program slow. -#ifndef VMA_HEAVY_ASSERT - #ifdef NDEBUG - #define VMA_HEAVY_ASSERT(expr) - #else - #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr) - #endif -#endif - #ifndef VMA_ALIGN_OF - #define VMA_ALIGN_OF(type) (__alignof(type)) + #define VMA_ALIGN_OF(type) (alignof(type)) #endif #ifndef VMA_SYSTEM_ALIGNED_MALLOC @@ -2808,16 +2874,35 @@ static void vma_aligned_free(void* VMA_NULLABLE ptr) #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp) #endif -#ifndef VMA_DEBUG_LOG - #define VMA_DEBUG_LOG(format, ...) +#ifndef VMA_DEBUG_LOG_FORMAT + #define VMA_DEBUG_LOG_FORMAT(format, ...) /* - #define VMA_DEBUG_LOG(format, ...) do { \ - printf(format, __VA_ARGS__); \ + #define VMA_DEBUG_LOG_FORMAT(format, ...) do { \ + printf((format), __VA_ARGS__); \ printf("\n"); \ } while(false) */ #endif +#ifndef VMA_DEBUG_LOG + #define VMA_DEBUG_LOG(str) VMA_DEBUG_LOG_FORMAT("%s", (str)) +#endif + +#ifndef VMA_CLASS_NO_COPY + #define VMA_CLASS_NO_COPY(className) \ + private: \ + className(const className&) = delete; \ + className& operator=(const className&) = delete; +#endif +#ifndef VMA_CLASS_NO_COPY_NO_MOVE + #define VMA_CLASS_NO_COPY_NO_MOVE(className) \ + private: \ + className(const className&) = delete; \ + className(className&&) = delete; \ + className& operator=(const className&) = delete; \ + className& operator=(className&&) = delete; +#endif + // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString. #if VMA_STATS_STRING_ENABLED static inline void VmaUint32ToStr(char* VMA_NOT_NULL outStr, size_t strLen, uint32_t num) @@ -2837,7 +2922,9 @@ static void vma_aligned_free(void* VMA_NULLABLE ptr) #ifndef VMA_MUTEX class VmaMutex { + VMA_CLASS_NO_COPY_NO_MOVE(VmaMutex) public: + VmaMutex() { } void Lock() { m_Mutex.lock(); } void Unlock() { m_Mutex.unlock(); } bool TryLock() { return m_Mutex.try_lock(); } @@ -2997,19 +3084,12 @@ Mapping hysteresis is a logic that launches when vmaMapMemory/vmaUnmapMemory is or a persistently mapped allocation is created and destroyed several times in a row. It keeps additional +1 mapping of a device memory block to prevent calling actual vkMapMemory/vkUnmapMemory too many times, which may improve performance and help -tools like RenderDOc. +tools like RenderDoc. */ #ifndef VMA_MAPPING_HYSTERESIS_ENABLED #define VMA_MAPPING_HYSTERESIS_ENABLED 1 #endif -#ifndef VMA_CLASS_NO_COPY - #define VMA_CLASS_NO_COPY(className) \ - private: \ - className(const className&) = delete; \ - className& operator=(const className&) = delete; -#endif - #define VMA_VALIDATE(cond) do { if(!(cond)) { \ VMA_ASSERT(0 && "Validation failed: " #cond); \ return false; \ @@ -3092,7 +3172,7 @@ enum class VmaAllocationRequestType #ifndef _VMA_FORWARD_DECLARATIONS // Opaque handle used by allocation algorithms to identify single allocation in any conforming way. -VK_DEFINE_NON_DISPATCHABLE_HANDLE(VmaAllocHandle); +VK_DEFINE_NON_DISPATCHABLE_HANDLE(VmaAllocHandle) struct VmaMutexLock; struct VmaMutexLockRead; @@ -3307,7 +3387,7 @@ static inline T VmaAlignUp(T val, T alignment) return (val + alignment - 1) & ~(alignment - 1); } -// Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8. +// Aligns given value down to nearest multiply of align value. For example: VmaAlignDown(11, 8) = 8. // Use types like uint32_t, uint64_t as T. template <typename T> static inline T VmaAlignDown(T val, T alignment) @@ -3502,7 +3582,7 @@ new element with value (key) should be inserted. template <typename CmpLess, typename IterT, typename KeyT> static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT& key, const CmpLess& cmp) { - size_t down = 0, up = (end - beg); + size_t down = 0, up = size_t(end - beg); while (down < up) { const size_t mid = down + (up - down) / 2; // Overflow-safe midpoint calculation @@ -3628,18 +3708,21 @@ static bool FindMemoryPreferences( // CPU random access - e.g. a buffer written to or transferred from GPU to read back on CPU. if(hostAccessRandom) { - if(!isIntegratedGPU && deviceAccess && hostAccessAllowTransferInstead && !preferHost) + // Prefer cached. Cannot require it, because some platforms don't have it (e.g. Raspberry Pi - see #362)! + outPreferredFlags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + + if (!isIntegratedGPU && deviceAccess && hostAccessAllowTransferInstead && !preferHost) { // Nice if it will end up in HOST_VISIBLE, but more importantly prefer DEVICE_LOCAL. // Omitting HOST_VISIBLE here is intentional. // In case there is DEVICE_LOCAL | HOST_VISIBLE | HOST_CACHED, it will pick that one. // Otherwise, this will give same weight to DEVICE_LOCAL as HOST_VISIBLE | HOST_CACHED and select the former if occurs first on the list. - outPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + outPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; } else { - // Always CPU memory, cached. - outRequiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + // Always CPU memory. + outRequiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; } } // CPU sequential write - may be CPU or host-visible GPU memory, uncached and write-combined. @@ -3678,19 +3761,18 @@ static bool FindMemoryPreferences( // No CPU access else { - // GPU access, no CPU access (e.g. a color attachment image) - prefer GPU memory - if(deviceAccess) - { - // ...unless there is a clear preference from the user not to do so. - if(preferHost) - outNotPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - else - outPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - } + // if(deviceAccess) + // + // GPU access, no CPU access (e.g. a color attachment image) - prefer GPU memory, + // unless there is a clear preference from the user not to do so. + // + // else: + // // No direct GPU access, no CPU access, just transfers. // It may be staging copy intended for e.g. preserving image for next frame (then better GPU memory) or // a "swap file" copy to free some GPU memory (then better CPU memory). // Up to the user to decide. If no preferece, assume the former and choose GPU memory. + if(preferHost) outNotPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; else @@ -3909,7 +3991,7 @@ static void VmaAddDetailedStatistics(VmaDetailedStatistics& inoutStats, const Vm // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope). struct VmaMutexLock { - VMA_CLASS_NO_COPY(VmaMutexLock) + VMA_CLASS_NO_COPY_NO_MOVE(VmaMutexLock) public: VmaMutexLock(VMA_MUTEX& mutex, bool useMutex = true) : m_pMutex(useMutex ? &mutex : VMA_NULL) @@ -3925,7 +4007,7 @@ private: // Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for reading. struct VmaMutexLockRead { - VMA_CLASS_NO_COPY(VmaMutexLockRead) + VMA_CLASS_NO_COPY_NO_MOVE(VmaMutexLockRead) public: VmaMutexLockRead(VMA_RW_MUTEX& mutex, bool useMutex) : m_pMutex(useMutex ? &mutex : VMA_NULL) @@ -3941,7 +4023,7 @@ private: // Helper RAII class to lock a RW mutex in constructor and unlock it in destructor (at the end of scope), for writing. struct VmaMutexLockWrite { - VMA_CLASS_NO_COPY(VmaMutexLockWrite) + VMA_CLASS_NO_COPY_NO_MOVE(VmaMutexLockWrite) public: VmaMutexLockWrite(VMA_RW_MUTEX& mutex, bool useMutex) : m_pMutex(useMutex ? &mutex : VMA_NULL) @@ -3964,11 +4046,11 @@ private: #ifndef _VMA_ATOMIC_TRANSACTIONAL_INCREMENT // An object that increments given atomic but decrements it back in the destructor unless Commit() is called. -template<typename T> +template<typename AtomicT> struct AtomicTransactionalIncrement { public: - typedef std::atomic<T> AtomicT; + using T = decltype(AtomicT().load()); ~AtomicTransactionalIncrement() { @@ -4396,7 +4478,7 @@ allocator can create multiple blocks. template<typename T> class VmaPoolAllocator { - VMA_CLASS_NO_COPY(VmaPoolAllocator) + VMA_CLASS_NO_COPY_NO_MOVE(VmaPoolAllocator) public: VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, uint32_t firstBlockCapacity); ~VmaPoolAllocator(); @@ -4529,7 +4611,7 @@ struct VmaListItem template<typename T> class VmaRawList { - VMA_CLASS_NO_COPY(VmaRawList) + VMA_CLASS_NO_COPY_NO_MOVE(VmaRawList) public: typedef VmaListItem<T> ItemType; @@ -4792,7 +4874,7 @@ VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value) template<typename T, typename AllocatorT> class VmaList { - VMA_CLASS_NO_COPY(VmaList) + VMA_CLASS_NO_COPY_NO_MOVE(VmaList) public: class reverse_iterator; class const_iterator; @@ -5384,7 +5466,7 @@ void VmaStringBuilder::AddNumber(uint32_t num) char* p = &buf[10]; do { - *--p = '0' + (num % 10); + *--p = '0' + (char)(num % 10); num /= 10; } while (num); Add(p); @@ -5397,7 +5479,7 @@ void VmaStringBuilder::AddNumber(uint64_t num) char* p = &buf[20]; do { - *--p = '0' + (num % 10); + *--p = '0' + (char)(num % 10); num /= 10; } while (num); Add(p); @@ -5419,7 +5501,7 @@ VmaStringBuilder passed to the constructor. */ class VmaJsonWriter { - VMA_CLASS_NO_COPY(VmaJsonWriter) + VMA_CLASS_NO_COPY_NO_MOVE(VmaJsonWriter) public: // sb - string builder to write the document to. Must remain alive for the whole lifetime of this object. VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb); @@ -5453,7 +5535,6 @@ public: // Posts next part of an open string. The number is converted to decimal characters. void ContinueString(uint32_t n); void ContinueString(uint64_t n); - void ContinueString_Size(size_t n); // Posts next part of an open string. Pointer value is converted to characters // using "%p" formatting - shown as hexadecimal number, e.g.: 000000081276Ad00 void ContinueString_Pointer(const void* ptr); @@ -5463,7 +5544,6 @@ public: // Writes a number value. void WriteNumber(uint32_t n); void WriteNumber(uint64_t n); - void WriteSize(size_t n); // Writes a boolean value - false or true. void WriteBool(bool b); // Writes a null value. @@ -5488,11 +5568,6 @@ private: VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack; bool m_InsideString; - // Write size_t for less than 64bits - void WriteSize(size_t n, std::integral_constant<bool, false>) { m_SB.AddNumber(static_cast<uint32_t>(n)); } - // Write size_t for 64bits - void WriteSize(size_t n, std::integral_constant<bool, true>) { m_SB.AddNumber(static_cast<uint64_t>(n)); } - void BeginValue(bool isString); void WriteIndent(bool oneLess = false); }; @@ -5618,7 +5693,6 @@ void VmaJsonWriter::ContinueString(const char* pStr) break; default: VMA_ASSERT(0 && "Character not currently supported."); - break; } } } @@ -5635,14 +5709,6 @@ void VmaJsonWriter::ContinueString(uint64_t n) m_SB.AddNumber(n); } -void VmaJsonWriter::ContinueString_Size(size_t n) -{ - VMA_ASSERT(m_InsideString); - // Fix for AppleClang incorrect type casting - // TODO: Change to if constexpr when C++17 used as minimal standard - WriteSize(n, std::is_same<size_t, uint64_t>{}); -} - void VmaJsonWriter::ContinueString_Pointer(const void* ptr) { VMA_ASSERT(m_InsideString); @@ -5674,15 +5740,6 @@ void VmaJsonWriter::WriteNumber(uint64_t n) m_SB.AddNumber(n); } -void VmaJsonWriter::WriteSize(size_t n) -{ - VMA_ASSERT(!m_InsideString); - BeginValue(false); - // Fix for AppleClang incorrect type casting - // TODO: Change to if constexpr when C++17 used as minimal standard - WriteSize(n, std::is_same<size_t, uint64_t>{}); -} - void VmaJsonWriter::WriteBool(bool b) { VMA_ASSERT(!m_InsideString); @@ -5782,7 +5839,7 @@ static void VmaPrintDetailedStatistics(VmaJsonWriter& json, const VmaDetailedSta class VmaMappingHysteresis { - VMA_CLASS_NO_COPY(VmaMappingHysteresis) + VMA_CLASS_NO_COPY_NO_MOVE(VmaMappingHysteresis) public: VmaMappingHysteresis() = default; @@ -5889,7 +5946,7 @@ Thread-safety: */ class VmaDeviceMemoryBlock { - VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock) + VMA_CLASS_NO_COPY_NO_MOVE(VmaDeviceMemoryBlock) public: VmaBlockMetadata* m_pMetadata; @@ -5918,7 +5975,7 @@ public: // Call when allocation/free was made from m_pMetadata. // Used for m_MappingHysteresis. - void PostAlloc() { m_MappingHysteresis.PostAlloc(); } + void PostAlloc(VmaAllocator hAllocator); void PostFree(VmaAllocator hAllocator); // Validates all data structures inside this object. If not valid, returns false. @@ -6112,6 +6169,7 @@ Thread-safe, synchronized internally. */ class VmaDedicatedAllocationList { + VMA_CLASS_NO_COPY_NO_MOVE(VmaDedicatedAllocationList) public: VmaDedicatedAllocationList() {} ~VmaDedicatedAllocationList(); @@ -6297,6 +6355,7 @@ in a single VkDeviceMemory block. */ class VmaBlockMetadata { + VMA_CLASS_NO_COPY_NO_MOVE(VmaBlockMetadata) public: // pAllocationCallbacks, if not null, must be owned externally - alive and unchanged for the whole lifetime of this object. VmaBlockMetadata(const VkAllocationCallbacks* pAllocationCallbacks, @@ -6363,7 +6422,7 @@ public: protected: const VkAllocationCallbacks* GetAllocationCallbacks() const { return m_pAllocationCallbacks; } VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; } - VkDeviceSize GetDebugMargin() const { return IsVirtual() ? 0 : VMA_DEBUG_MARGIN; } + VkDeviceSize GetDebugMargin() const { return VkDeviceSize(IsVirtual() ? 0 : VMA_DEBUG_MARGIN); } void DebugLogAllocation(VkDeviceSize offset, VkDeviceSize size, void* userData) const; #if VMA_STATS_STRING_ENABLED @@ -6399,7 +6458,7 @@ void VmaBlockMetadata::DebugLogAllocation(VkDeviceSize offset, VkDeviceSize size { if (IsVirtual()) { - VMA_DEBUG_LOG("UNFREED VIRTUAL ALLOCATION; Offset: %llu; Size: %llu; UserData: %p", offset, size, userData); + VMA_DEBUG_LOG_FORMAT("UNFREED VIRTUAL ALLOCATION; Offset: %llu; Size: %llu; UserData: %p", offset, size, userData); } else { @@ -6410,12 +6469,12 @@ void VmaBlockMetadata::DebugLogAllocation(VkDeviceSize offset, VkDeviceSize size const char* name = allocation->GetName(); #if VMA_STATS_STRING_ENABLED - VMA_DEBUG_LOG("UNFREED ALLOCATION; Offset: %llu; Size: %llu; UserData: %p; Name: %s; Type: %s; Usage: %u", + VMA_DEBUG_LOG_FORMAT("UNFREED ALLOCATION; Offset: %llu; Size: %llu; UserData: %p; Name: %s; Type: %s; Usage: %u", offset, size, userData, name ? name : "vma_empty", VMA_SUBALLOCATION_TYPE_NAMES[allocation->GetSuballocationType()], allocation->GetBufferImageUsage()); #else - VMA_DEBUG_LOG("UNFREED ALLOCATION; Offset: %llu; Size: %llu; UserData: %p; Name: %s; Type: %u", + VMA_DEBUG_LOG_FORMAT("UNFREED ALLOCATION; Offset: %llu; Size: %llu; UserData: %p; Name: %s; Type: %u", offset, size, userData, name ? name : "vma_empty", (uint32_t)allocation->GetSuballocationType()); #endif // VMA_STATS_STRING_ENABLED @@ -6431,13 +6490,13 @@ void VmaBlockMetadata::PrintDetailedMap_Begin(class VmaJsonWriter& json, json.WriteNumber(GetSize()); json.WriteString("UnusedBytes"); - json.WriteSize(unusedBytes); + json.WriteNumber(unusedBytes); json.WriteString("Allocations"); - json.WriteSize(allocationCount); + json.WriteNumber((uint64_t)allocationCount); json.WriteString("UnusedRanges"); - json.WriteSize(unusedRangeCount); + json.WriteNumber((uint64_t)unusedRangeCount); json.WriteString("Suballocations"); json.BeginArray(); @@ -6722,7 +6781,7 @@ uint32_t VmaBlockBufferImageGranularity::OffsetToPageIndex(VkDeviceSize offset) void VmaBlockBufferImageGranularity::AllocPage(RegionInfo& page, uint8_t allocType) { - // When current alloc type is free then it can be overriden by new type + // When current alloc type is free then it can be overridden by new type if (page.allocCount == 0 || (page.allocCount > 0 && page.allocType == VMA_SUBALLOCATION_TYPE_FREE)) page.allocType = allocType; @@ -6737,7 +6796,7 @@ class VmaBlockMetadata_Generic : public VmaBlockMetadata { friend class VmaDefragmentationAlgorithm_Generic; friend class VmaDefragmentationAlgorithm_Fast; - VMA_CLASS_NO_COPY(VmaBlockMetadata_Generic) + VMA_CLASS_NO_COPY_NO_MOVE(VmaBlockMetadata_Generic) public: VmaBlockMetadata_Generic(const VkAllocationCallbacks* pAllocationCallbacks, VkDeviceSize bufferImageGranularity, bool isVirtual); @@ -6747,7 +6806,7 @@ public: VkDeviceSize GetSumFreeSize() const override { return m_SumFreeSize; } bool IsEmpty() const override { return (m_Suballocations.size() == 1) && (m_FreeCount == 1); } void Free(VmaAllocHandle allocHandle) override { FreeSuballocation(FindAtOffset((VkDeviceSize)allocHandle - 1)); } - VkDeviceSize GetAllocationOffset(VmaAllocHandle allocHandle) const override { return (VkDeviceSize)allocHandle - 1; }; + VkDeviceSize GetAllocationOffset(VmaAllocHandle allocHandle) const override { return (VkDeviceSize)allocHandle - 1; } void Init(VkDeviceSize size) override; bool Validate() const override; @@ -7576,7 +7635,7 @@ GetSize() +-------+ */ class VmaBlockMetadata_Linear : public VmaBlockMetadata { - VMA_CLASS_NO_COPY(VmaBlockMetadata_Linear) + VMA_CLASS_NO_COPY_NO_MOVE(VmaBlockMetadata_Linear) public: VmaBlockMetadata_Linear(const VkAllocationCallbacks* pAllocationCallbacks, VkDeviceSize bufferImageGranularity, bool isVirtual); @@ -7584,7 +7643,7 @@ public: VkDeviceSize GetSumFreeSize() const override { return m_SumFreeSize; } bool IsEmpty() const override { return GetAllocationCount() == 0; } - VkDeviceSize GetAllocationOffset(VmaAllocHandle allocHandle) const override { return (VkDeviceSize)allocHandle - 1; }; + VkDeviceSize GetAllocationOffset(VmaAllocHandle allocHandle) const override { return (VkDeviceSize)allocHandle - 1; } void Init(VkDeviceSize size) override; bool Validate() const override; @@ -8057,30 +8116,17 @@ void VmaBlockMetadata_Linear::AddStatistics(VmaStatistics& inoutStats) const { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - // 1. Process free space before this allocation. - if (lastOffset < suballoc.offset) - { - // There is free space from lastOffset to suballoc.offset. - const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; - } - - // 2. Process this allocation. + // Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++inoutStats.allocationCount; - // 3. Prepare for next iteration. + // Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc2ndIndex; } // We are at the end. else { - if (lastOffset < freeSpace2ndTo1stEnd) - { - // There is free space from lastOffset to freeSpace2ndTo1stEnd. - const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset; - } - // End of loop. lastOffset = freeSpace2ndTo1stEnd; } @@ -8104,30 +8150,17 @@ void VmaBlockMetadata_Linear::AddStatistics(VmaStatistics& inoutStats) const { const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; - // 1. Process free space before this allocation. - if (lastOffset < suballoc.offset) - { - // There is free space from lastOffset to suballoc.offset. - const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; - } - - // 2. Process this allocation. + // Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++inoutStats.allocationCount; - // 3. Prepare for next iteration. + // Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc1stIndex; } // We are at the end. else { - if (lastOffset < freeSpace1stTo2ndEnd) - { - // There is free space from lastOffset to freeSpace1stTo2ndEnd. - const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset; - } - // End of loop. lastOffset = freeSpace1stTo2ndEnd; } @@ -8150,30 +8183,17 @@ void VmaBlockMetadata_Linear::AddStatistics(VmaStatistics& inoutStats) const { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - // 1. Process free space before this allocation. - if (lastOffset < suballoc.offset) - { - // There is free space from lastOffset to suballoc.offset. - const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; - } - - // 2. Process this allocation. + // Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++inoutStats.allocationCount; - // 3. Prepare for next iteration. + // Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; --nextAlloc2ndIndex; } // We are at the end. else { - if (lastOffset < size) - { - // There is free space from lastOffset to size. - const VkDeviceSize unusedRangeSize = size - lastOffset; - } - // End of loop. lastOffset = size; } @@ -9234,7 +9254,7 @@ m_LevelCount is the maximum number of levels to use in the current object. */ class VmaBlockMetadata_Buddy : public VmaBlockMetadata { - VMA_CLASS_NO_COPY(VmaBlockMetadata_Buddy) + VMA_CLASS_NO_COPY_NO_MOVE(VmaBlockMetadata_Buddy) public: VmaBlockMetadata_Buddy(const VkAllocationCallbacks* pAllocationCallbacks, VkDeviceSize bufferImageGranularity, bool isVirtual); @@ -9244,7 +9264,7 @@ public: VkDeviceSize GetSumFreeSize() const override { return m_SumFreeSize + GetUnusableSize(); } bool IsEmpty() const override { return m_Root->type == Node::TYPE_FREE; } VkResult CheckCorruption(const void* pBlockData) override { return VK_ERROR_FEATURE_NOT_PRESENT; } - VkDeviceSize GetAllocationOffset(VmaAllocHandle allocHandle) const override { return (VkDeviceSize)allocHandle - 1; }; + VkDeviceSize GetAllocationOffset(VmaAllocHandle allocHandle) const override { return (VkDeviceSize)allocHandle - 1; } void DebugLogAllAllocations() const override { DebugLogAllAllocationNode(m_Root, 0); } void Init(VkDeviceSize size) override; @@ -9933,7 +9953,7 @@ void VmaBlockMetadata_Buddy::PrintDetailedMapNode(class VmaJsonWriter& json, con // VMA_ALLOCATION_CREATE_STRATEGY_MIN_TIME_BIT for fastest alloc time possible. class VmaBlockMetadata_TLSF : public VmaBlockMetadata { - VMA_CLASS_NO_COPY(VmaBlockMetadata_TLSF) + VMA_CLASS_NO_COPY_NO_MOVE(VmaBlockMetadata_TLSF) public: VmaBlockMetadata_TLSF(const VkAllocationCallbacks* pAllocationCallbacks, VkDeviceSize bufferImageGranularity, bool isVirtual); @@ -9943,7 +9963,7 @@ public: size_t GetFreeRegionsCount() const override { return m_BlocksFreeCount + 1; } VkDeviceSize GetSumFreeSize() const override { return m_BlocksFreeSize + m_NullBlock->size; } bool IsEmpty() const override { return m_NullBlock->offset == 0; } - VkDeviceSize GetAllocationOffset(VmaAllocHandle allocHandle) const override { return ((Block*)allocHandle)->offset; }; + VkDeviceSize GetAllocationOffset(VmaAllocHandle allocHandle) const override { return ((Block*)allocHandle)->offset; } void Init(VkDeviceSize size) override; bool Validate() const override; @@ -10095,7 +10115,7 @@ void VmaBlockMetadata_TLSF::Init(VkDeviceSize size) else m_ListsCount += 4; - m_MemoryClasses = memoryClass + 2; + m_MemoryClasses = memoryClass + uint8_t(2); memset(m_InnerIsFreeBitmap, 0, MAX_MEMORY_CLASSES * sizeof(uint32_t)); m_FreeList = vma_new_array(GetAllocationCallbacks(), Block*, m_ListsCount); @@ -10288,7 +10308,7 @@ bool VmaBlockMetadata_TLSF::CreateAllocationRequest( // Round up to the next block VkDeviceSize sizeForNextList = allocSize; - VkDeviceSize smallSizeStep = SMALL_BUFFER_SIZE / (IsVirtual() ? 1 << SECOND_LEVEL_INDEX : 4); + VkDeviceSize smallSizeStep = VkDeviceSize(SMALL_BUFFER_SIZE / (IsVirtual() ? 1 << SECOND_LEVEL_INDEX : 4)); if (allocSize > SMALL_BUFFER_SIZE) { sizeForNextList += (1ULL << (VMA_BITSCAN_MSB(allocSize) - SECOND_LEVEL_INDEX)); @@ -10298,8 +10318,8 @@ bool VmaBlockMetadata_TLSF::CreateAllocationRequest( else sizeForNextList += smallSizeStep; - uint32_t nextListIndex = 0; - uint32_t prevListIndex = 0; + uint32_t nextListIndex = m_ListsCount; + uint32_t prevListIndex = m_ListsCount; Block* nextListBlock = VMA_NULL; Block* prevListBlock = VMA_NULL; @@ -10697,7 +10717,7 @@ void VmaBlockMetadata_TLSF::DebugLogAllAllocations() const uint8_t VmaBlockMetadata_TLSF::SizeToMemoryClass(VkDeviceSize size) const { if (size > SMALL_BUFFER_SIZE) - return VMA_BITSCAN_MSB(size) - MEMORY_CLASS_SHIFT; + return uint8_t(VMA_BITSCAN_MSB(size) - MEMORY_CLASS_SHIFT); return 0; } @@ -10785,7 +10805,7 @@ void VmaBlockMetadata_TLSF::InsertFreeBlock(Block* block) void VmaBlockMetadata_TLSF::MergeBlock(Block* block, Block* prev) { - VMA_ASSERT(block->prevPhysical == prev && "Cannot merge seperate physical regions!"); + VMA_ASSERT(block->prevPhysical == prev && "Cannot merge separate physical regions!"); VMA_ASSERT(!prev->IsFree() && "Cannot merge block that belongs to free list!"); block->offset = prev->offset; @@ -10802,10 +10822,10 @@ VmaBlockMetadata_TLSF::Block* VmaBlockMetadata_TLSF::FindFreeBlock(VkDeviceSize uint32_t innerFreeMap = m_InnerIsFreeBitmap[memoryClass] & (~0U << SizeToSecondIndex(size, memoryClass)); if (!innerFreeMap) { - // Check higher levels for avaiable blocks + // Check higher levels for available blocks uint32_t freeMap = m_IsFreeBitmap & (~0UL << (memoryClass + 1)); if (!freeMap) - return VMA_NULL; // No more memory avaible + return VMA_NULL; // No more memory available // Find lowest free region memoryClass = VMA_BITSCAN_LSB(freeMap); @@ -10872,7 +10892,7 @@ Synchronized internally with a mutex. class VmaBlockVector { friend struct VmaDefragmentationContext_T; - VMA_CLASS_NO_COPY(VmaBlockVector) + VMA_CLASS_NO_COPY_NO_MOVE(VmaBlockVector) public: VmaBlockVector( VmaAllocator hAllocator, @@ -10991,7 +11011,7 @@ private: #ifndef _VMA_DEFRAGMENTATION_CONTEXT struct VmaDefragmentationContext_T { - VMA_CLASS_NO_COPY(VmaDefragmentationContext_T) + VMA_CLASS_NO_COPY_NO_MOVE(VmaDefragmentationContext_T) public: VmaDefragmentationContext_T( VmaAllocator hAllocator, @@ -11041,6 +11061,8 @@ private: const VkDeviceSize m_MaxPassBytes; const uint32_t m_MaxPassAllocations; + const PFN_vmaCheckDefragmentationBreakFunction m_BreakCallback; + void* m_BreakCallbackUserData; VmaStlAllocator<VmaDefragmentationMove> m_MoveAllocator; VmaVector<VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove>> m_Moves; @@ -11078,7 +11100,7 @@ private: struct VmaPool_T { friend struct VmaPoolListItemTraits; - VMA_CLASS_NO_COPY(VmaPool_T) + VMA_CLASS_NO_COPY_NO_MOVE(VmaPool_T) public: VmaBlockVector m_BlockVector; VmaDedicatedAllocationList m_DedicatedAllocations; @@ -11120,6 +11142,9 @@ struct VmaPoolListItemTraits #ifndef _VMA_CURRENT_BUDGET_DATA struct VmaCurrentBudgetData { + VMA_CLASS_NO_COPY_NO_MOVE(VmaCurrentBudgetData) +public: + VMA_ATOMIC_UINT32 m_BlockCount[VK_MAX_MEMORY_HEAPS]; VMA_ATOMIC_UINT32 m_AllocationCount[VK_MAX_MEMORY_HEAPS]; VMA_ATOMIC_UINT64 m_BlockBytes[VK_MAX_MEMORY_HEAPS]; @@ -11188,7 +11213,7 @@ Thread-safe wrapper over VmaPoolAllocator free list, for allocation of VmaAlloca */ class VmaAllocationObjectAllocator { - VMA_CLASS_NO_COPY(VmaAllocationObjectAllocator) + VMA_CLASS_NO_COPY_NO_MOVE(VmaAllocationObjectAllocator) public: VmaAllocationObjectAllocator(const VkAllocationCallbacks* pAllocationCallbacks) : m_Allocator(pAllocationCallbacks, 1024) {} @@ -11218,7 +11243,7 @@ void VmaAllocationObjectAllocator::Free(VmaAllocation hAlloc) #ifndef _VMA_VIRTUAL_BLOCK_T struct VmaVirtualBlock_T { - VMA_CLASS_NO_COPY(VmaVirtualBlock_T) + VMA_CLASS_NO_COPY_NO_MOVE(VmaVirtualBlock_T) public: const bool m_AllocationCallbacksSpecified; const VkAllocationCallbacks m_AllocationCallbacks; @@ -11254,14 +11279,15 @@ VmaVirtualBlock_T::VmaVirtualBlock_T(const VmaVirtualBlockCreateInfo& createInfo const uint32_t algorithm = createInfo.flags & VMA_VIRTUAL_BLOCK_CREATE_ALGORITHM_MASK; switch (algorithm) { - default: - VMA_ASSERT(0); case 0: m_Metadata = vma_new(GetAllocationCallbacks(), VmaBlockMetadata_TLSF)(VK_NULL_HANDLE, 1, true); break; case VMA_VIRTUAL_BLOCK_CREATE_LINEAR_ALGORITHM_BIT: m_Metadata = vma_new(GetAllocationCallbacks(), VmaBlockMetadata_Linear)(VK_NULL_HANDLE, 1, true); break; + default: + VMA_ASSERT(0); + m_Metadata = vma_new(GetAllocationCallbacks(), VmaBlockMetadata_TLSF)(VK_NULL_HANDLE, 1, true); } m_Metadata->Init(createInfo.size); @@ -11269,7 +11295,7 @@ VmaVirtualBlock_T::VmaVirtualBlock_T(const VmaVirtualBlockCreateInfo& createInfo VmaVirtualBlock_T::~VmaVirtualBlock_T() { - // Define macro VMA_DEBUG_LOG to receive the list of the unfreed allocations + // Define macro VMA_DEBUG_LOG_FORMAT to receive the list of the unfreed allocations if (!m_Metadata->IsEmpty()) m_Metadata->DebugLogAllAllocations(); // This is the most important assert in the entire library. @@ -11357,7 +11383,7 @@ void VmaVirtualBlock_T::BuildStatsString(bool detailedMap, VmaStringBuilder& sb) // Main allocator object. struct VmaAllocator_T { - VMA_CLASS_NO_COPY(VmaAllocator_T) + VMA_CLASS_NO_COPY_NO_MOVE(VmaAllocator_T) public: bool m_UseMutex; uint32_t m_VulkanApiVersion; @@ -11745,14 +11771,16 @@ void VmaDeviceMemoryBlock::Init( switch (algorithm) { + case 0: + m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_TLSF)(hAllocator->GetAllocationCallbacks(), + bufferImageGranularity, false); // isVirtual + break; case VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT: m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Linear)(hAllocator->GetAllocationCallbacks(), bufferImageGranularity, false); // isVirtual break; default: VMA_ASSERT(0); - // Fall-through. - case 0: m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_TLSF)(hAllocator->GetAllocationCallbacks(), bufferImageGranularity, false); // isVirtual } @@ -11761,7 +11789,7 @@ void VmaDeviceMemoryBlock::Init( void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator) { - // Define macro VMA_DEBUG_LOG to receive the list of the unfreed allocations + // Define macro VMA_DEBUG_LOG_FORMAT to receive the list of the unfreed allocations if (!m_pMetadata->IsEmpty()) m_pMetadata->DebugLogAllAllocations(); // This is the most important assert in the entire library. @@ -11776,8 +11804,15 @@ void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator) m_pMetadata = VMA_NULL; } +void VmaDeviceMemoryBlock::PostAlloc(VmaAllocator hAllocator) +{ + VmaMutexLock lock(m_MapAndBindMutex, hAllocator->m_UseMutex); + m_MappingHysteresis.PostAlloc(); +} + void VmaDeviceMemoryBlock::PostFree(VmaAllocator hAllocator) { + VmaMutexLock lock(m_MapAndBindMutex, hAllocator->m_UseMutex); if(m_MappingHysteresis.PostFree()) { VMA_ASSERT(m_MappingHysteresis.GetExtraMapping() == 0); @@ -12462,7 +12497,7 @@ VkResult VmaBlockVector::AllocatePage( pCurrBlock, size, alignment, createInfo.flags, createInfo.pUserData, suballocType, strategy, pAllocation); if (res == VK_SUCCESS) { - VMA_DEBUG_LOG(" Returned from last block #%u", pCurrBlock->GetId()); + VMA_DEBUG_LOG_FORMAT(" Returned from last block #%u", pCurrBlock->GetId()); IncrementallySortBlocks(); return VK_SUCCESS; } @@ -12498,7 +12533,7 @@ VkResult VmaBlockVector::AllocatePage( pCurrBlock, size, alignment, createInfo.flags, createInfo.pUserData, suballocType, strategy, pAllocation); if (res == VK_SUCCESS) { - VMA_DEBUG_LOG(" Returned from existing block #%u", pCurrBlock->GetId()); + VMA_DEBUG_LOG_FORMAT(" Returned from existing block #%u", pCurrBlock->GetId()); IncrementallySortBlocks(); return VK_SUCCESS; } @@ -12517,7 +12552,7 @@ VkResult VmaBlockVector::AllocatePage( pCurrBlock, size, alignment, createInfo.flags, createInfo.pUserData, suballocType, strategy, pAllocation); if (res == VK_SUCCESS) { - VMA_DEBUG_LOG(" Returned from existing block #%u", pCurrBlock->GetId()); + VMA_DEBUG_LOG_FORMAT(" Returned from existing block #%u", pCurrBlock->GetId()); IncrementallySortBlocks(); return VK_SUCCESS; } @@ -12534,7 +12569,7 @@ VkResult VmaBlockVector::AllocatePage( VkResult res = AllocateFromBlock(pCurrBlock, size, alignment, createInfo.flags, createInfo.pUserData, suballocType, strategy, pAllocation); if (res == VK_SUCCESS) { - VMA_DEBUG_LOG(" Returned from existing block #%u", pCurrBlock->GetId()); + VMA_DEBUG_LOG_FORMAT(" Returned from existing block #%u", pCurrBlock->GetId()); IncrementallySortBlocks(); return VK_SUCCESS; } @@ -12601,7 +12636,7 @@ VkResult VmaBlockVector::AllocatePage( pBlock, size, alignment, createInfo.flags, createInfo.pUserData, suballocType, strategy, pAllocation); if (res == VK_SUCCESS) { - VMA_DEBUG_LOG(" Created new block #%u Size=%llu", pBlock->GetId(), newBlockSize); + VMA_DEBUG_LOG_FORMAT(" Created new block #%u Size=%llu", pBlock->GetId(), newBlockSize); IncrementallySortBlocks(); return VK_SUCCESS; } @@ -12650,7 +12685,7 @@ void VmaBlockVector::Free(const VmaAllocation hAllocation) pBlock->PostFree(m_hAllocator); VMA_HEAVY_ASSERT(pBlock->Validate()); - VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", m_MemoryTypeIndex); + VMA_DEBUG_LOG_FORMAT(" Freed from MemoryTypeIndex=%u", m_MemoryTypeIndex); const bool canDeleteBlock = m_Blocks.size() > m_MinBlockCount; // pBlock became empty after this deallocation. @@ -12683,7 +12718,7 @@ void VmaBlockVector::Free(const VmaAllocation hAllocation) // lock, for performance reason. if (pBlockToDelete != VMA_NULL) { - VMA_DEBUG_LOG(" Deleted empty block #%u", pBlockToDelete->GetId()); + VMA_DEBUG_LOG_FORMAT(" Deleted empty block #%u", pBlockToDelete->GetId()); pBlockToDelete->Destroy(m_hAllocator); vma_delete(m_hAllocator, pBlockToDelete); } @@ -12786,7 +12821,7 @@ VkResult VmaBlockVector::CommitAllocationRequest( const bool isMappingAllowed = (allocFlags & (VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT | VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT)) != 0; - pBlock->PostAlloc(); + pBlock->PostAlloc(m_hAllocator); // Allocate from pCurrBlock. if (mapped) { @@ -12958,6 +12993,8 @@ VmaDefragmentationContext_T::VmaDefragmentationContext_T( const VmaDefragmentationInfo& info) : m_MaxPassBytes(info.maxBytesPerPass == 0 ? VK_WHOLE_SIZE : info.maxBytesPerPass), m_MaxPassAllocations(info.maxAllocationsPerPass == 0 ? UINT32_MAX : info.maxAllocationsPerPass), + m_BreakCallback(info.pfnBreakCallback), + m_BreakCallbackUserData(info.pBreakCallbackUserData), m_MoveAllocator(hAllocator->GetAllocationCallbacks()), m_Moves(m_MoveAllocator) { @@ -12991,20 +13028,18 @@ VmaDefragmentationContext_T::VmaDefragmentationContext_T( { case 0: // Default algorithm m_Algorithm = VMA_DEFRAGMENTATION_FLAG_ALGORITHM_BALANCED_BIT; + m_AlgorithmState = vma_new_array(hAllocator, StateBalanced, m_BlockVectorCount); + break; case VMA_DEFRAGMENTATION_FLAG_ALGORITHM_BALANCED_BIT: - { m_AlgorithmState = vma_new_array(hAllocator, StateBalanced, m_BlockVectorCount); break; - } case VMA_DEFRAGMENTATION_FLAG_ALGORITHM_EXTENSIVE_BIT: - { if (hAllocator->GetBufferImageGranularity() > 1) { m_AlgorithmState = vma_new_array(hAllocator, StateExtensive, m_BlockVectorCount); } break; } - } } VmaDefragmentationContext_T::~VmaDefragmentationContext_T() @@ -13214,29 +13249,24 @@ VkResult VmaDefragmentationContext_T::DefragmentPassEnd(VmaDefragmentationPassMo m_PassStats.bytesFreed += freedBlockSize; } - switch (m_Algorithm) - { - case VMA_DEFRAGMENTATION_FLAG_ALGORITHM_EXTENSIVE_BIT: + if(m_Algorithm == VMA_DEFRAGMENTATION_FLAG_ALGORITHM_EXTENSIVE_BIT && + m_AlgorithmState != VMA_NULL) { - if (m_AlgorithmState != VMA_NULL) + // Avoid unnecessary tries to allocate when new free block is available + StateExtensive& state = reinterpret_cast<StateExtensive*>(m_AlgorithmState)[vectorIndex]; + if (state.firstFreeBlock != SIZE_MAX) { - // Avoid unnecessary tries to allocate when new free block is avaiable - StateExtensive& state = reinterpret_cast<StateExtensive*>(m_AlgorithmState)[vectorIndex]; - if (state.firstFreeBlock != SIZE_MAX) + const size_t diff = prevCount - currentCount; + if (state.firstFreeBlock >= diff) { - const size_t diff = prevCount - currentCount; - if (state.firstFreeBlock >= diff) - { - state.firstFreeBlock -= diff; - if (state.firstFreeBlock != 0) - state.firstFreeBlock -= vector->GetBlock(state.firstFreeBlock - 1)->m_pMetadata->IsEmpty(); - } - else - state.firstFreeBlock = 0; + state.firstFreeBlock -= diff; + if (state.firstFreeBlock != 0) + state.firstFreeBlock -= vector->GetBlock(state.firstFreeBlock - 1)->m_pMetadata->IsEmpty(); } + else + state.firstFreeBlock = 0; } } - } } moveInfo.moveCount = 0; moveInfo.pMoves = VMA_NULL; @@ -13252,51 +13282,50 @@ VkResult VmaDefragmentationContext_T::DefragmentPassEnd(VmaDefragmentationPassMo // Move blocks with immovable allocations according to algorithm if (immovableBlocks.size() > 0) { - switch (m_Algorithm) - { - case VMA_DEFRAGMENTATION_FLAG_ALGORITHM_EXTENSIVE_BIT: + do { - if (m_AlgorithmState != VMA_NULL) + if(m_Algorithm == VMA_DEFRAGMENTATION_FLAG_ALGORITHM_EXTENSIVE_BIT) { - bool swapped = false; - // Move to the start of free blocks range - for (const FragmentedBlock& block : immovableBlocks) + if (m_AlgorithmState != VMA_NULL) { - StateExtensive& state = reinterpret_cast<StateExtensive*>(m_AlgorithmState)[block.data]; - if (state.operation != StateExtensive::Operation::Cleanup) + bool swapped = false; + // Move to the start of free blocks range + for (const FragmentedBlock& block : immovableBlocks) { - VmaBlockVector* vector = m_pBlockVectors[block.data]; - VmaMutexLockWrite lock(vector->GetMutex(), vector->GetAllocator()->m_UseMutex); - - for (size_t i = 0, count = vector->GetBlockCount() - m_ImmovableBlockCount; i < count; ++i) + StateExtensive& state = reinterpret_cast<StateExtensive*>(m_AlgorithmState)[block.data]; + if (state.operation != StateExtensive::Operation::Cleanup) { - if (vector->GetBlock(i) == block.block) + VmaBlockVector* vector = m_pBlockVectors[block.data]; + VmaMutexLockWrite lock(vector->GetMutex(), vector->GetAllocator()->m_UseMutex); + + for (size_t i = 0, count = vector->GetBlockCount() - m_ImmovableBlockCount; i < count; ++i) { - VMA_SWAP(vector->m_Blocks[i], vector->m_Blocks[vector->GetBlockCount() - ++m_ImmovableBlockCount]); - if (state.firstFreeBlock != SIZE_MAX) + if (vector->GetBlock(i) == block.block) { - if (i + 1 < state.firstFreeBlock) + VMA_SWAP(vector->m_Blocks[i], vector->m_Blocks[vector->GetBlockCount() - ++m_ImmovableBlockCount]); + if (state.firstFreeBlock != SIZE_MAX) { - if (state.firstFreeBlock > 1) - VMA_SWAP(vector->m_Blocks[i], vector->m_Blocks[--state.firstFreeBlock]); - else - --state.firstFreeBlock; + if (i + 1 < state.firstFreeBlock) + { + if (state.firstFreeBlock > 1) + VMA_SWAP(vector->m_Blocks[i], vector->m_Blocks[--state.firstFreeBlock]); + else + --state.firstFreeBlock; + } } + swapped = true; + break; } - swapped = true; - break; } } } + if (swapped) + result = VK_INCOMPLETE; + break; } - if (swapped) - result = VK_INCOMPLETE; - break; } - } - default: - { - // Move to the begining + + // Move to the beginning for (const FragmentedBlock& block : immovableBlocks) { VmaBlockVector* vector = m_pBlockVectors[block.data]; @@ -13311,9 +13340,7 @@ VkResult VmaDefragmentationContext_T::DefragmentPassEnd(VmaDefragmentationPassMo } } } - break; - } - } + } while (false); } // Bulk-map destination blocks @@ -13331,14 +13358,15 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation(VmaBlockVector& vector, { case VMA_DEFRAGMENTATION_FLAG_ALGORITHM_FAST_BIT: return ComputeDefragmentation_Fast(vector); - default: - VMA_ASSERT(0); case VMA_DEFRAGMENTATION_FLAG_ALGORITHM_BALANCED_BIT: return ComputeDefragmentation_Balanced(vector, index, true); case VMA_DEFRAGMENTATION_FLAG_ALGORITHM_FULL_BIT: return ComputeDefragmentation_Full(vector); case VMA_DEFRAGMENTATION_FLAG_ALGORITHM_EXTENSIVE_BIT: return ComputeDefragmentation_Extensive(vector, index); + default: + VMA_ASSERT(0); + return ComputeDefragmentation_Balanced(vector, index, true); } } @@ -13362,6 +13390,10 @@ VmaDefragmentationContext_T::MoveAllocationData VmaDefragmentationContext_T::Get VmaDefragmentationContext_T::CounterStatus VmaDefragmentationContext_T::CheckCounters(VkDeviceSize bytes) { + // Check custom criteria if exists + if (m_BreakCallback && m_BreakCallback(m_BreakCallbackUserData)) + return CounterStatus::End; + // Ignore allocation if will exceed max size for copy if (m_PassStats.bytesMoved + bytes > m_MaxPassBytes) { @@ -13370,6 +13402,8 @@ VmaDefragmentationContext_T::CounterStatus VmaDefragmentationContext_T::CheckCou else return CounterStatus::End; } + else + m_IgnoredAllocs = 0; return CounterStatus::Pass; } @@ -13379,8 +13413,8 @@ bool VmaDefragmentationContext_T::IncrementCounters(VkDeviceSize bytes) // Early return when max found if (++m_PassStats.allocationsMoved >= m_MaxPassAllocations || m_PassStats.bytesMoved >= m_MaxPassBytes) { - VMA_ASSERT(m_PassStats.allocationsMoved == m_MaxPassAllocations || - m_PassStats.bytesMoved == m_MaxPassBytes && "Exceeded maximal pass threshold!"); + VMA_ASSERT((m_PassStats.allocationsMoved == m_MaxPassAllocations || + m_PassStats.bytesMoved == m_MaxPassBytes) && "Exceeded maximal pass threshold!"); return true; } return false; @@ -13404,10 +13438,10 @@ bool VmaDefragmentationContext_T::ReallocWithinBlock(VmaBlockVector& vector, Vma continue; case CounterStatus::End: return true; - default: - VMA_ASSERT(0); case CounterStatus::Pass: break; + default: + VMA_ASSERT(0); } VkDeviceSize offset = moveData.move.srcAllocation->GetOffset(); @@ -13493,10 +13527,10 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Fast(VmaBlockVector& ve continue; case CounterStatus::End: return true; - default: - VMA_ASSERT(0); case CounterStatus::Pass: break; + default: + VMA_ASSERT(0); } // Check all previous blocks for free space @@ -13511,7 +13545,7 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Balanced(VmaBlockVector { // Go over every allocation and try to fit it in previous blocks at lowest offsets, // if not possible: realloc within single block to minimize offset (exclude offset == 0), - // but only if there are noticable gaps between them (some heuristic, ex. average size of allocation in block) + // but only if there are noticeable gaps between them (some heuristic, ex. average size of allocation in block) VMA_ASSERT(m_AlgorithmState != VMA_NULL); StateBalanced& vectorState = reinterpret_cast<StateBalanced*>(m_AlgorithmState)[index]; @@ -13540,10 +13574,10 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Balanced(VmaBlockVector continue; case CounterStatus::End: return true; - default: - VMA_ASSERT(0); case CounterStatus::Pass: break; + default: + VMA_ASSERT(0); } // Check all previous blocks for free space @@ -13594,7 +13628,7 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Balanced(VmaBlockVector } } - // No moves perfomed, update statistics to current vector state + // No moves performed, update statistics to current vector state if (startMoveCount == m_Moves.size() && !update) { vectorState.avgAllocSize = UINT64_MAX; @@ -13627,10 +13661,10 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Full(VmaBlockVector& ve continue; case CounterStatus::End: return true; - default: - VMA_ASSERT(0); case CounterStatus::Pass: break; + default: + VMA_ASSERT(0); } // Check all previous blocks for free space @@ -13718,10 +13752,10 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Extensive(VmaBlockVecto continue; case CounterStatus::End: return true; - default: - VMA_ASSERT(0); case CounterStatus::Pass: break; + default: + VMA_ASSERT(0); } // Check all previous blocks for free space @@ -13729,7 +13763,7 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Extensive(VmaBlockVecto { // Full clear performed already if (prevMoveCount != m_Moves.size() && freeMetadata->GetNextAllocation(handle) == VK_NULL_HANDLE) - reinterpret_cast<size_t*>(m_AlgorithmState)[index] = last; + vectorState.firstFreeBlock = last; return true; } } @@ -13759,14 +13793,15 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Extensive(VmaBlockVecto case StateExtensive::Operation::FindFreeBlockBuffer: vectorState.operation = StateExtensive::Operation::MoveBuffers; break; - default: - VMA_ASSERT(0); case StateExtensive::Operation::FindFreeBlockTexture: vectorState.operation = StateExtensive::Operation::MoveTextures; break; case StateExtensive::Operation::FindFreeBlockAll: vectorState.operation = StateExtensive::Operation::MoveAll; break; + default: + VMA_ASSERT(0); + vectorState.operation = StateExtensive::Operation::MoveTextures; } vectorState.firstFreeBlock = last; // Nothing done, block found without reallocations, can perform another reallocs in same pass @@ -13798,6 +13833,7 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Extensive(VmaBlockVecto } else break; + VMA_FALLTHROUGH; // Fallthrough } case StateExtensive::Operation::MoveBuffers: { @@ -13822,6 +13858,7 @@ bool VmaDefragmentationContext_T::ComputeDefragmentation_Extensive(VmaBlockVecto } else break; + VMA_FALLTHROUGH; // Fallthrough } case StateExtensive::Operation::MoveAll: { @@ -13904,10 +13941,10 @@ bool VmaDefragmentationContext_T::MoveDataToFreeBlocks(VmaSuballocationType curr continue; case CounterStatus::End: return true; - default: - VMA_ASSERT(0); case CounterStatus::Pass: break; + default: + VMA_ASSERT(0); } // Move only single type of resources at once @@ -14037,6 +14074,12 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT is set but required extension or Vulkan 1.2 is not available in your Vulkan header or its support in VMA has been disabled by a preprocessor macro."); } #endif +#if VMA_VULKAN_VERSION < 1003000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 3, 0)) + { + VMA_ASSERT(0 && "vulkanApiVersion >= VK_API_VERSION_1_3 but required Vulkan version is disabled by preprocessor macros."); + } +#endif #if VMA_VULKAN_VERSION < 1002000 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 2, 0)) { @@ -14213,6 +14256,12 @@ void VmaAllocator_T::ImportVulkanFunctions_Static() m_VulkanFunctions.vkGetImageMemoryRequirements2KHR = (PFN_vkGetImageMemoryRequirements2)vkGetImageMemoryRequirements2; m_VulkanFunctions.vkBindBufferMemory2KHR = (PFN_vkBindBufferMemory2)vkBindBufferMemory2; m_VulkanFunctions.vkBindImageMemory2KHR = (PFN_vkBindImageMemory2)vkBindImageMemory2; + } +#endif + +#if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties2KHR = (PFN_vkGetPhysicalDeviceMemoryProperties2)vkGetPhysicalDeviceMemoryProperties2; } #endif @@ -14265,7 +14314,7 @@ void VmaAllocator_T::ImportVulkanFunctions_Custom(const VmaVulkanFunctions* pVul VMA_COPY_IF_NOT_NULL(vkBindImageMemory2KHR); #endif -#if VMA_MEMORY_BUDGET +#if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000 VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties2KHR); #endif @@ -14320,8 +14369,18 @@ void VmaAllocator_T::ImportVulkanFunctions_Dynamic() VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements2KHR, PFN_vkGetImageMemoryRequirements2, "vkGetImageMemoryRequirements2"); VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory2KHR, PFN_vkBindBufferMemory2, "vkBindBufferMemory2"); VMA_FETCH_DEVICE_FUNC(vkBindImageMemory2KHR, PFN_vkBindImageMemory2, "vkBindImageMemory2"); + } +#endif + +#if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2, "vkGetPhysicalDeviceMemoryProperties2"); } + else if(m_UseExtMemoryBudget) + { + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2, "vkGetPhysicalDeviceMemoryProperties2KHR"); + } #endif #if VMA_DEDICATED_ALLOCATION @@ -14340,8 +14399,12 @@ void VmaAllocator_T::ImportVulkanFunctions_Dynamic() } #endif // #if VMA_BIND_MEMORY2 -#if VMA_MEMORY_BUDGET - if(m_UseExtMemoryBudget) +#if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2KHR, "vkGetPhysicalDeviceMemoryProperties2"); + } + else if(m_UseExtMemoryBudget) { VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2KHR, "vkGetPhysicalDeviceMemoryProperties2KHR"); } @@ -14438,7 +14501,7 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( VmaAllocation* pAllocations) { VMA_ASSERT(pAllocations != VMA_NULL); - VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, AllocationCount=%zu, Size=%llu", memTypeIndex, allocationCount, size); + VMA_DEBUG_LOG_FORMAT(" AllocateMemory: MemoryTypeIndex=%u, AllocationCount=%zu, Size=%llu", memTypeIndex, allocationCount, size); VmaAllocationCreateInfo finalCreateInfo = createInfo; VkResult res = CalcMemTypeParams( @@ -14487,7 +14550,8 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( // Protection against creating each allocation as dedicated when we reach or exceed heap size/budget, // which can quickly deplete maxMemoryAllocationCount: Don't prefer dedicated allocations when above // 3/4 of the maximum allocation count. - if(m_DeviceMemoryCount.load() > m_PhysicalDeviceProperties.limits.maxMemoryAllocationCount * 3 / 4) + if(m_PhysicalDeviceProperties.limits.maxMemoryAllocationCount < UINT32_MAX / 4 && + m_DeviceMemoryCount.load() > m_PhysicalDeviceProperties.limits.maxMemoryAllocationCount * 3 / 4) { dedicatedPreferred = false; } @@ -14515,7 +14579,7 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( blockVector.GetAllocationNextPtr()); if(res == VK_SUCCESS) { - // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here. + // Succeeded: AllocateDedicatedMemory function already filled pMemory, nothing more to do here. VMA_DEBUG_LOG(" Allocated as DedicatedMemory"); return VK_SUCCESS; } @@ -14556,7 +14620,7 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( blockVector.GetAllocationNextPtr()); if(res == VK_SUCCESS) { - // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here. + // Succeeded: AllocateDedicatedMemory function already filled pMemory, nothing more to do here. VMA_DEBUG_LOG(" Allocated as DedicatedMemory"); return VK_SUCCESS; } @@ -14683,7 +14747,7 @@ VkResult VmaAllocator_T::AllocateDedicatedMemory( { dedicatedAllocations.Register(pAllocations[allocIndex]); } - VMA_DEBUG_LOG(" Allocated DedicatedMemory Count=%zu, MemoryTypeIndex=#%u", allocationCount, memTypeIndex); + VMA_DEBUG_LOG_FORMAT(" Allocated DedicatedMemory Count=%zu, MemoryTypeIndex=#%u", allocationCount, memTypeIndex); } else { @@ -15241,7 +15305,7 @@ void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationI VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool) { - VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u, flags=%u", pCreateInfo->memoryTypeIndex, pCreateInfo->flags); + VMA_DEBUG_LOG_FORMAT(" CreatePool: MemoryTypeIndex=%u, flags=%u", pCreateInfo->memoryTypeIndex, pCreateInfo->flags); VmaPoolCreateInfo newCreateInfo = *pCreateInfo; @@ -15385,7 +15449,7 @@ VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits) VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory) { - AtomicTransactionalIncrement<uint32_t> deviceMemoryCountIncrement; + AtomicTransactionalIncrement<VMA_ATOMIC_UINT32> deviceMemoryCountIncrement; const uint64_t prevDeviceMemoryCount = deviceMemoryCountIncrement.Increment(&m_DeviceMemoryCount); #if VMA_DEBUG_DONT_EXCEED_MAX_MEMORY_ALLOCATION_COUNT if(prevDeviceMemoryCount >= m_PhysicalDeviceProperties.limits.maxMemoryAllocationCount) @@ -15542,6 +15606,7 @@ VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData) } return res; } + VMA_FALLTHROUGH; // Fallthrough case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: return hAllocation->DedicatedAllocMap(this, ppData); default: @@ -15575,7 +15640,7 @@ VkResult VmaAllocator_T::BindBufferMemory( VkBuffer hBuffer, const void* pNext) { - VkResult res = VK_SUCCESS; + VkResult res = VK_ERROR_UNKNOWN; switch(hAllocation->GetType()) { case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: @@ -15600,7 +15665,7 @@ VkResult VmaAllocator_T::BindImageMemory( VkImage hImage, const void* pNext) { - VkResult res = VK_SUCCESS; + VkResult res = VK_ERROR_UNKNOWN; switch(hAllocation->GetType()) { case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: @@ -15720,7 +15785,7 @@ void VmaAllocator_T::FreeDedicatedMemory(const VmaAllocation allocation) m_Budget.RemoveAllocation(MemoryTypeIndexToHeapIndex(allocation->GetMemoryTypeIndex()), allocation->GetSize()); m_AllocationObjectAllocator.Free(allocation); - VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex); + VMA_DEBUG_LOG_FORMAT(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex); } uint32_t VmaAllocator_T::CalculateGpuDefragmentationMemoryTypeBits() const @@ -15965,7 +16030,7 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) { json.WriteString("Name"); json.BeginString(); - json.ContinueString_Size(index++); + json.ContinueString((uint64_t)index++); if (pool->GetName()) { json.ContinueString(" - "); @@ -16122,11 +16187,11 @@ VMA_CALL_PRE void VMA_CALL_POST vmaBuildStatsString( json.WriteString("apiVersion"); json.BeginString(); - json.ContinueString(VK_API_VERSION_MAJOR(deviceProperties.apiVersion)); + json.ContinueString(VK_VERSION_MAJOR(deviceProperties.apiVersion)); json.ContinueString("."); - json.ContinueString(VK_API_VERSION_MINOR(deviceProperties.apiVersion)); + json.ContinueString(VK_VERSION_MINOR(deviceProperties.apiVersion)); json.ContinueString("."); - json.ContinueString(VK_API_VERSION_PATCH(deviceProperties.apiVersion)); + json.ContinueString(VK_VERSION_PATCH(deviceProperties.apiVersion)); json.EndString(); json.WriteString("GPU"); @@ -17178,9 +17243,20 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingBuffer( const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo, VkBuffer VMA_NULLABLE_NON_DISPATCHABLE* VMA_NOT_NULL pBuffer) { + return vmaCreateAliasingBuffer2(allocator, allocation, 0, pBufferCreateInfo, pBuffer); +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingBuffer2( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize allocationLocalOffset, + const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo, + VkBuffer VMA_NULLABLE_NON_DISPATCHABLE* VMA_NOT_NULL pBuffer) +{ VMA_ASSERT(allocator && pBufferCreateInfo && pBuffer && allocation); + VMA_ASSERT(allocationLocalOffset + pBufferCreateInfo->size <= allocation->GetSize()); - VMA_DEBUG_LOG("vmaCreateAliasingBuffer"); + VMA_DEBUG_LOG("vmaCreateAliasingBuffer2"); *pBuffer = VK_NULL_HANDLE; @@ -17206,7 +17282,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingBuffer( if (res >= 0) { // 2. Bind buffer with memory. - res = allocator->BindBufferMemory(allocation, 0, *pBuffer, VMA_NULL); + res = allocator->BindBufferMemory(allocation, allocationLocalOffset, *pBuffer, VMA_NULL); if (res >= 0) { return VK_SUCCESS; @@ -17343,11 +17419,21 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingImage( const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo, VkImage VMA_NULLABLE_NON_DISPATCHABLE* VMA_NOT_NULL pImage) { + return vmaCreateAliasingImage2(allocator, allocation, 0, pImageCreateInfo, pImage); +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingImage2( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize allocationLocalOffset, + const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo, + VkImage VMA_NULLABLE_NON_DISPATCHABLE* VMA_NOT_NULL pImage) +{ VMA_ASSERT(allocator && pImageCreateInfo && pImage && allocation); *pImage = VK_NULL_HANDLE; - VMA_DEBUG_LOG("vmaCreateImage"); + VMA_DEBUG_LOG("vmaCreateImage2"); if (pImageCreateInfo->extent.width == 0 || pImageCreateInfo->extent.height == 0 || @@ -17369,7 +17455,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAliasingImage( if (res >= 0) { // 2. Bind image with memory. - res = allocator->BindImageMemory(allocation, 0, *pImage, VMA_NULL); + res = allocator->BindImageMemory(allocation, allocationLocalOffset, *pImage, VMA_NULL); if (res >= 0) { return VK_SUCCESS; @@ -17573,7 +17659,7 @@ them before every `#include` of this library. This library is written in C++, but has C-compatible interface. Thus you can include and use vk_mem_alloc.h in C or C++ code, but full implementation with `VMA_IMPLEMENTATION` macro must be compiled as C++, NOT as C. -Some features of C++14 used. STL containers, RTTI, or C++ exceptions are not used. +Some features of C++14 are used. STL containers, RTTI, or C++ exceptions are not used. \section quick_start_initialization Initialization @@ -17590,6 +17676,34 @@ VmaAllocatorCreateInfo::vulkanApiVersion and which extensions did you enable by setting VmaAllocatorCreateInfo::flags (like #VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT for VK_KHR_buffer_device_address). Otherwise, VMA would use only features of Vulkan 1.0 core with no extensions. +\subsection quick_start_initialization_selecting_vulkan_version Selecting Vulkan version + +VMA supports Vulkan version down to 1.0, for backward compatibility. +If you want to use higher version, you need to inform the library about it. +This is a two-step process. + +<b>Step 1: Compile time.</b> By default, VMA compiles with code supporting the highest +Vulkan version found in the included `<vulkan/vulkan.h>` that is also supported by the library. +If this is OK, you don't need to do anything. +However, if you want to compile VMA as if only some lower Vulkan version was available, +define macro `VMA_VULKAN_VERSION` before every `#include "vk_mem_alloc.h"`. +It should have decimal numeric value in form of ABBBCCC, where A = major, BBB = minor, CCC = patch Vulkan version. +For example, to compile against Vulkan 1.2: + +\code +#define VMA_VULKAN_VERSION 1002000 // Vulkan 1.2 +#include "vk_mem_alloc.h" +\endcode + +<b>Step 2: Runtime.</b> Even when compiled with higher Vulkan version available, +VMA can use only features of a lower version, which is configurable during creation of the #VmaAllocator object. +By default, only Vulkan 1.0 is used. +To initialize the allocator with support for higher Vulkan version, you need to set member +VmaAllocatorCreateInfo::vulkanApiVersion to an appropriate value, e.g. using constants like `VK_API_VERSION_1_2`. +See code sample below. + +\subsection quick_start_initialization_importing_vulkan_functions Importing Vulkan functions + You may need to configure importing Vulkan functions. There are 3 ways to do this: -# **If you link with Vulkan static library** (e.g. "vulkan-1.lib" on Windows): @@ -17606,7 +17720,15 @@ You may need to configure importing Vulkan functions. There are 3 ways to do thi - Define `VMA_STATIC_VULKAN_FUNCTIONS` and `VMA_DYNAMIC_VULKAN_FUNCTIONS` to 0. - Pass these pointers via structure #VmaVulkanFunctions. +Example for case 2: + \code +#define VMA_STATIC_VULKAN_FUNCTIONS 0 +#define VMA_DYNAMIC_VULKAN_FUNCTIONS 1 +#include "vk_mem_alloc.h" + +... + VmaVulkanFunctions vulkanFunctions = {}; vulkanFunctions.vkGetInstanceProcAddr = &vkGetInstanceProcAddr; vulkanFunctions.vkGetDeviceProcAddr = &vkGetDeviceProcAddr; @@ -17726,8 +17848,8 @@ This will help the library decide about preferred memory type to ensure it has ` so you can map it. For example, a staging buffer that will be filled via mapped pointer and then -used as a source of transfer to the buffer decribed previously can be created like this. -It will likely and up in a memory type that is `HOST_VISIBLE` and `HOST_COHERENT` +used as a source of transfer to the buffer described previously can be created like this. +It will likely end up in a memory type that is `HOST_VISIBLE` and `HOST_COHERENT` but not `HOST_CACHED` (meaning uncached, write-combined) and not `DEVICE_LOCAL` (meaning system RAM). \code @@ -17750,7 +17872,7 @@ Usage values `VMA_MEMORY_USAGE_AUTO*` are legal to use only when the library kno about the resource being created by having `VkBufferCreateInfo` / `VkImageCreateInfo` passed, so they work with functions like: vmaCreateBuffer(), vmaCreateImage(), vmaFindMemoryTypeIndexForBufferInfo() etc. If you allocate raw memory using function vmaAllocateMemory(), you have to use other means of selecting -memory type, as decribed below. +memory type, as described below. \note Old usage values (`VMA_MEMORY_USAGE_GPU_ONLY`, `VMA_MEMORY_USAGE_CPU_ONLY`, @@ -17898,7 +18020,7 @@ object that you wanted to map. \section memory_mapping_persistently_mapped_memory Persistently mapped memory -Kepping your memory persistently mapped is generally OK in Vulkan. +Keeping your memory persistently mapped is generally OK in Vulkan. You don't need to unmap it before using its data on the GPU. The library provides a special feature designed for that: Allocations made with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag set in @@ -18119,6 +18241,12 @@ vkDestroyImage(allocator, img2, nullptr); vkDestroyImage(allocator, img1, nullptr); \endcode +VMA also provides convenience functions that create a buffer or image and bind it to memory +represented by an existing #VmaAllocation: +vmaCreateAliasingBuffer(), vmaCreateAliasingBuffer2(), +vmaCreateAliasingImage(), vmaCreateAliasingImage2(). +Versions with "2" offer additional parameter `allocationLocalOffset`. + Remember that using resources that alias in memory requires proper synchronization. You need to issue a memory barrier to make sure commands that use `img1` and `img2` don't overlap on GPU timeline. @@ -18383,7 +18511,7 @@ for(;;) { // Inspect pass.pMoves[i].srcAllocation, identify what buffer/image it represents. VmaAllocationInfo allocInfo; - vmaGetAllocationInfo(allocator, pMoves[i].srcAllocation, &allocInfo); + vmaGetAllocationInfo(allocator, pass.pMoves[i].srcAllocation, &allocInfo); MyEngineResourceData* resData = (MyEngineResourceData*)allocInfo.pUserData; // Recreate and bind this buffer/image at: pass.pMoves[i].dstMemory, pass.pMoves[i].dstOffset. @@ -18391,7 +18519,7 @@ for(;;) VkImage newImg; res = vkCreateImage(device, &imgCreateInfo, nullptr, &newImg); // Check res... - res = vmaBindImageMemory(allocator, pMoves[i].dstTmpAllocation, newImg); + res = vmaBindImageMemory(allocator, pass.pMoves[i].dstTmpAllocation, newImg); // Check res... // Issue a vkCmdCopyBuffer/vkCmdCopyImage to copy its content to the new place. @@ -18501,7 +18629,7 @@ especially the amount of memory allocated from Vulkan. If you need to obtain basic statistics about memory usage per heap, together with current budget, you can call function vmaGetHeapBudgets() and inspect structure #VmaBudget. -This is useful to keep track of memory usage and stay withing budget +This is useful to keep track of memory usage and stay within budget (see also \ref staying_within_budget). Example: @@ -18644,7 +18772,7 @@ VkResult res = vmaCreateVirtualBlock(&blockCreateInfo, &block); #VmaVirtualBlock object contains internal data structure that keeps track of free and occupied regions using the same code as the main Vulkan memory allocator. Similarly to #VmaAllocation for standard GPU allocations, there is #VmaVirtualAllocation type -that represents an opaque handle to an allocation withing the virtual block. +that represents an opaque handle to an allocation within the virtual block. In order to make such allocation: @@ -18970,7 +19098,7 @@ to decrease chances to be evicted to system memory by the operating system. \section usage_patterns_staging_copy_upload Staging copy for upload <b>When:</b> -A "staging" buffer than you want to map and fill from CPU code, then use as a source od transfer +A "staging" buffer than you want to map and fill from CPU code, then use as a source of transfer to some GPU resource. <b>What to do:</b> @@ -19037,13 +19165,13 @@ const float* downloadedData = (const float*)allocInfo.pMappedData; \section usage_patterns_advanced_data_uploading Advanced data uploading For resources that you frequently write on CPU via mapped pointer and -freqnently read on GPU e.g. as a uniform buffer (also called "dynamic"), multiple options are possible: +frequently read on GPU e.g. as a uniform buffer (also called "dynamic"), multiple options are possible: -# Easiest solution is to have one copy of the resource in `HOST_VISIBLE` memory, even if it means system RAM (not `DEVICE_LOCAL`) on systems with a discrete graphics card, and make the device reach out to that resource directly. - Reads performed by the device will then go through PCI Express bus. - The performace of this access may be limited, but it may be fine depending on the size + The performance of this access may be limited, but it may be fine depending on the size of this resource (whether it is small enough to quickly end up in GPU cache) and the sparsity of access. -# On systems with unified memory (e.g. AMD APU or Intel integrated graphics, mobile chips), @@ -19116,6 +19244,7 @@ else // [Executed in runtime]: memcpy(stagingAllocInfo.pMappedData, myData, myDataSize); + vmaFlushAllocation(allocator, stagingAlloc, 0, VK_WHOLE_SIZE); //vkCmdPipelineBarrier: VK_ACCESS_HOST_WRITE_BIT --> VK_ACCESS_TRANSFER_READ_BIT VkBufferCopy bufCopy = { 0, // srcOffset @@ -19499,7 +19628,7 @@ which means version numbers follow convention: Major.Minor.Patch (e.g. 2.3.0), w All changes between official releases are documented in file "CHANGELOG.md". -\warning Backward compatiblity is considered on the level of C++ source code, not binary linkage. +\warning Backward compatibility is considered on the level of C++ source code, not binary linkage. Adding new members to existing structures is treated as backward compatible if initializing the new members to binary zero results in the old behavior. You should always fully initialize all library structures to zeros and not rely on their |