diff options
Diffstat (limited to 'drivers/vulkan/rendering_device_vulkan.cpp')
| -rw-r--r-- | drivers/vulkan/rendering_device_vulkan.cpp | 517 |
1 files changed, 328 insertions, 189 deletions
diff --git a/drivers/vulkan/rendering_device_vulkan.cpp b/drivers/vulkan/rendering_device_vulkan.cpp index 6cf8f2dfac..a9f77c9072 100644 --- a/drivers/vulkan/rendering_device_vulkan.cpp +++ b/drivers/vulkan/rendering_device_vulkan.cpp @@ -2038,12 +2038,12 @@ RID RenderingDeviceVulkan::texture_create_shared(const TextureView &p_view, RID _THREAD_SAFE_METHOD_ Texture *src_texture = texture_owner.get_or_null(p_with_texture); - ERR_FAIL_COND_V(!src_texture, RID()); + ERR_FAIL_NULL_V(src_texture, RID()); if (src_texture->owner.is_valid()) { // Ahh this is a share. p_with_texture = src_texture->owner; src_texture = texture_owner.get_or_null(src_texture->owner); - ERR_FAIL_COND_V(!src_texture, RID()); // This is a bug. + ERR_FAIL_NULL_V(src_texture, RID()); // This is a bug. } // Create view. @@ -2167,7 +2167,7 @@ RID RenderingDeviceVulkan::texture_create_shared(const TextureView &p_view, RID return id; } -RID RenderingDeviceVulkan::texture_create_from_extension(TextureType p_type, DataFormat p_format, TextureSamples p_samples, uint64_t p_flags, uint64_t p_image, uint64_t p_width, uint64_t p_height, uint64_t p_depth, uint64_t p_layers) { +RID RenderingDeviceVulkan::texture_create_from_extension(TextureType p_type, DataFormat p_format, TextureSamples p_samples, BitField<RenderingDevice::TextureUsageBits> p_flags, uint64_t p_image, uint64_t p_width, uint64_t p_height, uint64_t p_depth, uint64_t p_layers) { _THREAD_SAFE_METHOD_ // This method creates a texture object using a VkImage created by an extension, module or other external source (OpenXR uses this). VkImage image = (VkImage)p_image; @@ -2313,12 +2313,12 @@ RID RenderingDeviceVulkan::texture_create_shared_from_slice(const TextureView &p _THREAD_SAFE_METHOD_ Texture *src_texture = texture_owner.get_or_null(p_with_texture); - ERR_FAIL_COND_V(!src_texture, RID()); + ERR_FAIL_NULL_V(src_texture, RID()); if (src_texture->owner.is_valid()) { // Ahh this is a share. p_with_texture = src_texture->owner; src_texture = texture_owner.get_or_null(src_texture->owner); - ERR_FAIL_COND_V(!src_texture, RID()); // This is a bug. + ERR_FAIL_NULL_V(src_texture, RID()); // This is a bug. } ERR_FAIL_COND_V_MSG(p_slice_type == TEXTURE_SLICE_CUBEMAP && (src_texture->type != TEXTURE_TYPE_CUBE && src_texture->type != TEXTURE_TYPE_CUBE_ARRAY), RID(), @@ -2469,12 +2469,12 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co "Updating textures is forbidden during creation of a draw or compute list"); Texture *texture = texture_owner.get_or_null(p_texture); - ERR_FAIL_COND_V(!texture, ERR_INVALID_PARAMETER); + ERR_FAIL_NULL_V(texture, ERR_INVALID_PARAMETER); if (texture->owner != RID()) { p_texture = texture->owner; texture = texture_owner.get_or_null(texture->owner); - ERR_FAIL_COND_V(!texture, ERR_BUG); // This is a bug. + ERR_FAIL_NULL_V(texture, ERR_BUG); // This is a bug. } ERR_FAIL_COND_V_MSG(texture->bound, ERR_CANT_ACQUIRE_RESOURCE, @@ -2622,7 +2622,7 @@ Error RenderingDeviceVulkan::_texture_update(RID p_texture, uint32_t p_layer, co vkCmdCopyBufferToImage(command_buffer, staging_buffer_blocks[staging_buffer_current].buffer, texture->image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &buffer_image_copy); - staging_buffer_blocks.write[staging_buffer_current].fill_amount += alloc_size; + staging_buffer_blocks.write[staging_buffer_current].fill_amount = alloc_offset + alloc_size; } } } @@ -2755,7 +2755,7 @@ Vector<uint8_t> RenderingDeviceVulkan::texture_get_data(RID p_texture, uint32_t _THREAD_SAFE_METHOD_ Texture *tex = texture_owner.get_or_null(p_texture); - ERR_FAIL_COND_V(!tex, Vector<uint8_t>()); + ERR_FAIL_NULL_V(tex, Vector<uint8_t>()); ERR_FAIL_COND_V_MSG(tex->bound, Vector<uint8_t>(), "Texture can't be retrieved while a draw list that uses it as part of a framebuffer is being created. Ensure the draw list is finalized (and that the color/depth texture using it is not set to `RenderingDevice.FINAL_ACTION_CONTINUE`) to retrieve this texture."); @@ -2885,7 +2885,7 @@ bool RenderingDeviceVulkan::texture_is_shared(RID p_texture) { _THREAD_SAFE_METHOD_ Texture *tex = texture_owner.get_or_null(p_texture); - ERR_FAIL_COND_V(!tex, false); + ERR_FAIL_NULL_V(tex, false); return tex->owner.is_valid(); } @@ -2893,11 +2893,34 @@ bool RenderingDeviceVulkan::texture_is_valid(RID p_texture) { return texture_owner.owns(p_texture); } +RD::TextureFormat RenderingDeviceVulkan::texture_get_format(RID p_texture) { + _THREAD_SAFE_METHOD_ + + Texture *tex = texture_owner.get_or_null(p_texture); + ERR_FAIL_NULL_V(tex, TextureFormat()); + + TextureFormat tf; + + tf.format = tex->format; + tf.width = tex->width; + tf.height = tex->height; + tf.depth = tex->depth; + tf.array_layers = tex->layers; + tf.mipmaps = tex->mipmaps; + tf.texture_type = tex->type; + tf.samples = tex->samples; + tf.usage_bits = tex->usage_flags; + tf.shareable_formats = tex->allowed_shared_formats; + tf.is_resolve_buffer = tex->is_resolve_buffer; + + return tf; +} + Size2i RenderingDeviceVulkan::texture_size(RID p_texture) { _THREAD_SAFE_METHOD_ Texture *tex = texture_owner.get_or_null(p_texture); - ERR_FAIL_COND_V(!tex, Size2i()); + ERR_FAIL_NULL_V(tex, Size2i()); return Size2i(tex->width, tex->height); } @@ -2905,7 +2928,7 @@ uint64_t RenderingDeviceVulkan::texture_get_native_handle(RID p_texture) { _THREAD_SAFE_METHOD_ Texture *tex = texture_owner.get_or_null(p_texture); - ERR_FAIL_COND_V(!tex, 0); + ERR_FAIL_NULL_V(tex, 0); return (uint64_t)tex->image; } @@ -2914,7 +2937,7 @@ Error RenderingDeviceVulkan::texture_copy(RID p_from_texture, RID p_to_texture, _THREAD_SAFE_METHOD_ Texture *src_tex = texture_owner.get_or_null(p_from_texture); - ERR_FAIL_COND_V(!src_tex, ERR_INVALID_PARAMETER); + ERR_FAIL_NULL_V(src_tex, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V_MSG(src_tex->bound, ERR_INVALID_PARAMETER, "Source texture can't be copied while a draw list that uses it as part of a framebuffer is being created. Ensure the draw list is finalized (and that the color/depth texture using it is not set to `RenderingDevice.FINAL_ACTION_CONTINUE`) to copy this texture."); @@ -2935,7 +2958,7 @@ Error RenderingDeviceVulkan::texture_copy(RID p_from_texture, RID p_to_texture, ERR_FAIL_COND_V(p_src_layer >= src_layer_count, ERR_INVALID_PARAMETER); Texture *dst_tex = texture_owner.get_or_null(p_to_texture); - ERR_FAIL_COND_V(!dst_tex, ERR_INVALID_PARAMETER); + ERR_FAIL_NULL_V(dst_tex, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V_MSG(dst_tex->bound, ERR_INVALID_PARAMETER, "Destination texture can't be copied while a draw list that uses it as part of a framebuffer is being created. Ensure the draw list is finalized (and that the color/depth texture using it is not set to `RenderingDevice.FINAL_ACTION_CONTINUE`) to copy this texture."); @@ -3089,7 +3112,7 @@ Error RenderingDeviceVulkan::texture_copy(RID p_from_texture, RID p_to_texture, image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; image_memory_barrier.image = dst_tex->image; - image_memory_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_memory_barrier.subresourceRange.aspectMask = dst_tex->read_aspect_mask; image_memory_barrier.subresourceRange.baseMipLevel = p_src_mipmap; image_memory_barrier.subresourceRange.levelCount = 1; image_memory_barrier.subresourceRange.baseArrayLayer = p_src_layer; @@ -3113,7 +3136,7 @@ Error RenderingDeviceVulkan::texture_resolve_multisample(RID p_from_texture, RID _THREAD_SAFE_METHOD_ Texture *src_tex = texture_owner.get_or_null(p_from_texture); - ERR_FAIL_COND_V(!src_tex, ERR_INVALID_PARAMETER); + ERR_FAIL_NULL_V(src_tex, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V_MSG(src_tex->bound, ERR_INVALID_PARAMETER, "Source texture can't be copied while a draw list that uses it as part of a framebuffer is being created. Ensure the draw list is finalized (and that the color/depth texture using it is not set to `RenderingDevice.FINAL_ACTION_CONTINUE`) to copy this texture."); @@ -3124,7 +3147,7 @@ Error RenderingDeviceVulkan::texture_resolve_multisample(RID p_from_texture, RID ERR_FAIL_COND_V_MSG(src_tex->samples == TEXTURE_SAMPLES_1, ERR_INVALID_PARAMETER, "Source texture must be multisampled."); Texture *dst_tex = texture_owner.get_or_null(p_to_texture); - ERR_FAIL_COND_V(!dst_tex, ERR_INVALID_PARAMETER); + ERR_FAIL_NULL_V(dst_tex, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V_MSG(dst_tex->bound, ERR_INVALID_PARAMETER, "Destination texture can't be copied while a draw list that uses it as part of a framebuffer is being created. Ensure the draw list is finalized (and that the color/depth texture using it is not set to `RenderingDevice.FINAL_ACTION_CONTINUE`) to copy this texture."); @@ -3271,7 +3294,7 @@ Error RenderingDeviceVulkan::texture_resolve_multisample(RID p_from_texture, RID image_memory_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; image_memory_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; image_memory_barrier.image = dst_tex->image; - image_memory_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_memory_barrier.subresourceRange.aspectMask = dst_tex->read_aspect_mask; image_memory_barrier.subresourceRange.baseMipLevel = dst_tex->base_mipmap; image_memory_barrier.subresourceRange.levelCount = 1; image_memory_barrier.subresourceRange.baseArrayLayer = dst_tex->base_layer; @@ -3288,7 +3311,7 @@ Error RenderingDeviceVulkan::texture_clear(RID p_texture, const Color &p_color, _THREAD_SAFE_METHOD_ Texture *src_tex = texture_owner.get_or_null(p_texture); - ERR_FAIL_COND_V(!src_tex, ERR_INVALID_PARAMETER); + ERR_FAIL_NULL_V(src_tex, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V_MSG(src_tex->bound, ERR_INVALID_PARAMETER, "Source texture can't be cleared while a draw list that uses it as part of a framebuffer is being created. Ensure the draw list is finalized (and that the color/depth texture using it is not set to `RenderingDevice.FINAL_ACTION_CONTINUE`) to clear this texture."); @@ -4276,7 +4299,7 @@ RenderingDevice::FramebufferFormatID RenderingDeviceVulkan::framebuffer_get_form _THREAD_SAFE_METHOD_ Framebuffer *framebuffer = framebuffer_owner.get_or_null(p_framebuffer); - ERR_FAIL_COND_V(!framebuffer, INVALID_ID); + ERR_FAIL_NULL_V(framebuffer, INVALID_ID); return framebuffer->format_id; } @@ -4291,7 +4314,7 @@ void RenderingDeviceVulkan::framebuffer_set_invalidation_callback(RID p_framebuf _THREAD_SAFE_METHOD_ Framebuffer *framebuffer = framebuffer_owner.get_or_null(p_framebuffer); - ERR_FAIL_COND(!framebuffer); + ERR_FAIL_NULL(framebuffer); framebuffer->invalidated_callback = p_callback; framebuffer->invalidated_callback_userdata = p_userdata; @@ -4476,6 +4499,7 @@ RID RenderingDeviceVulkan::vertex_array_create(uint32_t p_vertex_count, VertexFo if (atf.frequency == VERTEX_FREQUENCY_VERTEX) { // Validate size for regular drawing. uint64_t total_size = uint64_t(atf.stride) * (p_vertex_count - 1) + atf.offset + element_size; + ERR_FAIL_COND_V_MSG(total_size > buffer->size, RID(), "Attachment (" + itos(i) + ") will read past the end of the buffer."); @@ -4596,7 +4620,7 @@ static VkShaderStageFlagBits shader_stage_masks[RenderingDevice::SHADER_STAGE_MA String RenderingDeviceVulkan::_shader_uniform_debug(RID p_shader, int p_set) { String ret; const Shader *shader = shader_owner.get_or_null(p_shader); - ERR_FAIL_COND_V(!shader, String()); + ERR_FAIL_NULL_V(shader, String()); for (int i = 0; i < shader->sets.size(); i++) { if (p_set >= 0 && i != p_set) { continue; @@ -4615,8 +4639,9 @@ String RenderingDeviceVulkan::_shader_uniform_debug(RID p_shader, int p_set) { // Version 1: initial. // Version 2: Added shader name. // Version 3: Added writable. +// Version 4: 64-bit vertex input mask. -#define SHADER_BINARY_VERSION 3 +#define SHADER_BINARY_VERSION 4 String RenderingDeviceVulkan::shader_get_binary_cache_key() const { return "Vulkan-SV" + itos(SHADER_BINARY_VERSION); @@ -4642,7 +4667,7 @@ struct RenderingDeviceVulkanShaderBinarySpecializationConstant { }; struct RenderingDeviceVulkanShaderBinaryData { - uint32_t vertex_input_mask; + uint64_t vertex_input_mask; uint32_t fragment_output_mask; uint32_t specialization_constants_count; uint32_t is_compute; @@ -4664,7 +4689,7 @@ Vector<uint8_t> RenderingDeviceVulkan::shader_compile_binary_from_spirv(const Ve "Number of uniform sets is larger than what is supported by the hardware (" + itos(limits.maxBoundDescriptorSets) + ")."); // Collect reflection data into binary data. - RenderingDeviceVulkanShaderBinaryData binary_data; + RenderingDeviceVulkanShaderBinaryData binary_data{}; Vector<Vector<RenderingDeviceVulkanShaderBinaryDataBinding>> uniform_info; // Set bindings. Vector<RenderingDeviceVulkanShaderBinarySpecializationConstant> specialization_constants; { @@ -4789,11 +4814,14 @@ Vector<uint8_t> RenderingDeviceVulkan::shader_compile_binary_from_spirv(const Ve offset += sizeof(uint32_t); memcpy(binptr + offset, &binary_data, sizeof(RenderingDeviceVulkanShaderBinaryData)); offset += sizeof(RenderingDeviceVulkanShaderBinaryData); - memcpy(binptr + offset, shader_name_utf.ptr(), binary_data.shader_name_len); - offset += binary_data.shader_name_len; - if ((binary_data.shader_name_len % 4) != 0) { // Alignment rules are really strange. - offset += 4 - (binary_data.shader_name_len % 4); + if (binary_data.shader_name_len > 0) { + memcpy(binptr + offset, shader_name_utf.ptr(), binary_data.shader_name_len); + offset += binary_data.shader_name_len; + + if ((binary_data.shader_name_len % 4) != 0) { // Alignment rules are really strange. + offset += 4 - (binary_data.shader_name_len % 4); + } } for (int i = 0; i < uniform_info.size(); i++) { @@ -4835,7 +4863,7 @@ Vector<uint8_t> RenderingDeviceVulkan::shader_compile_binary_from_spirv(const Ve return ret; } -RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_shader_binary) { +RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_shader_binary, RID p_placeholder) { const uint8_t *binptr = p_shader_binary.ptr(); uint32_t binsize = p_shader_binary.size(); @@ -4855,7 +4883,7 @@ RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_ push_constant.size = binary_data.push_constant_size; push_constant.vk_stages_mask = binary_data.push_constant_vk_stages_mask; - uint32_t vertex_input_mask = binary_data.vertex_input_mask; + uint64_t vertex_input_mask = binary_data.vertex_input_mask; uint32_t fragment_output_mask = binary_data.fragment_output_mask; @@ -5020,17 +5048,25 @@ RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_ _THREAD_SAFE_METHOD_ - Shader shader; + RID id; + if (p_placeholder.is_null()) { + id = shader_owner.make_rid(); + } else { + id = p_placeholder; + } + + Shader *shader = shader_owner.get_or_null(id); + ERR_FAIL_NULL_V(shader, RID()); - shader.vertex_input_mask = vertex_input_mask; - shader.fragment_output_mask = fragment_output_mask; - shader.push_constant = push_constant; - shader.is_compute = is_compute; - shader.compute_local_size[0] = compute_local_size[0]; - shader.compute_local_size[1] = compute_local_size[1]; - shader.compute_local_size[2] = compute_local_size[2]; - shader.specialization_constants = specialization_constants; - shader.name = name; + shader->vertex_input_mask = vertex_input_mask; + shader->fragment_output_mask = fragment_output_mask; + shader->push_constant = push_constant; + shader->is_compute = is_compute; + shader->compute_local_size[0] = compute_local_size[0]; + shader->compute_local_size[1] = compute_local_size[1]; + shader->compute_local_size[2] = compute_local_size[2]; + shader->specialization_constants = specialization_constants; + shader->name = name; String error_text; @@ -5062,7 +5098,7 @@ RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_ shader_stage.pName = "main"; shader_stage.pSpecializationInfo = nullptr; - shader.pipeline_stages.push_back(shader_stage); + shader->pipeline_stages.push_back(shader_stage); } // Proceed to create descriptor sets. @@ -5105,8 +5141,8 @@ RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_ } } - shader.sets.push_back(set); - shader.set_formats.push_back(format); + shader->sets.push_back(set); + shader->set_formats.push_back(format); } } @@ -5116,13 +5152,13 @@ RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_ pipeline_layout_create_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; pipeline_layout_create_info.pNext = nullptr; pipeline_layout_create_info.flags = 0; - pipeline_layout_create_info.setLayoutCount = shader.sets.size(); + pipeline_layout_create_info.setLayoutCount = shader->sets.size(); Vector<VkDescriptorSetLayout> layouts; - layouts.resize(shader.sets.size()); + layouts.resize(shader->sets.size()); for (int i = 0; i < layouts.size(); i++) { - layouts.write[i] = shader.sets[i].descriptor_set_layout; + layouts.write[i] = shader->sets[i].descriptor_set_layout; } pipeline_layout_create_info.pSetLayouts = layouts.ptr(); @@ -5141,7 +5177,7 @@ RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_ pipeline_layout_create_info.pPushConstantRanges = nullptr; } - VkResult err = vkCreatePipelineLayout(device, &pipeline_layout_create_info, nullptr, &shader.pipeline_layout); + VkResult err = vkCreatePipelineLayout(device, &pipeline_layout_create_info, nullptr, &shader->pipeline_layout); if (err) { error_text = "Error (" + itos(err) + ") creating pipeline layout."; @@ -5151,29 +5187,35 @@ RID RenderingDeviceVulkan::shader_create_from_bytecode(const Vector<uint8_t> &p_ if (!success) { // Clean up if failed. - for (int i = 0; i < shader.pipeline_stages.size(); i++) { - vkDestroyShaderModule(device, shader.pipeline_stages[i].module, nullptr); + for (int i = 0; i < shader->pipeline_stages.size(); i++) { + vkDestroyShaderModule(device, shader->pipeline_stages[i].module, nullptr); } - for (int i = 0; i < shader.sets.size(); i++) { - vkDestroyDescriptorSetLayout(device, shader.sets[i].descriptor_set_layout, nullptr); + for (int i = 0; i < shader->sets.size(); i++) { + vkDestroyDescriptorSetLayout(device, shader->sets[i].descriptor_set_layout, nullptr); } + shader_owner.free(id); + ERR_FAIL_V_MSG(RID(), error_text); } - RID id = shader_owner.make_rid(shader); #ifdef DEV_ENABLED set_resource_name(id, "RID:" + itos(id.get_id())); #endif return id; } -uint32_t RenderingDeviceVulkan::shader_get_vertex_input_attribute_mask(RID p_shader) { +RID RenderingDeviceVulkan::shader_create_placeholder() { + Shader shader; + return shader_owner.make_rid(shader); +} + +uint64_t RenderingDeviceVulkan::shader_get_vertex_input_attribute_mask(RID p_shader) { _THREAD_SAFE_METHOD_ const Shader *shader = shader_owner.get_or_null(p_shader); - ERR_FAIL_COND_V(!shader, 0); + ERR_FAIL_NULL_V(shader, 0); return shader->vertex_input_mask; } @@ -5390,7 +5432,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, ERR_FAIL_COND_V(p_uniforms.size() == 0, RID()); Shader *shader = shader_owner.get_or_null(p_shader); - ERR_FAIL_COND_V(!shader, RID()); + ERR_FAIL_NULL_V(shader, RID()); ERR_FAIL_COND_V_MSG(p_shader_set >= (uint32_t)shader->sets.size() || shader->sets[p_shader_set].uniform_info.size() == 0, RID(), "Desired set (" + itos(p_shader_set) + ") not used by shader."); @@ -5459,7 +5501,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, for (uint32_t j = 0; j < uniform.get_id_count(); j++) { VkSampler *sampler = sampler_owner.get_or_null(uniform.get_id(j)); - ERR_FAIL_COND_V_MSG(!sampler, RID(), "Sampler (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid sampler."); + ERR_FAIL_NULL_V_MSG(sampler, RID(), "Sampler (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid sampler."); VkDescriptorImageInfo img_info; img_info.sampler = *sampler; @@ -5492,10 +5534,10 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, for (uint32_t j = 0; j < uniform.get_id_count(); j += 2) { VkSampler *sampler = sampler_owner.get_or_null(uniform.get_id(j + 0)); - ERR_FAIL_COND_V_MSG(!sampler, RID(), "SamplerBuffer (binding: " + itos(uniform.binding) + ", index " + itos(j + 1) + ") is not a valid sampler."); + ERR_FAIL_NULL_V_MSG(sampler, RID(), "SamplerBuffer (binding: " + itos(uniform.binding) + ", index " + itos(j + 1) + ") is not a valid sampler."); Texture *texture = texture_owner.get_or_null(uniform.get_id(j + 1)); - ERR_FAIL_COND_V_MSG(!texture, RID(), "Texture (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid texture."); + ERR_FAIL_NULL_V_MSG(texture, RID(), "Texture (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid texture."); ERR_FAIL_COND_V_MSG(!(texture->usage_flags & TEXTURE_USAGE_SAMPLING_BIT), RID(), "Texture (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") needs the TEXTURE_USAGE_SAMPLING_BIT usage flag set in order to be used as uniform."); @@ -5546,7 +5588,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, for (uint32_t j = 0; j < uniform.get_id_count(); j++) { Texture *texture = texture_owner.get_or_null(uniform.get_id(j)); - ERR_FAIL_COND_V_MSG(!texture, RID(), "Texture (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid texture."); + ERR_FAIL_NULL_V_MSG(texture, RID(), "Texture (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid texture."); ERR_FAIL_COND_V_MSG(!(texture->usage_flags & TEXTURE_USAGE_SAMPLING_BIT), RID(), "Texture (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") needs the TEXTURE_USAGE_SAMPLING_BIT usage flag set in order to be used as uniform."); @@ -5597,7 +5639,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, for (uint32_t j = 0; j < uniform.get_id_count(); j++) { Texture *texture = texture_owner.get_or_null(uniform.get_id(j)); - ERR_FAIL_COND_V_MSG(!texture, RID(), + ERR_FAIL_NULL_V_MSG(texture, RID(), "Image (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid texture."); ERR_FAIL_COND_V_MSG(!(texture->usage_flags & TEXTURE_USAGE_STORAGE_BIT), RID(), @@ -5643,7 +5685,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, for (uint32_t j = 0; j < uniform.get_id_count(); j++) { TextureBuffer *buffer = texture_buffer_owner.get_or_null(uniform.get_id(j)); - ERR_FAIL_COND_V_MSG(!buffer, RID(), "Texture Buffer (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid texture buffer."); + ERR_FAIL_NULL_V_MSG(buffer, RID(), "Texture Buffer (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid texture buffer."); buffer_info.push_back(buffer->buffer.buffer_info); buffer_view.push_back(buffer->view); @@ -5674,7 +5716,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, for (uint32_t j = 0; j < uniform.get_id_count(); j += 2) { VkSampler *sampler = sampler_owner.get_or_null(uniform.get_id(j + 0)); - ERR_FAIL_COND_V_MSG(!sampler, RID(), "SamplerBuffer (binding: " + itos(uniform.binding) + ", index " + itos(j + 1) + ") is not a valid sampler."); + ERR_FAIL_NULL_V_MSG(sampler, RID(), "SamplerBuffer (binding: " + itos(uniform.binding) + ", index " + itos(j + 1) + ") is not a valid sampler."); TextureBuffer *buffer = texture_buffer_owner.get_or_null(uniform.get_id(j + 1)); @@ -5685,7 +5727,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, image_info.push_back(img_info); - ERR_FAIL_COND_V_MSG(!buffer, RID(), "SamplerBuffer (binding: " + itos(uniform.binding) + ", index " + itos(j + 1) + ") is not a valid texture buffer."); + ERR_FAIL_NULL_V_MSG(buffer, RID(), "SamplerBuffer (binding: " + itos(uniform.binding) + ", index " + itos(j + 1) + ") is not a valid texture buffer."); buffer_info.push_back(buffer->buffer.buffer_info); buffer_view.push_back(buffer->view); @@ -5709,10 +5751,10 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, "Uniform buffer supplied (binding: " + itos(uniform.binding) + ") must provide one ID (" + itos(uniform.get_id_count()) + " provided)."); Buffer *buffer = uniform_buffer_owner.get_or_null(uniform.get_id(0)); - ERR_FAIL_COND_V_MSG(!buffer, RID(), "Uniform buffer supplied (binding: " + itos(uniform.binding) + ") is invalid."); + ERR_FAIL_NULL_V_MSG(buffer, RID(), "Uniform buffer supplied (binding: " + itos(uniform.binding) + ") is invalid."); - ERR_FAIL_COND_V_MSG(buffer->size != (uint32_t)set_uniform.length, RID(), - "Uniform buffer supplied (binding: " + itos(uniform.binding) + ") size (" + itos(buffer->size) + " does not match size of shader uniform: (" + itos(set_uniform.length) + ")."); + ERR_FAIL_COND_V_MSG(buffer->size < (uint32_t)set_uniform.length, RID(), + "Uniform buffer supplied (binding: " + itos(uniform.binding) + ") size (" + itos(buffer->size) + " is smaller than size of shader uniform: (" + itos(set_uniform.length) + ")."); write.dstArrayElement = 0; write.descriptorCount = 1; @@ -5735,7 +5777,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, ERR_FAIL_COND_V_MSG(!(buffer->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT), RID(), "Vertex buffer supplied (binding: " + itos(uniform.binding) + ") was not created with storage flag."); } - ERR_FAIL_COND_V_MSG(!buffer, RID(), "Storage buffer supplied (binding: " + itos(uniform.binding) + ") is invalid."); + ERR_FAIL_NULL_V_MSG(buffer, RID(), "Storage buffer supplied (binding: " + itos(uniform.binding) + ") is invalid."); // If 0, then it's sized on link time. ERR_FAIL_COND_V_MSG(set_uniform.length > 0 && buffer->size != (uint32_t)set_uniform.length, RID(), @@ -5764,7 +5806,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, for (uint32_t j = 0; j < uniform.get_id_count(); j++) { Texture *texture = texture_owner.get_or_null(uniform.get_id(j)); - ERR_FAIL_COND_V_MSG(!texture, RID(), + ERR_FAIL_NULL_V_MSG(texture, RID(), "InputAttachment (binding: " + itos(uniform.binding) + ", index " + itos(j) + ") is not a valid texture."); ERR_FAIL_COND_V_MSG(!(texture->usage_flags & TEXTURE_USAGE_SAMPLING_BIT), RID(), @@ -5804,7 +5846,7 @@ RID RenderingDeviceVulkan::uniform_set_create(const Vector<Uniform> &p_uniforms, // Need a descriptor pool. DescriptorPool *pool = _descriptor_pool_allocate(pool_key); - ERR_FAIL_COND_V(!pool, RID()); + ERR_FAIL_NULL_V(pool, RID()); VkDescriptorSetAllocateInfo descriptor_set_allocate_info; @@ -5864,11 +5906,69 @@ bool RenderingDeviceVulkan::uniform_set_is_valid(RID p_uniform_set) { void RenderingDeviceVulkan::uniform_set_set_invalidation_callback(RID p_uniform_set, InvalidationCallback p_callback, void *p_userdata) { UniformSet *us = uniform_set_owner.get_or_null(p_uniform_set); - ERR_FAIL_COND(!us); + ERR_FAIL_NULL(us); us->invalidated_callback = p_callback; us->invalidated_callback_userdata = p_userdata; } +Error RenderingDeviceVulkan::buffer_copy(RID p_src_buffer, RID p_dst_buffer, uint32_t p_src_offset, uint32_t p_dst_offset, uint32_t p_size, BitField<BarrierMask> p_post_barrier) { + _THREAD_SAFE_METHOD_ + + ERR_FAIL_COND_V_MSG(draw_list, ERR_INVALID_PARAMETER, + "Copying buffers is forbidden during creation of a draw list"); + ERR_FAIL_COND_V_MSG(compute_list, ERR_INVALID_PARAMETER, + "Copying buffers is forbidden during creation of a compute list"); + + // This method assumes the barriers have been pushed prior to being called, therefore no barriers are pushed + // for the source or destination buffers before performing the copy. These masks are effectively ignored. + VkPipelineStageFlags src_stage_mask = 0; + VkAccessFlags src_access_mask = 0; + Buffer *src_buffer = _get_buffer_from_owner(p_src_buffer, src_stage_mask, src_access_mask, BARRIER_MASK_NO_BARRIER); + if (!src_buffer) { + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Source buffer argument is not a valid buffer of any type."); + } + + VkPipelineStageFlags dst_stage_mask = 0; + VkAccessFlags dst_access = 0; + if (p_post_barrier.has_flag(BARRIER_MASK_TRANSFER)) { + // If the post barrier mask defines it, we indicate the destination buffer will require a barrier with these flags set + // after the copy command is queued. + dst_stage_mask = VK_PIPELINE_STAGE_TRANSFER_BIT; + dst_access = VK_ACCESS_TRANSFER_WRITE_BIT; + } + + Buffer *dst_buffer = _get_buffer_from_owner(p_dst_buffer, dst_stage_mask, dst_access, p_post_barrier); + if (!dst_buffer) { + ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Destination buffer argument is not a valid buffer of any type."); + } + + // Validate the copy's dimensions for both buffers. + ERR_FAIL_COND_V_MSG((p_size + p_src_offset) > src_buffer->size, ERR_INVALID_PARAMETER, "Size is larger than the source buffer."); + ERR_FAIL_COND_V_MSG((p_size + p_dst_offset) > dst_buffer->size, ERR_INVALID_PARAMETER, "Size is larger than the destination buffer."); + + // Perform the copy. + VkBufferCopy region; + region.srcOffset = p_src_offset; + region.dstOffset = p_dst_offset; + region.size = p_size; + vkCmdCopyBuffer(frames[frame].draw_command_buffer, src_buffer->buffer, dst_buffer->buffer, 1, ®ion); + +#ifdef FORCE_FULL_BARRIER + _full_barrier(true); +#else + if (dst_stage_mask == 0) { + dst_stage_mask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; + } + + // As indicated by the post barrier mask, push a new barrier. + if (p_post_barrier != RD::BARRIER_MASK_NO_BARRIER) { + _buffer_memory_barrier(dst_buffer->buffer, p_dst_offset, p_size, VK_PIPELINE_STAGE_TRANSFER_BIT, dst_stage_mask, VK_ACCESS_TRANSFER_WRITE_BIT, dst_access, true); + } +#endif + + return OK; +} + Error RenderingDeviceVulkan::buffer_update(RID p_buffer, uint32_t p_offset, uint32_t p_size, const void *p_data, BitField<BarrierMask> p_post_barrier) { _THREAD_SAFE_METHOD_ @@ -5941,9 +6041,6 @@ Error RenderingDeviceVulkan::buffer_clear(RID p_buffer, uint32_t p_offset, uint3 ERR_FAIL_COND_V_MSG(p_offset + p_size > buffer->size, ERR_INVALID_PARAMETER, "Attempted to write buffer (" + itos((p_offset + p_size) - buffer->size) + " bytes) past the end."); - // Should not be needed. - // _buffer_memory_barrier(buffer->buffer, p_offset, p_size, dst_stage_mask, VK_PIPELINE_STAGE_TRANSFER_BIT, dst_access, VK_ACCESS_TRANSFER_WRITE_BIT, p_post_barrier); - vkCmdFillBuffer(frames[frame].draw_command_buffer, buffer->buffer, p_offset, p_size, 0); #ifdef FORCE_FULL_BARRIER @@ -5953,7 +6050,7 @@ Error RenderingDeviceVulkan::buffer_clear(RID p_buffer, uint32_t p_offset, uint3 dst_stage_mask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; } - _buffer_memory_barrier(buffer->buffer, p_offset, p_size, VK_PIPELINE_STAGE_TRANSFER_BIT, dst_stage_mask, VK_ACCESS_TRANSFER_WRITE_BIT, dst_access, dst_stage_mask); + _buffer_memory_barrier(buffer->buffer, p_offset, p_size, VK_PIPELINE_STAGE_TRANSFER_BIT, dst_stage_mask, VK_ACCESS_TRANSFER_WRITE_BIT, dst_access, true); #endif return OK; @@ -6021,7 +6118,7 @@ RID RenderingDeviceVulkan::render_pipeline_create(RID p_shader, FramebufferForma // Needs a shader. Shader *shader = shader_owner.get_or_null(p_shader); - ERR_FAIL_COND_V(!shader, RID()); + ERR_FAIL_NULL_V(shader, RID()); ERR_FAIL_COND_V_MSG(shader->is_compute, RID(), "Compute shaders can't be used in render pipelines"); @@ -6057,8 +6154,8 @@ RID RenderingDeviceVulkan::render_pipeline_create(RID p_shader, FramebufferForma pipeline_vertex_input_state_create_info = vd.create_info; // Validate with inputs. - for (uint32_t i = 0; i < 32; i++) { - if (!(shader->vertex_input_mask & (1UL << i))) { + for (uint64_t i = 0; i < 64; i++) { + if (!(shader->vertex_input_mask & (1ULL << i))) { continue; } bool found = false; @@ -6493,7 +6590,7 @@ RID RenderingDeviceVulkan::compute_pipeline_create(RID p_shader, const Vector<Pi // Needs a shader. Shader *shader = shader_owner.get_or_null(p_shader); - ERR_FAIL_COND_V(!shader, RID()); + ERR_FAIL_NULL_V(shader, RID()); ERR_FAIL_COND_V_MSG(!shader->is_compute, RID(), "Non-compute shaders can't be used in compute pipelines"); @@ -6907,7 +7004,7 @@ RenderingDevice::DrawListID RenderingDeviceVulkan::draw_list_begin(RID p_framebu ERR_FAIL_COND_V_MSG(compute_list != nullptr && !compute_list->state.allow_draw_overlap, INVALID_ID, "Only one draw/compute list can be active at the same time."); Framebuffer *framebuffer = framebuffer_owner.get_or_null(p_framebuffer); - ERR_FAIL_COND_V(!framebuffer, INVALID_ID); + ERR_FAIL_NULL_V(framebuffer, INVALID_ID); Point2i viewport_offset; Point2i viewport_size = framebuffer->size; @@ -7015,7 +7112,7 @@ Error RenderingDeviceVulkan::draw_list_begin_split(RID p_framebuffer, uint32_t p ERR_FAIL_COND_V(p_splits < 1, ERR_INVALID_DECLARATION); Framebuffer *framebuffer = framebuffer_owner.get_or_null(p_framebuffer); - ERR_FAIL_COND_V(!framebuffer, ERR_INVALID_DECLARATION); + ERR_FAIL_NULL_V(framebuffer, ERR_INVALID_DECLARATION); Point2i viewport_offset; Point2i viewport_size = framebuffer->size; @@ -7145,7 +7242,7 @@ RenderingDeviceVulkan::DrawList *RenderingDeviceVulkan::_get_draw_list_ptr(DrawL void RenderingDeviceVulkan::draw_list_set_blend_constants(DrawListID p_list, const Color &p_color) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif @@ -7155,13 +7252,13 @@ void RenderingDeviceVulkan::draw_list_set_blend_constants(DrawListID p_list, con void RenderingDeviceVulkan::draw_list_bind_render_pipeline(DrawListID p_list, RID p_render_pipeline) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif const RenderPipeline *pipeline = render_pipeline_owner.get_or_null(p_render_pipeline); - ERR_FAIL_COND(!pipeline); + ERR_FAIL_NULL(pipeline); #ifdef DEBUG_ENABLED ERR_FAIL_COND(pipeline->validation.framebuffer_format != draw_list_framebuffer_format && pipeline->validation.render_pass != draw_list_current_subpass); #endif @@ -7229,14 +7326,14 @@ void RenderingDeviceVulkan::draw_list_bind_uniform_set(DrawListID p_list, RID p_ "Attempting to bind a descriptor set (" + itos(p_index) + ") greater than what the hardware supports (" + itos(limits.maxBoundDescriptorSets) + ")."); #endif DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif const UniformSet *uniform_set = uniform_set_owner.get_or_null(p_uniform_set); - ERR_FAIL_COND(!uniform_set); + ERR_FAIL_NULL(uniform_set); if (p_index > dl->state.set_count) { dl->state.set_count = p_index; @@ -7278,13 +7375,13 @@ void RenderingDeviceVulkan::draw_list_bind_uniform_set(DrawListID p_list, RID p_ void RenderingDeviceVulkan::draw_list_bind_vertex_array(DrawListID p_list, RID p_vertex_array) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif const VertexArray *vertex_array = vertex_array_owner.get_or_null(p_vertex_array); - ERR_FAIL_COND(!vertex_array); + ERR_FAIL_NULL(vertex_array); if (dl->state.vertex_array == p_vertex_array) { return; // Already set. @@ -7302,13 +7399,13 @@ void RenderingDeviceVulkan::draw_list_bind_vertex_array(DrawListID p_list, RID p void RenderingDeviceVulkan::draw_list_bind_index_array(DrawListID p_list, RID p_index_array) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif const IndexArray *index_array = index_array_owner.get_or_null(p_index_array); - ERR_FAIL_COND(!index_array); + ERR_FAIL_NULL(index_array); if (dl->state.index_array == p_index_array) { return; // Already set. @@ -7326,7 +7423,7 @@ void RenderingDeviceVulkan::draw_list_bind_index_array(DrawListID p_list, RID p_ void RenderingDeviceVulkan::draw_list_set_line_width(DrawListID p_list, float p_width) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif @@ -7336,7 +7433,7 @@ void RenderingDeviceVulkan::draw_list_set_line_width(DrawListID p_list, float p_ void RenderingDeviceVulkan::draw_list_set_push_constant(DrawListID p_list, const void *p_data, uint32_t p_data_size) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); @@ -7354,7 +7451,7 @@ void RenderingDeviceVulkan::draw_list_set_push_constant(DrawListID p_list, const void RenderingDeviceVulkan::draw_list_draw(DrawListID p_list, bool p_use_indices, uint32_t p_instances, uint32_t p_procedural_vertices) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif @@ -7460,7 +7557,7 @@ void RenderingDeviceVulkan::draw_list_draw(DrawListID p_list, bool p_use_indices void RenderingDeviceVulkan::draw_list_enable_scissor(DrawListID p_list, const Rect2 &p_rect) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif @@ -7483,7 +7580,7 @@ void RenderingDeviceVulkan::draw_list_enable_scissor(DrawListID p_list, const Re void RenderingDeviceVulkan::draw_list_disable_scissor(DrawListID p_list) { DrawList *dl = _get_draw_list_ptr(p_list); - ERR_FAIL_COND(!dl); + ERR_FAIL_NULL(dl); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(!dl->validation.active, "Submitted Draw Lists can no longer be modified."); #endif @@ -7502,7 +7599,7 @@ uint32_t RenderingDeviceVulkan::draw_list_get_current_pass() { RenderingDevice::DrawListID RenderingDeviceVulkan::draw_list_switch_to_next_pass() { _THREAD_SAFE_METHOD_ - ERR_FAIL_COND_V(draw_list == nullptr, INVALID_ID); + ERR_FAIL_NULL_V(draw_list, INVALID_ID); ERR_FAIL_COND_V(draw_list_current_subpass >= draw_list_subpass_count - 1, INVALID_FORMAT_ID); draw_list_current_subpass++; @@ -7518,7 +7615,7 @@ RenderingDevice::DrawListID RenderingDeviceVulkan::draw_list_switch_to_next_pass } Error RenderingDeviceVulkan::draw_list_switch_to_next_pass_split(uint32_t p_splits, DrawListID *r_split_ids) { _THREAD_SAFE_METHOD_ - ERR_FAIL_COND_V(draw_list == nullptr, ERR_INVALID_PARAMETER); + ERR_FAIL_NULL_V(draw_list, ERR_INVALID_PARAMETER); ERR_FAIL_COND_V(draw_list_current_subpass >= draw_list_subpass_count - 1, ERR_INVALID_PARAMETER); draw_list_current_subpass++; @@ -7659,7 +7756,7 @@ void RenderingDeviceVulkan::_draw_list_free(Rect2i *r_last_viewport) { void RenderingDeviceVulkan::draw_list_end(BitField<BarrierMask> p_post_barrier) { _THREAD_SAFE_METHOD_ - ERR_FAIL_COND_MSG(!draw_list, "Immediate draw list is already inactive."); + ERR_FAIL_NULL_MSG(draw_list, "Immediate draw list is already inactive."); _draw_list_free(); @@ -7709,8 +7806,10 @@ void RenderingDeviceVulkan::draw_list_end(BitField<BarrierMask> p_post_barrier) image_barriers = (VkImageMemoryBarrier *)alloca(sizeof(VkImageMemoryBarrier) * draw_list_storage_textures.size()); } - uint32_t src_stage = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; - uint32_t src_access = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + uint32_t src_stage = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; + uint32_t src_access = + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; if (image_barrier_count) { src_stage |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; @@ -7766,6 +7865,8 @@ void RenderingDeviceVulkan::draw_list_end(BitField<BarrierMask> p_post_barrier) /***********************/ RenderingDevice::ComputeListID RenderingDeviceVulkan::compute_list_begin(bool p_allow_draw_overlap) { + _THREAD_SAFE_METHOD_ + ERR_FAIL_COND_V_MSG(!p_allow_draw_overlap && draw_list != nullptr, INVALID_ID, "Only one draw list can be active at the same time."); ERR_FAIL_COND_V_MSG(compute_list != nullptr, INVALID_ID, "Only one draw/compute list can be active at the same time."); @@ -7780,13 +7881,15 @@ RenderingDevice::ComputeListID RenderingDeviceVulkan::compute_list_begin(bool p_ } void RenderingDeviceVulkan::compute_list_bind_compute_pipeline(ComputeListID p_list, RID p_compute_pipeline) { + // Must be called within a compute list, the class mutex is locked during that time + ERR_FAIL_COND(p_list != ID_TYPE_COMPUTE_LIST); - ERR_FAIL_COND(!compute_list); + ERR_FAIL_NULL(compute_list); ComputeList *cl = compute_list; const ComputePipeline *pipeline = compute_pipeline_owner.get_or_null(p_compute_pipeline); - ERR_FAIL_COND(!pipeline); + ERR_FAIL_NULL(pipeline); if (p_compute_pipeline == cl->state.pipeline) { return; // Redundant state, return. @@ -7844,8 +7947,10 @@ void RenderingDeviceVulkan::compute_list_bind_compute_pipeline(ComputeListID p_l } void RenderingDeviceVulkan::compute_list_bind_uniform_set(ComputeListID p_list, RID p_uniform_set, uint32_t p_index) { + // Must be called within a compute list, the class mutex is locked during that time + ERR_FAIL_COND(p_list != ID_TYPE_COMPUTE_LIST); - ERR_FAIL_COND(!compute_list); + ERR_FAIL_NULL(compute_list); ComputeList *cl = compute_list; @@ -7859,7 +7964,7 @@ void RenderingDeviceVulkan::compute_list_bind_uniform_set(ComputeListID p_list, #endif UniformSet *uniform_set = uniform_set_owner.get_or_null(p_uniform_set); - ERR_FAIL_COND(!uniform_set); + ERR_FAIL_NULL(uniform_set); if (p_index > cl->state.set_count) { cl->state.set_count = p_index; @@ -7999,7 +8104,7 @@ void RenderingDeviceVulkan::compute_list_bind_uniform_set(ComputeListID p_list, void RenderingDeviceVulkan::compute_list_set_push_constant(ComputeListID p_list, const void *p_data, uint32_t p_data_size) { ERR_FAIL_COND(p_list != ID_TYPE_COMPUTE_LIST); - ERR_FAIL_COND(!compute_list); + ERR_FAIL_NULL(compute_list); ComputeList *cl = compute_list; @@ -8018,8 +8123,10 @@ void RenderingDeviceVulkan::compute_list_set_push_constant(ComputeListID p_list, } void RenderingDeviceVulkan::compute_list_dispatch(ComputeListID p_list, uint32_t p_x_groups, uint32_t p_y_groups, uint32_t p_z_groups) { + // Must be called within a compute list, the class mutex is locked during that time + ERR_FAIL_COND(p_list != ID_TYPE_COMPUTE_LIST); - ERR_FAIL_COND(!compute_list); + ERR_FAIL_NULL(compute_list); ComputeList *cl = compute_list; @@ -8079,7 +8186,7 @@ void RenderingDeviceVulkan::compute_list_dispatch(ComputeListID p_list, uint32_t void RenderingDeviceVulkan::compute_list_dispatch_threads(ComputeListID p_list, uint32_t p_x_threads, uint32_t p_y_threads, uint32_t p_z_threads) { ERR_FAIL_COND(p_list != ID_TYPE_COMPUTE_LIST); - ERR_FAIL_COND(!compute_list); + ERR_FAIL_NULL(compute_list); #ifdef DEBUG_ENABLED ERR_FAIL_COND_MSG(p_x_threads == 0, "Dispatch amount of X compute threads (" + itos(p_x_threads) + ") is zero."); @@ -8106,11 +8213,11 @@ void RenderingDeviceVulkan::compute_list_dispatch_threads(ComputeListID p_list, void RenderingDeviceVulkan::compute_list_dispatch_indirect(ComputeListID p_list, RID p_buffer, uint32_t p_offset) { ERR_FAIL_COND(p_list != ID_TYPE_COMPUTE_LIST); - ERR_FAIL_COND(!compute_list); + ERR_FAIL_NULL(compute_list); ComputeList *cl = compute_list; Buffer *buffer = storage_buffer_owner.get_or_null(p_buffer); - ERR_FAIL_COND(!buffer); + ERR_FAIL_NULL(buffer); ERR_FAIL_COND_MSG(!(buffer->usage & VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT), "Buffer provided was not created to do indirect dispatch."); @@ -8161,13 +8268,15 @@ void RenderingDeviceVulkan::compute_list_dispatch_indirect(ComputeListID p_list, } void RenderingDeviceVulkan::compute_list_add_barrier(ComputeListID p_list) { + // Must be called within a compute list, the class mutex is locked during that time + uint32_t barrier_flags = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; uint32_t access_flags = VK_ACCESS_SHADER_READ_BIT; _compute_list_add_barrier(BARRIER_MASK_COMPUTE, barrier_flags, access_flags); } void RenderingDeviceVulkan::_compute_list_add_barrier(BitField<BarrierMask> p_post_barrier, uint32_t p_barrier_flags, uint32_t p_access_flags) { - ERR_FAIL_COND(!compute_list); + ERR_FAIL_NULL(compute_list); VkImageMemoryBarrier *image_barriers = nullptr; @@ -8226,7 +8335,7 @@ void RenderingDeviceVulkan::_compute_list_add_barrier(BitField<BarrierMask> p_po } void RenderingDeviceVulkan::compute_list_end(BitField<BarrierMask> p_post_barrier) { - ERR_FAIL_COND(!compute_list); + ERR_FAIL_NULL(compute_list); uint32_t barrier_flags = 0; uint32_t access_flags = 0; @@ -8267,8 +8376,11 @@ void RenderingDeviceVulkan::barrier(BitField<BarrierMask> p_from, BitField<Barri src_access_flags |= VK_ACCESS_SHADER_WRITE_BIT; } if (p_from.has_flag(BARRIER_MASK_FRAGMENT)) { - src_barrier_flags |= VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; - src_access_flags |= VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + src_barrier_flags |= VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; + src_access_flags |= + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; } if (p_from.has_flag(BARRIER_MASK_TRANSFER)) { src_barrier_flags |= VK_PIPELINE_STAGE_TRANSFER_BIT; @@ -8313,7 +8425,7 @@ void RenderingDeviceVulkan::full_barrier() { #if 0 void RenderingDeviceVulkan::draw_list_render_secondary_to_framebuffer(ID p_framebuffer, ID *p_draw_lists, uint32_t p_draw_list_count, InitialAction p_initial_action, FinalAction p_final_action, const Vector<Variant> &p_clear_colors) { VkCommandBuffer frame_cmdbuf = frames[frame].frame_buffer; - ERR_FAIL_COND(!frame_cmdbuf); + ERR_FAIL_NULL(frame_cmdbuf); VkRenderPassBeginInfo render_pass_begin; render_pass_begin.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO; @@ -8649,6 +8761,8 @@ void RenderingDeviceVulkan::swap_buffers() { } void RenderingDeviceVulkan::submit() { + _THREAD_SAFE_METHOD_ + ERR_FAIL_COND_MSG(local_device.is_null(), "Only local devices can submit and sync."); ERR_FAIL_COND_MSG(local_device_processing, "device already submitted, call sync to wait until done."); @@ -8660,6 +8774,8 @@ void RenderingDeviceVulkan::submit() { } void RenderingDeviceVulkan::sync() { + _THREAD_SAFE_METHOD_ + ERR_FAIL_COND_MSG(local_device.is_null(), "Only local devices can submit and sync."); ERR_FAIL_COND_MSG(!local_device_processing, "sync can only be called after a submit"); @@ -9019,12 +9135,32 @@ void RenderingDeviceVulkan::initialize(VulkanContext *p_context, bool p_local_de draw_list_split = false; compute_list = nullptr; + + pipelines_cache.file_path = "user://vulkan/pipelines"; + pipelines_cache.file_path += "." + context->get_device_name().validate_filename().replace(" ", "_").to_lower(); + if (Engine::get_singleton()->is_editor_hint()) { + pipelines_cache.file_path += ".editor"; + } + pipelines_cache.file_path += ".cache"; + + // Prepare most fields now. + VkPhysicalDeviceProperties props; + vkGetPhysicalDeviceProperties(context->get_physical_device(), &props); + pipelines_cache.header.magic = 868 + VK_PIPELINE_CACHE_HEADER_VERSION_ONE; + pipelines_cache.header.device_id = props.deviceID; + pipelines_cache.header.vendor_id = props.vendorID; + pipelines_cache.header.driver_version = props.driverVersion; + memcpy(pipelines_cache.header.uuid, props.pipelineCacheUUID, VK_UUID_SIZE); + pipelines_cache.header.driver_abi = sizeof(void *); + _load_pipeline_cache(); print_verbose(vformat("Startup PSO cache (%.1f MiB)", pipelines_cache.buffer.size() / (1024.0f * 1024.0f))); VkPipelineCacheCreateInfo cache_info = {}; cache_info.sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO; cache_info.pNext = nullptr; - cache_info.flags = 0; + if (context->get_pipeline_cache_control_support()) { + cache_info.flags = VK_PIPELINE_CACHE_CREATE_EXTERNALLY_SYNCHRONIZED_BIT; + } cache_info.initialDataSize = pipelines_cache.buffer.size(); cache_info.pInitialData = pipelines_cache.buffer.ptr(); VkResult err = vkCreatePipelineCache(device, &cache_info, nullptr, &pipelines_cache.cache_object); @@ -9035,111 +9171,106 @@ void RenderingDeviceVulkan::initialize(VulkanContext *p_context, bool p_local_de } void RenderingDeviceVulkan::_load_pipeline_cache() { - if (!DirAccess::exists("user://vulkan/")) { - Ref<DirAccess> da = DirAccess::create(DirAccess::ACCESS_USERDATA); - - if (da.is_valid()) { - da->make_dir_recursive("user://vulkan/"); - } - } + DirAccess::make_dir_recursive_absolute(pipelines_cache.file_path.get_base_dir()); - if (FileAccess::exists("user://vulkan/pipelines.cache")) { + if (FileAccess::exists(pipelines_cache.file_path)) { Error file_error; - Vector<uint8_t> file_data = FileAccess::get_file_as_bytes("user://vulkan/pipelines.cache", &file_error); + Vector<uint8_t> file_data = FileAccess::get_file_as_bytes(pipelines_cache.file_path, &file_error); if (file_error != OK || file_data.size() <= (int)sizeof(PipelineCacheHeader)) { WARN_PRINT("Invalid/corrupt pipelines cache."); return; } - PipelineCacheHeader header = {}; - memcpy((char *)&header, file_data.ptr(), sizeof(PipelineCacheHeader)); - if (header.magic != 868 + VK_PIPELINE_CACHE_HEADER_VERSION_ONE) { + const PipelineCacheHeader *header = reinterpret_cast<const PipelineCacheHeader *>(file_data.ptr()); + if (header->magic != 868 + VK_PIPELINE_CACHE_HEADER_VERSION_ONE) { WARN_PRINT("Invalid pipelines cache magic number."); return; } - pipelines_cache.buffer.resize(file_data.size() - sizeof(PipelineCacheHeader)); - memcpy(pipelines_cache.buffer.ptrw(), file_data.ptr() + sizeof(PipelineCacheHeader), pipelines_cache.buffer.size()); - VkPhysicalDeviceProperties props; - vkGetPhysicalDeviceProperties(context->get_physical_device(), &props); - bool invalid_uuid = false; - for (size_t i = 0; i < VK_UUID_SIZE; i++) { - if (header.uuid[i] != props.pipelineCacheUUID[i]) { - invalid_uuid = true; - break; - } - } - if (header.data_hash != hash_murmur3_buffer(pipelines_cache.buffer.ptr(), pipelines_cache.buffer.size()) || header.data_size != (uint32_t)pipelines_cache.buffer.size() || header.vendor_id != props.vendorID || header.device_id != props.deviceID || header.driver_abi != sizeof(void *) || invalid_uuid) { + const uint8_t *loaded_buffer_start = file_data.ptr() + sizeof(PipelineCacheHeader); + uint32_t loaded_buffer_size = file_data.size() - sizeof(PipelineCacheHeader); + if (header->data_hash != hash_murmur3_buffer(loaded_buffer_start, loaded_buffer_size) || + header->data_size != loaded_buffer_size || + header->vendor_id != pipelines_cache.header.vendor_id || + header->device_id != pipelines_cache.header.device_id || + header->driver_version != pipelines_cache.header.driver_version || + memcmp(header->uuid, pipelines_cache.header.uuid, VK_UUID_SIZE) != 0 || + header->driver_abi != pipelines_cache.header.driver_abi) { WARN_PRINT("Invalid pipelines cache header."); pipelines_cache.current_size = 0; pipelines_cache.buffer.clear(); } else { - pipelines_cache.current_size = pipelines_cache.buffer.size(); + pipelines_cache.current_size = loaded_buffer_size; + pipelines_cache.buffer.resize(loaded_buffer_size); + memcpy(pipelines_cache.buffer.ptr(), loaded_buffer_start, pipelines_cache.buffer.size()); } } } void RenderingDeviceVulkan::_update_pipeline_cache(bool p_closing) { - size_t pso_blob_size = 0; - float save_interval = GLOBAL_GET("rendering/rendering_device/pipeline_cache/save_chunk_size_mb"); - VkResult vr = vkGetPipelineCacheData(device, pipelines_cache.cache_object, &pso_blob_size, nullptr); - ERR_FAIL_COND(vr); - size_t difference = (pso_blob_size - pipelines_cache.current_size) / (1024 * 1024); - if (p_closing && Engine::get_singleton()->is_editor_hint()) { - // This is mostly for the editor to check if after playing the game, game's pipeline cache size still matches with editor's cache. - _load_pipeline_cache(); - if (pipelines_cache.current_size > pso_blob_size) { - pso_blob_size = pipelines_cache.current_size; - if (pipelines_cache_save_task != WorkerThreadPool::INVALID_TASK_ID || !WorkerThreadPool::get_singleton()->is_task_completed(pipelines_cache_save_task)) { + { + bool still_saving = pipelines_cache_save_task != WorkerThreadPool::INVALID_TASK_ID && !WorkerThreadPool::get_singleton()->is_task_completed(pipelines_cache_save_task); + if (still_saving) { + if (p_closing) { WorkerThreadPool::get_singleton()->wait_for_task_completion(pipelines_cache_save_task); + pipelines_cache_save_task = WorkerThreadPool::INVALID_TASK_ID; + } else { + // We can't save until the currently running save is done. We'll retry next time; worst case, we'll save when exiting. + return; } } } - if (pso_blob_size == pipelines_cache.current_size) { - return; - } else if (difference < save_interval && !p_closing) { - return; - } - if (p_closing) { - if (pipelines_cache_save_task == WorkerThreadPool::INVALID_TASK_ID || WorkerThreadPool::get_singleton()->is_task_completed(pipelines_cache_save_task)) { - pipelines_cache_save_task = WorkerThreadPool::get_singleton()->add_template_task(this, &RenderingDeviceVulkan::_save_pipeline_cache_threaded, pso_blob_size, false, "PipelineCacheSave"); - WorkerThreadPool::get_singleton()->wait_for_task_completion(pipelines_cache_save_task); + { + // FIXME: + // We're letting the cache grow unboundedly. We may want to set at limit and see if implementations use LRU or the like. + // If we do, we won't be able to assume any longer that the cache is dirty if, and only if, it has grown. + size_t pso_blob_size = 0; + VkResult vr = vkGetPipelineCacheData(device, pipelines_cache.cache_object, &pso_blob_size, nullptr); + ERR_FAIL_COND(vr); + size_t difference = pso_blob_size - pipelines_cache.current_size; + + bool must_save = false; + + if (p_closing) { + must_save = difference > 0; } else { - WorkerThreadPool::get_singleton()->wait_for_task_completion(pipelines_cache_save_task); - pipelines_cache_save_task = WorkerThreadPool::get_singleton()->add_template_task(this, &RenderingDeviceVulkan::_save_pipeline_cache_threaded, pso_blob_size, false, "PipelineCacheSave"); - WorkerThreadPool::get_singleton()->wait_for_task_completion(pipelines_cache_save_task); + float save_interval = GLOBAL_GET("rendering/rendering_device/pipeline_cache/save_chunk_size_mb"); + must_save = difference > 0 && difference / (1024.0f * 1024.0f) >= save_interval; } - } else { - if (pipelines_cache_save_task == WorkerThreadPool::INVALID_TASK_ID || WorkerThreadPool::get_singleton()->is_task_completed(pipelines_cache_save_task)) { - pipelines_cache_save_task = WorkerThreadPool::get_singleton()->add_template_task(this, &RenderingDeviceVulkan::_save_pipeline_cache_threaded, pso_blob_size, false, "PipelineCacheSave"); + + if (must_save) { + pipelines_cache.current_size = pso_blob_size; + } else { + return; } } + + if (p_closing) { + _save_pipeline_cache(this); + } else { + pipelines_cache_save_task = WorkerThreadPool::get_singleton()->add_native_task(&_save_pipeline_cache, this, false, "PipelineCacheSave"); + } } -void RenderingDeviceVulkan::_save_pipeline_cache_threaded(size_t p_pso_blob_size) { - pipelines_cache.current_size = p_pso_blob_size; - pipelines_cache.buffer.clear(); - pipelines_cache.buffer.resize(p_pso_blob_size); - VkResult vr = vkGetPipelineCacheData(device, pipelines_cache.cache_object, &p_pso_blob_size, pipelines_cache.buffer.ptrw()); - ERR_FAIL_COND(vr); - print_verbose(vformat("Updated PSO cache (%.1f MiB)", p_pso_blob_size / (1024.0f * 1024.0f))); +void RenderingDeviceVulkan::_save_pipeline_cache(void *p_data) { + RenderingDeviceVulkan *self = static_cast<RenderingDeviceVulkan *>(p_data); - VkPhysicalDeviceProperties props; - vkGetPhysicalDeviceProperties(context->get_physical_device(), &props); - PipelineCacheHeader header = {}; - header.magic = 868 + VK_PIPELINE_CACHE_HEADER_VERSION_ONE; - header.data_size = pipelines_cache.buffer.size(); - header.data_hash = hash_murmur3_buffer(pipelines_cache.buffer.ptr(), pipelines_cache.buffer.size()); - header.device_id = props.deviceID; - header.vendor_id = props.vendorID; - header.driver_version = props.driverVersion; - for (size_t i = 0; i < VK_UUID_SIZE; i++) { - header.uuid[i] = props.pipelineCacheUUID[i]; - } - header.driver_abi = sizeof(void *); - Ref<FileAccess> f = FileAccess::open("user://vulkan/pipelines.cache", FileAccess::WRITE, nullptr); + self->pipelines_cache.buffer.resize(self->pipelines_cache.current_size); + + self->_thread_safe_.lock(); + VkResult vr = vkGetPipelineCacheData(self->device, self->pipelines_cache.cache_object, &self->pipelines_cache.current_size, self->pipelines_cache.buffer.ptr()); + self->_thread_safe_.unlock(); + ERR_FAIL_COND(vr != VK_SUCCESS && vr != VK_INCOMPLETE); // Incomplete is OK because the cache may have grown since the size was queried (unless when exiting). + print_verbose(vformat("Updated PSO cache (%.1f MiB)", self->pipelines_cache.current_size / (1024.0f * 1024.0f))); + + // The real buffer size may now be bigger than the updated current_size. + // We take into account the new size but keep the buffer resized in a worst-case fashion. + + self->pipelines_cache.header.data_size = self->pipelines_cache.current_size; + self->pipelines_cache.header.data_hash = hash_murmur3_buffer(self->pipelines_cache.buffer.ptr(), self->pipelines_cache.current_size); + Ref<FileAccess> f = FileAccess::open(self->pipelines_cache.file_path, FileAccess::WRITE, nullptr); if (f.is_valid()) { - f->store_buffer((const uint8_t *)&header, sizeof(PipelineCacheHeader)); - f->store_buffer(pipelines_cache.buffer); + f->store_buffer((const uint8_t *)&self->pipelines_cache.header, sizeof(PipelineCacheHeader)); + f->store_buffer(self->pipelines_cache.buffer.ptr(), self->pipelines_cache.current_size); } } @@ -9434,6 +9565,14 @@ uint64_t RenderingDeviceVulkan::limit_get(Limit p_limit) const { VulkanContext::SubgroupCapabilities subgroup_capabilities = context->get_subgroup_capabilities(); return subgroup_capabilities.size; } + case LIMIT_SUBGROUP_MIN_SIZE: { + VulkanContext::SubgroupCapabilities subgroup_capabilities = context->get_subgroup_capabilities(); + return subgroup_capabilities.min_size; + } + case LIMIT_SUBGROUP_MAX_SIZE: { + VulkanContext::SubgroupCapabilities subgroup_capabilities = context->get_subgroup_capabilities(); + return subgroup_capabilities.max_size; + } case LIMIT_SUBGROUP_IN_SHADERS: { VulkanContext::SubgroupCapabilities subgroup_capabilities = context->get_subgroup_capabilities(); return subgroup_capabilities.supported_stages_flags_rd(); |