From 6ef67aea84de23a4fd9212c952f4e4e4e2c46de8 Mon Sep 17 00:00:00 2001 From: Chinmay Garde Date: Thu, 15 Feb 2024 10:30:41 -0800 Subject: [PATCH] [Impeller] Implement YUV texture import and sampling for video player frames. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch rounds out support for importing Android Hardware Buffers as Vulkan images and sampling from them. Not Android Hardware Buffers are in formats that Vulkan (let alone Impeller) understands. Some YUV textures have formats with no equivalent `vk::Format` enum and need extra information on how to sample from them. This patch adds support for correctly importing and sampling from such formats. There are severe restrictions on how sampling from such external formats works. For one, it isn’t possible to assign a combined image sampler in the render pass. The pipeline itself needs to be rebuilt to reference a specially created immutable sampler. This immutable sampler is a combination of the usual information present in an Impeller SamplerDescriptor as well as a custom “conversion” object obtained while importing the Android Hardware Buffer. There is no way to predict what conversion object will be necessary ahead of time as this will depend on the source of the Android Hardware Buffers and is likely different for different video feeds, camera sources, and other Android Hardware Buffer texture sources. To handle this uncertainty, a new pipeline variant with a JIT determined immutable sampler will be hashed and cached before being used in a render pass. The number of pipeline variants created just-in-time will depend on the number of sampler variants used in the render pass to sample from wrapped Image. For instance, specifying a sampler with a different address mode will likely result in a new pipeline variant being created. In most cases however, there will just be one or two additional pipeline variants per application. Impellers sampler diversity is very low with most samplers being the usual NN samplers. It may be possible to preload even this pipeline by trying known conversions. As said previously, there can only be a handful of these conversions. More restrictions on sampling from such images includes being limited to `VK_FILTER_LINEAR` without additional format and extension wrangling and performance penalties. Fixes https://github.com/flutter/flutter/issues/142082. --- ci/licenses_golden/licenses_flutter | 8 + impeller/renderer/backend/vulkan/BUILD.gn | 4 + ...droid_hardware_buffer_texture_source_vk.cc | 463 +++++++++++++----- ...ndroid_hardware_buffer_texture_source_vk.h | 34 +- .../backend/vulkan/capabilities_vk.cc | 201 ++++++-- .../renderer/backend/vulkan/capabilities_vk.h | 99 +++- .../renderer/backend/vulkan/context_vk.cc | 10 +- impeller/renderer/backend/vulkan/context_vk.h | 4 + .../renderer/backend/vulkan/pipeline_vk.cc | 66 ++- .../renderer/backend/vulkan/pipeline_vk.h | 23 +- .../renderer/backend/vulkan/render_pass_vk.cc | 76 ++- .../renderer/backend/vulkan/render_pass_vk.h | 6 +- .../backend/vulkan/sampler_library_vk.cc | 37 +- .../renderer/backend/vulkan/sampler_vk.cc | 111 ++++- impeller/renderer/backend/vulkan/sampler_vk.h | 14 +- .../backend/vulkan/texture_source_vk.cc | 4 + .../backend/vulkan/texture_source_vk.h | 106 +++- .../renderer/backend/vulkan/texture_vk.cc | 23 + impeller/renderer/backend/vulkan/texture_vk.h | 8 +- .../vulkan/yuv_conversion_library_vk.cc | 34 ++ .../vulkan/yuv_conversion_library_vk.h | 65 +++ .../backend/vulkan/yuv_conversion_vk.cc | 118 +++++ .../backend/vulkan/yuv_conversion_vk.h | 109 +++++ .../android/image_external_texture_vk.cc | 11 +- 24 files changed, 1328 insertions(+), 306 deletions(-) create mode 100644 impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc create mode 100644 impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h create mode 100644 impeller/renderer/backend/vulkan/yuv_conversion_vk.cc create mode 100644 impeller/renderer/backend/vulkan/yuv_conversion_vk.h diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter index 04ebe1575b798..3d33f51e5d590 100644 --- a/ci/licenses_golden/licenses_flutter +++ b/ci/licenses_golden/licenses_flutter @@ -35316,6 +35316,10 @@ ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/vertex_descriptor_vk.h ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/vk.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/vma.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/vma.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc + ../../../flutter/LICENSE +ORIGIN: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_vk.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/blit_command.cc + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/blit_command.h + ../../../flutter/LICENSE ORIGIN: ../../../flutter/impeller/renderer/blit_pass.cc + ../../../flutter/LICENSE @@ -38163,6 +38167,10 @@ FILE: ../../../flutter/impeller/renderer/backend/vulkan/vertex_descriptor_vk.h FILE: ../../../flutter/impeller/renderer/backend/vulkan/vk.h FILE: ../../../flutter/impeller/renderer/backend/vulkan/vma.cc FILE: ../../../flutter/impeller/renderer/backend/vulkan/vma.h +FILE: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc +FILE: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h +FILE: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc +FILE: ../../../flutter/impeller/renderer/backend/vulkan/yuv_conversion_vk.h FILE: ../../../flutter/impeller/renderer/blit_command.cc FILE: ../../../flutter/impeller/renderer/blit_command.h FILE: ../../../flutter/impeller/renderer/blit_pass.cc diff --git a/impeller/renderer/backend/vulkan/BUILD.gn b/impeller/renderer/backend/vulkan/BUILD.gn index ac457094136bf..952ae0cb26b5f 100644 --- a/impeller/renderer/backend/vulkan/BUILD.gn +++ b/impeller/renderer/backend/vulkan/BUILD.gn @@ -115,6 +115,10 @@ impeller_component("vulkan") { "vk.h", "vma.cc", "vma.h", + "yuv_conversion_library_vk.cc", + "yuv_conversion_library_vk.h", + "yuv_conversion_vk.cc", + "yuv_conversion_vk.h", ] public_deps = [ diff --git a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc index 1f7619c6d95ae..0ec9e2725ff02 100644 --- a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc +++ b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.cc @@ -6,45 +6,94 @@ #include +#include "impeller/renderer/backend/vulkan/context_vk.h" #include "impeller/renderer/backend/vulkan/texture_source_vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h" #ifdef FML_OS_ANDROID namespace impeller { -namespace { +using AHBProperties = vk::StructureChain< + // For VK_ANDROID_external_memory_android_hardware_buffer + vk::AndroidHardwareBufferPropertiesANDROID, + // For VK_ANDROID_external_memory_android_hardware_buffer + vk::AndroidHardwareBufferFormatPropertiesANDROID>; -bool GetHardwareBufferProperties( +static vk::UniqueImage CreateVKImageWrapperForAndroidHarwareBuffer( const vk::Device& device, - struct AHardwareBuffer* hardware_buffer, - ::impeller::vk::AndroidHardwareBufferPropertiesANDROID* ahb_props, - ::impeller::vk::AndroidHardwareBufferFormatPropertiesANDROID* - ahb_format_props) { - FML_CHECK(ahb_format_props != nullptr); - FML_CHECK(ahb_props != nullptr); - ahb_props->pNext = ahb_format_props; - ::impeller::vk::Result result = - device.getAndroidHardwareBufferPropertiesANDROID(hardware_buffer, - ahb_props); - if (result != impeller::vk::Result::eSuccess) { - return false; - } - return true; -} + const AHBProperties& ahb_props, + const AHardwareBuffer_Desc& ahb_desc) { + const auto& ahb_format = + ahb_props.get(); + + vk::StructureChain + image_chain; + + auto& image_info = image_chain.get(); + + vk::ImageUsageFlags image_usage_flags; + if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) { + image_usage_flags |= vk::ImageUsageFlagBits::eSampled; + } + if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER) { + image_usage_flags |= vk::ImageUsageFlagBits::eColorAttachment; + } + + vk::ImageCreateFlags image_create_flags; + if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) { + image_create_flags |= vk::ImageCreateFlagBits::eProtected; + } + if (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_CUBE_MAP) { + image_create_flags |= vk::ImageCreateFlagBits::eCubeCompatible; + } -vk::ExternalFormatANDROID MakeExternalFormat( - const vk::AndroidHardwareBufferFormatPropertiesANDROID& format_props) { - vk::ExternalFormatANDROID external_format; - external_format.pNext = nullptr; - external_format.externalFormat = 0; - if (format_props.format == vk::Format::eUndefined) { - external_format.externalFormat = format_props.externalFormat; + image_info.imageType = vk::ImageType::e2D; + image_info.format = ahb_format.format; + image_info.extent.width = ahb_desc.width; + image_info.extent.height = ahb_desc.height; + image_info.extent.depth = 1; + image_info.mipLevels = + (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE) + ? ISize{ahb_desc.width, ahb_desc.height}.MipCount() + : 1u; + image_info.arrayLayers = ahb_desc.layers; + image_info.samples = vk::SampleCountFlagBits::e1; + image_info.tiling = vk::ImageTiling::eOptimal; + image_info.usage = image_usage_flags; + image_info.flags = image_create_flags; + image_info.sharingMode = vk::SharingMode::eExclusive; + image_info.initialLayout = vk::ImageLayout::eUndefined; + + image_chain.get().handleTypes = + vk::ExternalMemoryHandleTypeFlagBits::eAndroidHardwareBufferANDROID; + + // If the format isn't natively supported by Vulkan (i.e, be a part of the + // base vkFormat enum), an untyped "external format" must be specified when + // creating the image and the image views. Usually includes YUV formats. + if (ahb_format.format == vk::Format::eUndefined) { + image_chain.get().externalFormat = + ahb_format.externalFormat; + } else { + image_chain.unlink(); } - return external_format; + + auto image = device.createImageUnique(image_chain.get()); + if (image.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not create image for external buffer: " + << vk::to_string(image.result); + return {}; + } + + return std::move(image.value); } // Returns -1 if not found. -int FindMemoryTypeIndex( +static int FindMemoryTypeIndex( const vk::AndroidHardwareBufferPropertiesANDROID& props) { uint32_t memory_type_bits = props.memoryTypeBits; int32_t type_index = -1; @@ -58,132 +107,274 @@ int FindMemoryTypeIndex( return type_index; } -} // namespace - -AndroidHardwareBufferTextureSourceVK::AndroidHardwareBufferTextureSourceVK( - TextureDescriptor desc, +static vk::UniqueDeviceMemory ImportVKDeviceMemoryFromAndroidHarwareBuffer( const vk::Device& device, + const vk::Image& image, struct AHardwareBuffer* hardware_buffer, - const AHardwareBuffer_Desc& hardware_buffer_desc) - : TextureSourceVK(desc), device_(device) { - vk::AndroidHardwareBufferFormatPropertiesANDROID ahb_format_props; - vk::AndroidHardwareBufferPropertiesANDROID ahb_props; - if (!GetHardwareBufferProperties(device, hardware_buffer, &ahb_props, - &ahb_format_props)) { - return; - } - vk::ExternalFormatANDROID external_format = - MakeExternalFormat(ahb_format_props); - vk::ExternalMemoryImageCreateInfo external_memory_image_info; - external_memory_image_info.pNext = &external_format; - external_memory_image_info.handleTypes = - vk::ExternalMemoryHandleTypeFlagBits::eAndroidHardwareBufferANDROID; - const int memory_type_index = FindMemoryTypeIndex(ahb_props); + const AHBProperties& ahb_props) { + const int memory_type_index = FindMemoryTypeIndex(ahb_props.get()); if (memory_type_index < 0) { - FML_LOG(ERROR) << "Could not find memory type."; - return; + VALIDATION_LOG << "Could not find memory type of external image."; + return {}; } - vk::ImageCreateFlags image_create_flags; - vk::ImageUsageFlags image_usage_flags; - if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) { - image_usage_flags |= impeller::vk::ImageUsageFlagBits::eSampled | - impeller::vk::ImageUsageFlagBits::eInputAttachment; - } - if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_GPU_COLOR_OUTPUT) { - image_usage_flags |= impeller::vk::ImageUsageFlagBits::eColorAttachment; - } - if (hardware_buffer_desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) { - image_create_flags |= impeller::vk::ImageCreateFlagBits::eProtected; - } - - vk::ImageCreateInfo image_create_info; - image_create_info.pNext = &external_memory_image_info; - image_create_info.imageType = vk::ImageType::e2D; - image_create_info.format = ahb_format_props.format; - image_create_info.extent.width = hardware_buffer_desc.width; - image_create_info.extent.height = hardware_buffer_desc.height; - image_create_info.extent.depth = 1; - image_create_info.mipLevels = 1; - image_create_info.arrayLayers = 1; - image_create_info.samples = vk::SampleCountFlagBits::e1; - image_create_info.tiling = vk::ImageTiling::eOptimal; - image_create_info.usage = image_usage_flags; - image_create_info.flags = image_create_flags; - image_create_info.sharingMode = vk::SharingMode::eExclusive; - image_create_info.initialLayout = vk::ImageLayout::eUndefined; - - vk::ResultValue maybe_image = - device.createImage(image_create_info); - if (maybe_image.result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "device.createImage failed: " - << static_cast(maybe_image.result); - return; - } - vk::Image image = maybe_image.value; + vk::StructureChain + memory_chain; - vk::ImportAndroidHardwareBufferInfoANDROID ahb_import_info; - ahb_import_info.pNext = nullptr; - ahb_import_info.buffer = hardware_buffer; + auto& mem_alloc_info = memory_chain.get(); + mem_alloc_info.allocationSize = ahb_props.get().allocationSize; + mem_alloc_info.memoryTypeIndex = memory_type_index; - vk::MemoryDedicatedAllocateInfo dedicated_alloc_info; - dedicated_alloc_info.pNext = &ahb_import_info; + auto& dedicated_alloc_info = + memory_chain.get(); dedicated_alloc_info.image = image; - dedicated_alloc_info.buffer = VK_NULL_HANDLE; - vk::MemoryAllocateInfo mem_alloc_info; - mem_alloc_info.pNext = &dedicated_alloc_info; - mem_alloc_info.allocationSize = ahb_props.allocationSize; - mem_alloc_info.memoryTypeIndex = memory_type_index; + auto& ahb_import_info = + memory_chain.get(); + ahb_import_info.buffer = hardware_buffer; - vk::ResultValue allocate_result = - device.allocateMemory(mem_alloc_info); - if (allocate_result.result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "vkAllocateMemory failed : " - << static_cast(allocate_result.result); - device.destroyImage(image); - return; + auto device_memory = device.allocateMemoryUnique(memory_chain.get()); + if (device_memory.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not allocate device memory for external image : " + << vk::to_string(device_memory.result); + return {}; } - vk::DeviceMemory device_memory = allocate_result.value; - // Bind memory to the image object. - vk::Result bind_image_result = - device.bindImageMemory(image, device_memory, 0); - if (bind_image_result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "vkBindImageMemory failed : " - << static_cast(bind_image_result); - device.destroyImage(image); - device.freeMemory(device_memory); - return; + return std::move(device_memory.value); +} + +static std::shared_ptr CreateYUVConversion( + const ContextVK& context, + const AHBProperties& ahb_props) { + YUVConversionDescriptorVK conversion_chain; + + const auto& ahb_format = + ahb_props.get(); + + auto& conversion_info = conversion_chain.get(); + + conversion_info.format = ahb_format.format; + conversion_info.ycbcrModel = ahb_format.suggestedYcbcrModel; + conversion_info.ycbcrRange = ahb_format.suggestedYcbcrRange; + conversion_info.components = ahb_format.samplerYcbcrConversionComponents; + conversion_info.xChromaOffset = ahb_format.suggestedXChromaOffset; + conversion_info.yChromaOffset = ahb_format.suggestedYChromaOffset; + // If the potential format features of the sampler Y′CBCR conversion do not + // support VK_FORMAT_FEATURE_SAMPLED_IMAGE_YCBCR_CONVERSION_LINEAR_FILTER_BIT, + // chromaFilter must not be VK_FILTER_LINEAR. + // + // Since we are not checking, let's just default to a safe value. + conversion_info.chromaFilter = vk::Filter::eNearest; + conversion_info.forceExplicitReconstruction = false; + + if (conversion_info.format == vk::Format::eUndefined) { + auto& external_format = conversion_chain.get(); + external_format.externalFormat = ahb_format.externalFormat; + } else { + conversion_chain.unlink(); } - image_ = image; - device_memory_ = device_memory; - // Create image view. - vk::ImageViewCreateInfo view_info; - view_info.image = image_; + return context.GetYUVConversionLibrary()->GetConversion(conversion_chain); +} + +static vk::UniqueImageView CreateVKImageView( + const vk::Device& device, + const vk::Image& image, + const vk::SamplerYcbcrConversion& yuv_conversion, + const AHBProperties& ahb_props, + const AHardwareBuffer_Desc& ahb_desc) { + const auto& ahb_format = + ahb_props.get(); + + vk::StructureChain + view_chain; + + auto& view_info = view_chain.get(); + + view_info.image = image; view_info.viewType = vk::ImageViewType::e2D; - view_info.format = ToVKImageFormat(desc.format); + view_info.format = ahb_format.format; view_info.subresourceRange.aspectMask = vk::ImageAspectFlagBits::eColor; view_info.subresourceRange.baseMipLevel = 0u; view_info.subresourceRange.baseArrayLayer = 0u; - view_info.subresourceRange.levelCount = desc.mip_count; - view_info.subresourceRange.layerCount = ToArrayLayerCount(desc.type); - auto [view_result, view] = device.createImageViewUnique(view_info); - if (view_result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "createImageViewUnique failed : " - << static_cast(view_result); + view_info.subresourceRange.levelCount = + (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE) + ? ISize{ahb_desc.width, ahb_desc.height}.MipCount() + : 1u; + view_info.subresourceRange.layerCount = ahb_desc.layers; + + // We need a custom YUV conversion only if we don't recognize the format. + if (view_info.format == vk::Format::eUndefined) { + view_chain.get().conversion = + yuv_conversion; + } else { + view_chain.unlink(); + } + + auto image_view = device.createImageViewUnique(view_info); + if (image_view.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not create external image view: " + << vk::to_string(image_view.result); + return {}; + } + + return std::move(image_view.value); +} + +static PixelFormat ToPixelFormat(AHardwareBuffer_Format format) { + switch (format) { + case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM: + return PixelFormat::kR8G8B8A8UNormInt; + case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT: + return PixelFormat::kR16G16B16A16Float; + case AHARDWAREBUFFER_FORMAT_D24_UNORM_S8_UINT: + return PixelFormat::kD24UnormS8Uint; + case AHARDWAREBUFFER_FORMAT_D32_FLOAT_S8_UINT: + return PixelFormat::kD32FloatS8UInt; + case AHARDWAREBUFFER_FORMAT_S8_UINT: + return PixelFormat::kS8UInt; + case AHARDWAREBUFFER_FORMAT_R8_UNORM: + return PixelFormat::kR8UNormInt; + case AHARDWAREBUFFER_FORMAT_R10G10B10A10_UNORM: + case AHARDWAREBUFFER_FORMAT_R16G16_UINT: + case AHARDWAREBUFFER_FORMAT_D32_FLOAT: + case AHARDWAREBUFFER_FORMAT_R16_UINT: + case AHARDWAREBUFFER_FORMAT_D24_UNORM: + case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420: + case AHARDWAREBUFFER_FORMAT_YCbCr_P010: + case AHARDWAREBUFFER_FORMAT_BLOB: + case AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM: + case AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM: + case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM: + case AHARDWAREBUFFER_FORMAT_D16_UNORM: + case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM: + // Not understood by the rest of Impeller. Use a placeholder but create + // the native image and image views using the right external format. + break; + } + return PixelFormat::kR8G8B8A8UNormInt; +} + +static TextureType ToTextureType(const AHardwareBuffer_Desc& ahb_desc) { + if (ahb_desc.layers == 1u) { + return TextureType::kTexture2D; + } + if (ahb_desc.layers % 6u == 0 && + (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_CUBE_MAP)) { + return TextureType::kTextureCube; + } + // Our texture types seem to understand external OES textures. Should these be + // wired up instead? + return TextureType::kTexture2D; +} + +static TextureDescriptor ToTextureDescriptor( + const AHardwareBuffer_Desc& ahb_desc) { + const auto ahb_size = ISize{ahb_desc.width, ahb_desc.height}; + TextureDescriptor desc; + // We are not going to touch hardware buffers on the CPU or use them as + // transient attachments. Just treat them as device private. + desc.storage_mode = StorageMode::kDevicePrivate; + desc.format = + ToPixelFormat(static_cast(ahb_desc.format)); + desc.size = ahb_size; + desc.type = ToTextureType(ahb_desc); + desc.sample_count = SampleCount::kCount1; + desc.compression_type = CompressionType::kLossless; + desc.mip_count = (ahb_desc.usage & AHARDWAREBUFFER_USAGE_GPU_MIPMAP_COMPLETE) + ? ahb_size.MipCount() + : 1u; + return desc; +} + +AndroidHardwareBufferTextureSourceVK::AndroidHardwareBufferTextureSourceVK( + const std::shared_ptr& context, + struct AHardwareBuffer* ahb, + const AHardwareBuffer_Desc& ahb_desc) + : TextureSourceVK(ToTextureDescriptor(ahb_desc)) { + if (!context) { + VALIDATION_LOG << "Invalid context."; + return; + } + + const auto& device = context->GetDevice(); + + AHBProperties ahb_props; + + if (device.getAndroidHardwareBufferPropertiesANDROID(ahb, &ahb_props.get()) != + vk::Result::eSuccess) { + VALIDATION_LOG << "Could not determine properties of the Android hardware " + "buffer."; return; } - image_view_ = std::move(view); + + const auto& ahb_format = + ahb_props.get(); + + // Create an image to refer to our external image. + auto image = + CreateVKImageWrapperForAndroidHarwareBuffer(device, ahb_props, ahb_desc); + if (!image) { + return; + } + + // Create a device memory allocation to refer to our external image. + auto device_memory = ImportVKDeviceMemoryFromAndroidHarwareBuffer( + device, image.get(), ahb, ahb_props); + if (!device_memory) { + return; + } + + // Bind the image to the image memory. + if (auto result = device.bindImageMemory(image.get(), device_memory.get(), 0); + result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not bind external device memory to image : " + << vk::to_string(result); + return; + } + + // Figure out how to perform YUV conversions. + auto yuv_conversion = CreateYUVConversion(*context, ahb_props); + if (!yuv_conversion || !yuv_conversion->IsValid()) { + return; + } + + // Create image view for the newly created image. + auto image_view = CreateVKImageView(device, // + image.get(), // + yuv_conversion->GetConversion(), // + ahb_props, // + ahb_desc // + ); + if (!image_view) { + return; + } + + needs_yuv_conversion_ = ahb_format.format == vk::Format::eUndefined; + device_memory_ = std::move(device_memory); + image_ = std::move(image); + yuv_conversion_ = std::move(yuv_conversion); + image_view_ = std::move(image_view); + +#ifdef IMPELLER_DEBUG + context->SetDebugName(device_memory_.get(), "AHB Device Memory"); + context->SetDebugName(image_.get(), "AHB Image"); + context->SetDebugName(yuv_conversion_->GetConversion(), "AHB YUV Conversion"); + context->SetDebugName(image_view_.get(), "AHB ImageView"); +#endif // IMPELLER_DEBUG + is_valid_ = true; } // |TextureSourceVK| -AndroidHardwareBufferTextureSourceVK::~AndroidHardwareBufferTextureSourceVK() { - device_.destroyImage(image_); - device_.freeMemory(device_memory_); -} +AndroidHardwareBufferTextureSourceVK::~AndroidHardwareBufferTextureSourceVK() = + default; bool AndroidHardwareBufferTextureSourceVK::IsValid() const { return is_valid_; @@ -191,23 +382,31 @@ bool AndroidHardwareBufferTextureSourceVK::IsValid() const { // |TextureSourceVK| vk::Image AndroidHardwareBufferTextureSourceVK::GetImage() const { - FML_CHECK(IsValid()); - return image_; + return image_.get(); } // |TextureSourceVK| vk::ImageView AndroidHardwareBufferTextureSourceVK::GetImageView() const { - FML_CHECK(IsValid()); return image_view_.get(); } // |TextureSourceVK| vk::ImageView AndroidHardwareBufferTextureSourceVK::GetRenderTargetView() const { - FML_CHECK(IsValid()); return image_view_.get(); } +// |TextureSourceVK| +bool AndroidHardwareBufferTextureSourceVK::IsSwapchainImage() const { + return false; +} + +// |TextureSourceVK| +std::shared_ptr +AndroidHardwareBufferTextureSourceVK::GetYUVConversion() const { + return needs_yuv_conversion_ ? yuv_conversion_ : nullptr; +} + } // namespace impeller #endif diff --git a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h index 19cfab1a2b765..724308cb2b6fb 100644 --- a/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h +++ b/impeller/renderer/backend/vulkan/android_hardware_buffer_texture_source_vk.h @@ -15,17 +15,31 @@ #include "impeller/renderer/backend/vulkan/formats_vk.h" #include "impeller/renderer/backend/vulkan/texture_source_vk.h" #include "impeller/renderer/backend/vulkan/vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" #include #include namespace impeller { +class ContextVK; + +//------------------------------------------------------------------------------ +/// @brief A texture source that wraps an instance of AHardwareBuffer. +/// +/// The formats and conversions supported by Android Hardware +/// Buffers are a superset of those supported by Impeller (and +/// Vulkan for that matter). Impeller and Vulkan descriptors +/// obtained from the these texture sources are advisory and it +/// usually isn't possible to create copies of images and image +/// views held in these texture sources using the inferred +/// descriptors. The objects are meant to be used directly (either +/// as render targets or sources for sampling), not copied. +/// class AndroidHardwareBufferTextureSourceVK final : public TextureSourceVK { public: AndroidHardwareBufferTextureSourceVK( - TextureDescriptor desc, - const vk::Device& device, + const std::shared_ptr& context, struct AHardwareBuffer* hardware_buffer, const AHardwareBuffer_Desc& hardware_buffer_desc); @@ -43,14 +57,18 @@ class AndroidHardwareBufferTextureSourceVK final : public TextureSourceVK { bool IsValid() const; - bool IsSwapchainImage() const override { return false; } + // |TextureSourceVK| + bool IsSwapchainImage() const override; - private: - const vk::Device& device_; - vk::Image image_ = VK_NULL_HANDLE; - vk::UniqueImageView image_view_ = {}; - vk::DeviceMemory device_memory_ = VK_NULL_HANDLE; + // |TextureSourceVK| + std::shared_ptr GetYUVConversion() const override; + private: + vk::UniqueDeviceMemory device_memory_; + vk::UniqueImage image_; + vk::UniqueImageView image_view_; + std::shared_ptr yuv_conversion_; + bool needs_yuv_conversion_ = false; bool is_valid_ = false; AndroidHardwareBufferTextureSourceVK( diff --git a/impeller/renderer/backend/vulkan/capabilities_vk.cc b/impeller/renderer/backend/vulkan/capabilities_vk.cc index ab5214747b5b5..b5ad0aba23f93 100644 --- a/impeller/renderer/backend/vulkan/capabilities_vk.cc +++ b/impeller/renderer/backend/vulkan/capabilities_vk.cc @@ -9,7 +9,6 @@ #include "impeller/base/validation.h" #include "impeller/core/formats.h" #include "impeller/renderer/backend/vulkan/vk.h" -#include "vulkan/vulkan_core.h" namespace impeller { @@ -153,25 +152,58 @@ CapabilitiesVK::GetEnabledInstanceExtensions() const { return required; } -static const char* GetDeviceExtensionName(OptionalDeviceExtensionVK ext) { +static const char* GetExtensionName(RequiredCommonDeviceExtensionVK ext) { + switch (ext) { + case RequiredCommonDeviceExtensionVK::kKHRSwapchain: + return VK_KHR_SWAPCHAIN_EXTENSION_NAME; + case RequiredCommonDeviceExtensionVK::kLast: + return "Unknown"; + } + FML_UNREACHABLE(); +} + +static const char* GetExtensionName(RequiredAndroidDeviceExtensionVK ext) { + switch (ext) { + case RequiredAndroidDeviceExtensionVK:: + kANDROIDExternalMemoryAndroidHardwareBuffer: + return "VK_ANDROID_external_memory_android_hardware_buffer"; + case RequiredAndroidDeviceExtensionVK::kKHRSamplerYcbcrConversion: + return VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME; + case RequiredAndroidDeviceExtensionVK::kKHRExternalMemory: + return VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME; + case RequiredAndroidDeviceExtensionVK::kEXTQueueFamilyForeign: + return VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME; + case RequiredAndroidDeviceExtensionVK::kKHRDedicatedAllocation: + return VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME; + case RequiredAndroidDeviceExtensionVK::kLast: + return "Unknown"; + } + FML_UNREACHABLE(); +} + +static const char* GetExtensionName(OptionalDeviceExtensionVK ext) { switch (ext) { case OptionalDeviceExtensionVK::kEXTPipelineCreationFeedback: return VK_EXT_PIPELINE_CREATION_FEEDBACK_EXTENSION_NAME; + case OptionalDeviceExtensionVK::kVKKHRPortabilitySubset: + return "VK_KHR_portability_subset"; case OptionalDeviceExtensionVK::kLast: return "Unknown"; } - return "Unknown"; + FML_UNREACHABLE(); } -static void IterateOptionalDeviceExtensions( - const std::function& it) { +template +static bool IterateExtensions(const std::function& it) { if (!it) { - return; + return false; } - for (size_t i = 0; - i < static_cast(OptionalDeviceExtensionVK::kLast); i++) { - it(static_cast(i)); + for (size_t i = 0; i < static_cast(T::kLast); i++) { + if (!it(static_cast(i))) { + return false; + } } + return true; } static std::optional> GetSupportedDeviceExtensions( @@ -200,36 +232,49 @@ CapabilitiesVK::GetEnabledDeviceExtensions( std::vector enabled; - if (exts->find("VK_KHR_swapchain") == exts->end()) { - VALIDATION_LOG << "Device does not support the swapchain extension."; - return std::nullopt; - } - enabled.push_back("VK_KHR_swapchain"); - - // Required for non-conformant implementations like MoltenVK. - if (exts->find("VK_KHR_portability_subset") != exts->end()) { - enabled.push_back("VK_KHR_portability_subset"); - } + auto for_each_common_extension = [&](RequiredCommonDeviceExtensionVK ext) { + auto name = GetExtensionName(ext); + if (exts->find(name) == exts->end()) { + VALIDATION_LOG << "Device does not support required extension: " << name; + return false; + } + enabled.push_back(name); + return true; + }; + auto for_each_android_extension = [&](RequiredAndroidDeviceExtensionVK ext) { #ifdef FML_OS_ANDROID - if (exts->find("VK_ANDROID_external_memory_android_hardware_buffer") == - exts->end()) { - VALIDATION_LOG - << "Device does not support " - "VK_ANDROID_external_memory_android_hardware_buffer extension."; - return std::nullopt; - } - enabled.push_back("VK_ANDROID_external_memory_android_hardware_buffer"); - enabled.push_back("VK_EXT_queue_family_foreign"); -#endif + auto name = GetExtensionName(ext); + if (exts->find(name) == exts->end()) { + VALIDATION_LOG << "Device does not support required Android extension: " + << name; + return false; + } + enabled.push_back(name); +#endif // FML_OS_ANDROID + return true; + }; - // Enable all optional extensions if the device supports it. - IterateOptionalDeviceExtensions([&](auto ext) { - auto ext_name = GetDeviceExtensionName(ext); - if (exts->find(ext_name) != exts->end()) { - enabled.push_back(ext_name); + auto for_each_optional_extension = [&](OptionalDeviceExtensionVK ext) { + auto name = GetExtensionName(ext); + if (exts->find(name) != exts->end()) { + enabled.push_back(name); } - }); + return true; + }; + + const auto iterate_extensions = + IterateExtensions( + for_each_common_extension) && + IterateExtensions( + for_each_android_extension) && + IterateExtensions(for_each_optional_extension); + + if (!iterate_extensions) { + VALIDATION_LOG << "Device not suitable since required extensions are not " + "supported."; + return std::nullopt; + } return enabled; } @@ -282,7 +327,14 @@ static bool HasRequiredQueues(const vk::PhysicalDevice& physical_device) { vk::QueueFlagBits::eTransfer)); } -std::optional +template +static bool IsExtensionInList(const std::vector& list, + ExtensionEnum ext) { + const std::string name = GetExtensionName(ext); + return std::find(list.begin(), list.end(), name) != list.end(); +} + +std::optional CapabilitiesVK::GetEnabledDeviceFeatures( const vk::PhysicalDevice& device) const { if (!PhysicalDeviceSupportsRequiredFormats(device)) { @@ -300,20 +352,41 @@ CapabilitiesVK::GetEnabledDeviceFeatures( return std::nullopt; } - if (!GetEnabledDeviceExtensions(device).has_value()) { + const auto enabled_extensions = GetEnabledDeviceExtensions(device); + if (!enabled_extensions.has_value()) { VALIDATION_LOG << "Device doesn't support the required queues."; return std::nullopt; } - const auto device_features = device.getFeatures(); + PhysicalDeviceFeatures supported_chain; + device.getFeatures2(&supported_chain.get()); - vk::PhysicalDeviceFeatures required; + PhysicalDeviceFeatures required_chain; - // We require this for enabling wireframes in the playground. But its not - // necessarily a big deal if we don't have this feature. - required.fillModeNonSolid = device_features.fillModeNonSolid; + // Base features. + { + auto& required = required_chain.get().features; + const auto& supported = supported_chain.get().features; - return required; + // We require this for enabling wireframes in the playground. But its not + // necessarily a big deal if we don't have this feature. + required.fillModeNonSolid = supported.fillModeNonSolid; + } + // VK_KHR_sampler_ycbcr_conversion features. + if (IsExtensionInList( + enabled_extensions.value(), + RequiredAndroidDeviceExtensionVK::kKHRSamplerYcbcrConversion)) { + auto& required = + required_chain + .get(); + const auto& supported = + supported_chain + .get(); + + required.samplerYcbcrConversion = supported.samplerYcbcrConversion; + } + + return required_chain; } bool CapabilitiesVK::HasLayer(const std::string& layer) const { @@ -386,16 +459,33 @@ bool CapabilitiesVK::SetPhysicalDevice(const vk::PhysicalDevice& device) { // Determine the optional device extensions this physical device supports. { + required_common_device_extensions_.clear(); + required_android_device_extensions_.clear(); optional_device_extensions_.clear(); auto exts = GetSupportedDeviceExtensions(device); if (!exts.has_value()) { return false; } - IterateOptionalDeviceExtensions([&](auto ext) { - auto ext_name = GetDeviceExtensionName(ext); + IterateExtensions([&](auto ext) -> bool { + auto ext_name = GetExtensionName(ext); + if (exts->find(ext_name) != exts->end()) { + required_common_device_extensions_.insert(ext); + } + return true; + }); + IterateExtensions([&](auto ext) -> bool { + auto ext_name = GetExtensionName(ext); + if (exts->find(ext_name) != exts->end()) { + required_android_device_extensions_.insert(ext); + } + return true; + }); + IterateExtensions([&](auto ext) -> bool { + auto ext_name = GetExtensionName(ext); if (exts->find(ext_name) != exts->end()) { optional_device_extensions_.insert(ext); } + return true; }); } @@ -478,14 +568,23 @@ CapabilitiesVK::GetPhysicalDeviceProperties() const { return device_properties_; } -bool CapabilitiesVK::HasOptionalDeviceExtension( - OptionalDeviceExtensionVK extension) const { - return optional_device_extensions_.find(extension) != - optional_device_extensions_.end(); -} - PixelFormat CapabilitiesVK::GetDefaultGlyphAtlasFormat() const { return PixelFormat::kR8UNormInt; } +bool CapabilitiesVK::HasExtension(RequiredCommonDeviceExtensionVK ext) const { + return required_common_device_extensions_.find(ext) != + required_common_device_extensions_.end(); +} + +bool CapabilitiesVK::HasExtension(RequiredAndroidDeviceExtensionVK ext) const { + return required_android_device_extensions_.find(ext) != + required_android_device_extensions_.end(); +} + +bool CapabilitiesVK::HasExtension(OptionalDeviceExtensionVK ext) const { + return optional_device_extensions_.find(ext) != + optional_device_extensions_.end(); +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/capabilities_vk.h b/impeller/renderer/backend/vulkan/capabilities_vk.h index d3f5d70fbffee..f06732b8b206e 100644 --- a/impeller/renderer/backend/vulkan/capabilities_vk.h +++ b/impeller/renderer/backend/vulkan/capabilities_vk.h @@ -19,9 +19,91 @@ namespace impeller { class ContextVK; +//------------------------------------------------------------------------------ +/// @brief A device extension available on all platforms. Without the +/// presence of these extensions, context creation will fail. +/// +enum class RequiredCommonDeviceExtensionVK : uint32_t { + //---------------------------------------------------------------------------- + /// For displaying content in the window system. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_swapchain.html + /// + kKHRSwapchain, + + kLast, +}; + +//------------------------------------------------------------------------------ +/// @brief A device extension available on all Android platforms. Without +/// the presence of these extensions on Android, context creation +/// will fail. +/// +/// Platform agnostic code can still check if these Android +/// extensions are present. +/// +enum class RequiredAndroidDeviceExtensionVK : uint32_t { + //---------------------------------------------------------------------------- + /// For importing hardware buffers used in external texture composition. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_ANDROID_external_memory_android_hardware_buffer.html + /// + kANDROIDExternalMemoryAndroidHardwareBuffer, + + //---------------------------------------------------------------------------- + /// Dependency of kANDROIDExternalMemoryAndroidHardwareBuffer. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_sampler_ycbcr_conversion.html + /// + kKHRSamplerYcbcrConversion, + + //---------------------------------------------------------------------------- + /// Dependency of kANDROIDExternalMemoryAndroidHardwareBuffer. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_external_memory.html + /// + kKHRExternalMemory, + + //---------------------------------------------------------------------------- + /// Dependency of kANDROIDExternalMemoryAndroidHardwareBuffer. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_EXT_queue_family_foreign.html + /// + kEXTQueueFamilyForeign, + + //---------------------------------------------------------------------------- + /// Dependency of kANDROIDExternalMemoryAndroidHardwareBuffer. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_dedicated_allocation.html + /// + kKHRDedicatedAllocation, + + kLast, +}; + +//------------------------------------------------------------------------------ +/// @brief A device extension enabled if available. Subsystems cannot +/// assume availability and must check if these extensions are +/// available. +/// +/// @see `CapabilitiesVK::HasExtension`. +/// enum class OptionalDeviceExtensionVK : uint32_t { - // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_EXT_pipeline_creation_feedback.html + //---------------------------------------------------------------------------- + /// To instrument and profile PSO creation. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_EXT_pipeline_creation_feedback.html + /// kEXTPipelineCreationFeedback, + + //---------------------------------------------------------------------------- + /// To enable context creation on MoltenVK. A non-conformant Vulkan + /// implementation. + /// + /// https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/VK_KHR_portability_subset.html + /// + kVKKHRPortabilitySubset, + kLast, }; @@ -39,7 +121,11 @@ class CapabilitiesVK final : public Capabilities, bool AreValidationsEnabled() const; - bool HasOptionalDeviceExtension(OptionalDeviceExtensionVK extension) const; + bool HasExtension(RequiredCommonDeviceExtensionVK ext) const; + + bool HasExtension(RequiredAndroidDeviceExtensionVK ext) const; + + bool HasExtension(OptionalDeviceExtensionVK ext) const; std::optional> GetEnabledLayers() const; @@ -48,7 +134,11 @@ class CapabilitiesVK final : public Capabilities, std::optional> GetEnabledDeviceExtensions( const vk::PhysicalDevice& physical_device) const; - std::optional GetEnabledDeviceFeatures( + using PhysicalDeviceFeatures = + vk::StructureChain; + + std::optional GetEnabledDeviceFeatures( const vk::PhysicalDevice& physical_device) const; [[nodiscard]] bool SetPhysicalDevice( @@ -106,6 +196,9 @@ class CapabilitiesVK final : public Capabilities, private: bool validations_enabled_ = false; std::map> exts_; + std::set required_common_device_extensions_; + std::set + required_android_device_extensions_; std::set optional_device_extensions_; mutable PixelFormat default_color_format_ = PixelFormat::kUnknown; PixelFormat default_stencil_format_ = PixelFormat::kUnknown; diff --git a/impeller/renderer/backend/vulkan/context_vk.cc b/impeller/renderer/backend/vulkan/context_vk.cc index d32ab4e525b33..9b9caa9cd2d24 100644 --- a/impeller/renderer/backend/vulkan/context_vk.cc +++ b/impeller/renderer/backend/vulkan/context_vk.cc @@ -35,6 +35,7 @@ #include "impeller/renderer/backend/vulkan/gpu_tracer_vk.h" #include "impeller/renderer/backend/vulkan/resource_manager_vk.h" #include "impeller/renderer/backend/vulkan/surface_context_vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h" #include "impeller/renderer/capabilities.h" VULKAN_HPP_DEFAULT_DISPATCH_LOADER_DYNAMIC_STORAGE @@ -323,9 +324,9 @@ void ContextVK::Setup(Settings settings) { vk::DeviceCreateInfo device_info; + device_info.setPNext(&enabled_features.value().get()); device_info.setQueueCreateInfos(queue_create_infos); device_info.setPEnabledExtensionNames(enabled_device_extensions_c); - device_info.setPEnabledFeatures(&enabled_features.value()); // Device layers are deprecated and ignored. { @@ -443,6 +444,8 @@ void ContextVK::Setup(Settings settings) { shader_library_ = std::move(shader_library); sampler_library_ = std::move(sampler_library); pipeline_library_ = std::move(pipeline_library); + yuv_conversion_library_ = std::shared_ptr( + new YUVConversionLibraryVK(device_holder_)); queues_ = std::move(queues); device_capabilities_ = std::move(caps); fence_waiter_ = std::move(fence_waiter); @@ -613,4 +616,9 @@ void ContextVK::InitializeCommonlyUsedShadersIfNeeded() const { auto pass = builder.Build(GetDevice()); } +const std::shared_ptr& +ContextVK::GetYUVConversionLibrary() const { + return yuv_conversion_library_; +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/context_vk.h b/impeller/renderer/backend/vulkan/context_vk.h index 820d9a9c1ed6f..4650ad39e53f9 100644 --- a/impeller/renderer/backend/vulkan/context_vk.h +++ b/impeller/renderer/backend/vulkan/context_vk.h @@ -93,6 +93,9 @@ class ContextVK final : public Context, // |Context| const std::shared_ptr& GetCapabilities() const override; + const std::shared_ptr& GetYUVConversionLibrary() + const; + // |Context| void Shutdown() override; @@ -180,6 +183,7 @@ class ContextVK final : public Context, std::shared_ptr shader_library_; std::shared_ptr sampler_library_; std::shared_ptr pipeline_library_; + std::shared_ptr yuv_conversion_library_; QueuesVK queues_; std::shared_ptr device_capabilities_; std::shared_ptr fence_waiter_; diff --git a/impeller/renderer/backend/vulkan/pipeline_vk.cc b/impeller/renderer/backend/vulkan/pipeline_vk.cc index 13c13ab337a38..eb503e4b83911 100644 --- a/impeller/renderer/backend/vulkan/pipeline_vk.cc +++ b/impeller/renderer/backend/vulkan/pipeline_vk.cc @@ -11,6 +11,7 @@ #include "impeller/renderer/backend/vulkan/context_vk.h" #include "impeller/renderer/backend/vulkan/formats_vk.h" #include "impeller/renderer/backend/vulkan/render_pass_builder_vk.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" #include "impeller/renderer/backend/vulkan/shader_function_vk.h" #include "impeller/renderer/backend/vulkan/vertex_descriptor_vk.h" @@ -166,7 +167,8 @@ static vk::UniqueRenderPass CreateCompatRenderPassForPipeline( std::unique_ptr PipelineVK::Create( const PipelineDescriptor& desc, const std::shared_ptr& device_holder, - const std::weak_ptr& weak_library) { + const std::weak_ptr& weak_library, + std::shared_ptr immutable_sampler) { TRACE_EVENT0("flutter", "PipelineVK::Create"); auto library = weak_library.lock(); @@ -183,9 +185,8 @@ std::unique_ptr PipelineVK::Create( const auto* caps = pso_cache->GetCapabilities(); - const auto supports_pipeline_creation_feedback = - caps->HasOptionalDeviceExtension( - OptionalDeviceExtensionVK::kEXTPipelineCreationFeedback); + const auto supports_pipeline_creation_feedback = caps->HasExtension( + OptionalDeviceExtensionVK::kEXTPipelineCreationFeedback); if (!supports_pipeline_creation_feedback) { chain.unlink(); } @@ -353,19 +354,36 @@ std::unique_ptr PipelineVK::Create( //---------------------------------------------------------------------------- /// Pipeline Layout a.k.a the descriptor sets and uniforms. /// - std::vector desc_bindings; + std::vector set_bindings; + + vk::Sampler vk_immutable_sampler = + immutable_sampler ? immutable_sampler->GetSampler() : VK_NULL_HANDLE; for (auto layout : desc.GetVertexDescriptor()->GetDescriptorSetLayouts()) { - auto vk_desc_layout = ToVKDescriptorSetLayoutBinding(layout); - desc_bindings.push_back(vk_desc_layout); + vk::DescriptorSetLayoutBinding set_binding; + set_binding.binding = layout.binding; + set_binding.descriptorCount = 1u; + set_binding.descriptorType = ToVKDescriptorType(layout.descriptor_type); + set_binding.stageFlags = ToVkShaderStage(layout.shader_stage); + // TODO(143719): This specifies the immutable sampler for all sampled + // images. This is incorrect. In cases where the shader samples from the + // multiple images, there is currently no way to tell which sampler needs to + // be immutable and which one needs a binding set in the render pass. Expect + // errors if the shader has more than on sampled image. The sampling from + // the one that is expected to be non-immutable will be incorrect. + if (vk_immutable_sampler && + layout.descriptor_type == DescriptorType::kSampledImage) { + set_binding.setImmutableSamplers(vk_immutable_sampler); + } + set_bindings.push_back(set_binding); } - vk::DescriptorSetLayoutCreateInfo descs_layout_info; - descs_layout_info.setBindings(desc_bindings); + vk::DescriptorSetLayoutCreateInfo desc_set_layout_info; + desc_set_layout_info.setBindings(set_bindings); auto [descs_result, descs_layout] = device_holder->GetDevice().createDescriptorSetLayoutUnique( - descs_layout_info); + desc_set_layout_info); if (descs_result != vk::Result::eSuccess) { VALIDATION_LOG << "unable to create uniform descriptors"; return nullptr; @@ -434,7 +452,8 @@ std::unique_ptr PipelineVK::Create( std::move(pipeline), // std::move(render_pass), // std::move(pipeline_layout.value), // - std::move(descs_layout) // + std::move(descs_layout), // + std::move(immutable_sampler) // )); if (!pipeline_vk->IsValid()) { VALIDATION_LOG << "Could not create a valid pipeline."; @@ -449,13 +468,15 @@ PipelineVK::PipelineVK(std::weak_ptr device_holder, vk::UniquePipeline pipeline, vk::UniqueRenderPass render_pass, vk::UniquePipelineLayout layout, - vk::UniqueDescriptorSetLayout descriptor_set_layout) + vk::UniqueDescriptorSetLayout descriptor_set_layout, + std::shared_ptr immutable_sampler) : Pipeline(std::move(library), desc), device_holder_(std::move(device_holder)), pipeline_(std::move(pipeline)), render_pass_(std::move(render_pass)), layout_(std::move(layout)), - descriptor_set_layout_(std::move(descriptor_set_layout)) { + descriptor_set_layout_(std::move(descriptor_set_layout)), + immutable_sampler_(std::move(immutable_sampler)) { is_valid_ = pipeline_ && render_pass_ && layout_ && descriptor_set_layout_; } @@ -484,4 +505,23 @@ const vk::DescriptorSetLayout& PipelineVK::GetDescriptorSetLayout() const { return *descriptor_set_layout_; } +std::shared_ptr PipelineVK::CreateVariantForImmutableSamplers( + const std::shared_ptr& immutable_sampler) const { + if (!immutable_sampler) { + return nullptr; + } + auto cache_key = ImmutableSamplerKeyVK{*immutable_sampler}; + Lock lock(immutable_sampler_variants_mutex_); + auto found = immutable_sampler_variants_.find(cache_key); + if (found != immutable_sampler_variants_.end()) { + return found->second; + } + auto device_holder = device_holder_.lock(); + if (!device_holder) { + return nullptr; + } + return (immutable_sampler_variants_[cache_key] = + Create(desc_, device_holder, library_, immutable_sampler)); +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/pipeline_vk.h b/impeller/renderer/backend/vulkan/pipeline_vk.h index dfe75cfdafcc1..5040582d23328 100644 --- a/impeller/renderer/backend/vulkan/pipeline_vk.h +++ b/impeller/renderer/backend/vulkan/pipeline_vk.h @@ -10,10 +10,13 @@ #include "impeller/base/backend_cast.h" #include "impeller/base/thread.h" +#include "impeller/core/texture.h" #include "impeller/renderer/backend/vulkan/device_holder.h" #include "impeller/renderer/backend/vulkan/formats_vk.h" #include "impeller/renderer/backend/vulkan/pipeline_cache_vk.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" #include "impeller/renderer/backend/vulkan/vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" #include "impeller/renderer/pipeline.h" namespace impeller { @@ -29,7 +32,8 @@ class PipelineVK final static std::unique_ptr Create( const PipelineDescriptor& desc, const std::shared_ptr& device_holder, - const std::weak_ptr& weak_library); + const std::weak_ptr& weak_library, + std::shared_ptr immutable_sampler = {}); // |Pipeline| ~PipelineVK() override; @@ -40,15 +44,27 @@ class PipelineVK final const vk::DescriptorSetLayout& GetDescriptorSetLayout() const; + std::shared_ptr CreateVariantForImmutableSamplers( + const std::shared_ptr& immutable_sampler) const; + private: friend class PipelineLibraryVK; + using ImmutableSamplerVariants = + std::unordered_map, + ComparableHash, + ComparableEqual>; + std::weak_ptr device_holder_; vk::UniquePipeline pipeline_; vk::UniqueRenderPass render_pass_; vk::UniquePipelineLayout layout_; vk::UniqueDescriptorSetLayout descriptor_set_layout_; - + std::shared_ptr immutable_sampler_; + mutable Mutex immutable_sampler_variants_mutex_; + mutable ImmutableSamplerVariants immutable_sampler_variants_ IPLR_GUARDED_BY( + immutable_sampler_variants_mutex_); bool is_valid_ = false; PipelineVK(std::weak_ptr device_holder, @@ -57,7 +73,8 @@ class PipelineVK final vk::UniquePipeline pipeline, vk::UniqueRenderPass render_pass, vk::UniquePipelineLayout layout, - vk::UniqueDescriptorSetLayout descriptor_set_layout); + vk::UniqueDescriptorSetLayout descriptor_set_layout, + std::shared_ptr immutable_sampler); // |Pipeline| bool IsValid() const override; diff --git a/impeller/renderer/backend/vulkan/render_pass_vk.cc b/impeller/renderer/backend/vulkan/render_pass_vk.cc index 3149687947696..bc2ed6c5115d7 100644 --- a/impeller/renderer/backend/vulkan/render_pass_vk.cc +++ b/impeller/renderer/backend/vulkan/render_pass_vk.cc @@ -298,26 +298,18 @@ SharedHandleVK RenderPassVK::CreateVKFramebuffer( // |RenderPass| void RenderPassVK::SetPipeline( const std::shared_ptr>& pipeline) { - PipelineVK& pipeline_vk = PipelineVK::Cast(*pipeline); + pipeline_ = pipeline; - auto descriptor_result = - command_buffer_->GetEncoder()->AllocateDescriptorSets( - pipeline_vk.GetDescriptorSetLayout(), ContextVK::Cast(*context_)); - if (!descriptor_result.ok()) { + if (!pipeline_) { return; } - pipeline_valid_ = true; - descriptor_set_ = descriptor_result.value(); - pipeline_layout_ = pipeline_vk.GetPipelineLayout(); - command_buffer_vk_.bindPipeline(vk::PipelineBindPoint::eGraphics, - pipeline_vk.GetPipeline()); pipeline_uses_input_attachments_ = - pipeline->GetDescriptor().GetVertexDescriptor()->UsesInputAttacments(); + pipeline_->GetDescriptor().GetVertexDescriptor()->UsesInputAttacments(); if (pipeline_uses_input_attachments_) { if (bound_image_offset_ >= kMaxBindings) { - pipeline_valid_ = false; + pipeline_ = nullptr; return; } vk::DescriptorImageInfo image_info; @@ -433,14 +425,57 @@ bool RenderPassVK::SetVertexBuffer(VertexBuffer buffer) { // |RenderPass| fml::Status RenderPassVK::Draw() { - if (!pipeline_valid_) { + if (!pipeline_) { return fml::Status(fml::StatusCode::kCancelled, "No valid pipeline is bound to the RenderPass."); } - const ContextVK& context_vk = ContextVK::Cast(*context_); + //---------------------------------------------------------------------------- + /// If there are immutable samplers referenced in the render pass, the base + /// pipeline variant is no longer valid and needs to be re-constructed to + /// reference the samplers. + /// + /// This is an instance of JIT creation of PSOs that can cause jank. It is + /// unavoidable because it isn't possible to know all possible combinations of + /// target YUV conversions. Fortunately, this will only ever happen when + /// rendering to external textures. Like Android Hardware Buffers on Android. + /// + /// Even when JIT creation is unavoidable, pipelines will cache their variants + /// when able and all pipeline creation will happen via a base pipeline cache + /// anyway. So the jank can be mostly entirely ameliorated and it should only + /// ever happen when the first unknown YUV conversion is encountered. + /// + /// Jank can be completely eliminated by pre-populating known YUV conversion + /// pipelines. + if (immutable_sampler_) { + std::shared_ptr pipeline_variant = + PipelineVK::Cast(*pipeline_) + .CreateVariantForImmutableSamplers(immutable_sampler_); + if (!pipeline_variant) { + return fml::Status( + fml::StatusCode::kAborted, + "Could not create pipeline variant with immutable sampler."); + } + pipeline_ = std::move(pipeline_variant); + } + + const auto& context_vk = ContextVK::Cast(*context_); + const auto& pipeline_vk = PipelineVK::Cast(*pipeline_); + + auto descriptor_result = + command_buffer_->GetEncoder()->AllocateDescriptorSets( + pipeline_vk.GetDescriptorSetLayout(), context_vk); + if (!descriptor_result.ok()) { + return fml::Status(fml::StatusCode::kAborted, + "Could not allocate descriptor sets."); + } + const auto descriptor_set = descriptor_result.value(); + const auto pipeline_layout = pipeline_vk.GetPipelineLayout(); + command_buffer_vk_.bindPipeline(vk::PipelineBindPoint::eGraphics, + pipeline_vk.GetPipeline()); + for (auto i = 0u; i < descriptor_write_offset_; i++) { - write_workspace_[i].dstSet = descriptor_set_; + write_workspace_[i].dstSet = descriptor_set; } context_vk.GetDevice().updateDescriptorSets(descriptor_write_offset_, @@ -448,10 +483,10 @@ fml::Status RenderPassVK::Draw() { command_buffer_vk_.bindDescriptorSets( vk::PipelineBindPoint::eGraphics, // bind point - pipeline_layout_, // layout + pipeline_layout, // layout 0, // first set 1, // set count - &descriptor_set_, // sets + &descriptor_set, // sets 0, // offset count nullptr // offsets ); @@ -489,8 +524,9 @@ fml::Status RenderPassVK::Draw() { instance_count_ = 1u; base_vertex_ = 0u; vertex_count_ = 0u; - pipeline_valid_ = false; + pipeline_ = nullptr; pipeline_uses_input_attachments_ = false; + immutable_sampler_ = nullptr; return fml::Status(); } @@ -567,6 +603,10 @@ bool RenderPassVK::BindResource(ShaderStage stage, return false; } + if (!immutable_sampler_) { + immutable_sampler_ = texture_vk.GetImmutableSamplerVariant(sampler_vk); + } + vk::DescriptorImageInfo image_info; image_info.imageLayout = vk::ImageLayout::eShaderReadOnlyOptimal; image_info.sampler = sampler_vk.GetSampler(); diff --git a/impeller/renderer/backend/vulkan/render_pass_vk.h b/impeller/renderer/backend/vulkan/render_pass_vk.h index bf05e523d7ced..49a75fb98598a 100644 --- a/impeller/renderer/backend/vulkan/render_pass_vk.h +++ b/impeller/renderer/backend/vulkan/render_pass_vk.h @@ -16,6 +16,7 @@ namespace impeller { class CommandBufferVK; +class SamplerVK; class RenderPassVK final : public RenderPass { public: @@ -47,10 +48,9 @@ class RenderPassVK final : public RenderPass { size_t vertex_count_ = 0u; bool has_index_buffer_ = false; bool has_label_ = false; - bool pipeline_valid_ = false; + std::shared_ptr> pipeline_; bool pipeline_uses_input_attachments_ = false; - vk::DescriptorSet descriptor_set_ = {}; - vk::PipelineLayout pipeline_layout_ = {}; + std::shared_ptr immutable_sampler_; RenderPassVK(const std::shared_ptr& context, const RenderTarget& target, diff --git a/impeller/renderer/backend/vulkan/sampler_library_vk.cc b/impeller/renderer/backend/vulkan/sampler_library_vk.cc index 81fc820c240ec..97685794df799 100644 --- a/impeller/renderer/backend/vulkan/sampler_library_vk.cc +++ b/impeller/renderer/backend/vulkan/sampler_library_vk.cc @@ -28,41 +28,8 @@ const std::unique_ptr& SamplerLibraryVK::GetSampler( if (!device_holder || !device_holder->GetDevice()) { return kNullSampler; } - - const auto mip_map = ToVKSamplerMipmapMode(desc.mip_filter); - - const auto min_filter = ToVKSamplerMinMagFilter(desc.min_filter); - const auto mag_filter = ToVKSamplerMinMagFilter(desc.mag_filter); - - const auto address_mode_u = ToVKSamplerAddressMode(desc.width_address_mode); - const auto address_mode_v = ToVKSamplerAddressMode(desc.height_address_mode); - const auto address_mode_w = ToVKSamplerAddressMode(desc.depth_address_mode); - - const auto sampler_create_info = - vk::SamplerCreateInfo() - .setMagFilter(mag_filter) - .setMinFilter(min_filter) - .setAddressModeU(address_mode_u) - .setAddressModeV(address_mode_v) - .setAddressModeW(address_mode_w) - .setBorderColor(vk::BorderColor::eFloatTransparentBlack) - .setMipmapMode(mip_map); - - auto res = - device_holder->GetDevice().createSamplerUnique(sampler_create_info); - if (res.result != vk::Result::eSuccess) { - FML_LOG(ERROR) << "Failed to create sampler: " << vk::to_string(res.result); - return kNullSampler; - } - - auto sampler = std::make_unique(desc, std::move(res.value)); - - if (!desc.label.empty()) { - ContextVK::SetDebugName(device_holder->GetDevice(), sampler->GetSampler(), - desc.label.c_str()); - } - - return (samplers_[desc] = std::move(sampler)); + return (samplers_[desc] = + std::make_unique(device_holder->GetDevice(), desc)); } } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/sampler_vk.cc b/impeller/renderer/backend/vulkan/sampler_vk.cc index fd6c1ef542a43..175507d1ab9bf 100644 --- a/impeller/renderer/backend/vulkan/sampler_vk.cc +++ b/impeller/renderer/backend/vulkan/sampler_vk.cc @@ -4,11 +4,106 @@ #include "impeller/renderer/backend/vulkan/sampler_vk.h" +#include "impeller/renderer/backend/vulkan/context_vk.h" +#include "impeller/renderer/backend/vulkan/formats_vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" + namespace impeller { -SamplerVK::SamplerVK(SamplerDescriptor desc, vk::UniqueSampler sampler) +static vk::UniqueSampler CreateSampler( + const vk::Device& device, + const SamplerDescriptor& desc, + const std::shared_ptr& yuv_conversion) { + const auto mip_map = ToVKSamplerMipmapMode(desc.mip_filter); + + const auto min_filter = ToVKSamplerMinMagFilter(desc.min_filter); + const auto mag_filter = ToVKSamplerMinMagFilter(desc.mag_filter); + + const auto address_mode_u = ToVKSamplerAddressMode(desc.width_address_mode); + const auto address_mode_v = ToVKSamplerAddressMode(desc.height_address_mode); + const auto address_mode_w = ToVKSamplerAddressMode(desc.depth_address_mode); + + vk::StructureChain + sampler_chain; + + auto& sampler_info = sampler_chain.get(); + + sampler_info.magFilter = mag_filter; + sampler_info.minFilter = min_filter; + sampler_info.addressModeU = address_mode_u; + sampler_info.addressModeV = address_mode_v; + sampler_info.addressModeW = address_mode_w; + sampler_info.borderColor = vk::BorderColor::eFloatTransparentBlack; + sampler_info.mipmapMode = mip_map; + + if (yuv_conversion && yuv_conversion->IsValid()) { + sampler_chain.get().conversion = + yuv_conversion->GetConversion(); + + // + // TL;DR: When using YUV conversion, our samplers are somewhat hobbled and + // not all options configurable in Impeller (especially the linear + // filtering which is by far the most used form of filtering) can be + // supported. Switch to safe defaults. + // + // Spec: If sampler Y'CBCR conversion is enabled and the potential format + // features of the sampler Y'CBCR conversion do not support or enable + // separate reconstruction filters, minFilter and magFilter must be equal to + // the sampler Y'CBCR conversion's chromaFilter. + // + // Thing is, we don't enable separate reconstruction filters. By the time we + // are here, we also don't have access to the descriptor used to create this + // conversion. So we don't yet know what the chromaFilter is. But eNearest + // is a safe bet since the `AndroidHardwareBufferTextureSourceVK` defaults + // to that safe value. So just use that. + // + // See the validation VUID-VkSamplerCreateInfo-minFilter-01645 for more. + // + sampler_info.magFilter = vk::Filter::eNearest; + sampler_info.minFilter = vk::Filter::eNearest; + + // Spec: If sampler Y′CBCR conversion is enabled, addressModeU, + // addressModeV, and addressModeW must be + // VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, anisotropyEnable must be VK_FALSE, + // and unnormalizedCoordinates must be VK_FALSE. + // + // See the validation VUID-VkSamplerCreateInfo-addressModeU-01646 for more. + // + sampler_info.addressModeU = vk::SamplerAddressMode::eClampToEdge; + sampler_info.addressModeV = vk::SamplerAddressMode::eClampToEdge; + sampler_info.addressModeW = vk::SamplerAddressMode::eClampToEdge; + sampler_info.anisotropyEnable = false; + sampler_info.unnormalizedCoordinates = false; + } else { + sampler_chain.unlink(); + } + + auto sampler = device.createSamplerUnique(sampler_chain.get()); + if (sampler.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not create sampler: " + << vk::to_string(sampler.result); + return {}; + } + + if (!desc.label.empty()) { + ContextVK::SetDebugName(device, sampler.value.get(), desc.label.c_str()); + } + + return std::move(sampler.value); +} + +SamplerVK::SamplerVK(const vk::Device& device, + SamplerDescriptor desc, + std::shared_ptr yuv_conversion) : Sampler(std::move(desc)), - sampler_(MakeSharedVK(std::move(sampler))) {} + device_(device), + sampler_(MakeSharedVK( + CreateSampler(device, desc_, yuv_conversion))), + yuv_conversion_(std::move(yuv_conversion)) { + is_valid_ = sampler_ && !!sampler_->Get(); +} SamplerVK::~SamplerVK() = default; @@ -16,4 +111,16 @@ vk::Sampler SamplerVK::GetSampler() const { return *sampler_; } +std::shared_ptr SamplerVK::CreateVariantForConversion( + std::shared_ptr conversion) const { + if (!conversion || !is_valid_) { + return nullptr; + } + return std::make_shared(device_, desc_, std::move(conversion)); +} + +const std::shared_ptr& SamplerVK::GetYUVConversion() const { + return yuv_conversion_; +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/sampler_vk.h b/impeller/renderer/backend/vulkan/sampler_vk.h index 30b124a03c682..aa572e1d7a8e5 100644 --- a/impeller/renderer/backend/vulkan/sampler_vk.h +++ b/impeller/renderer/backend/vulkan/sampler_vk.h @@ -14,20 +14,30 @@ namespace impeller { class SamplerLibraryVK; +class YUVConversionVK; class SamplerVK final : public Sampler, public BackendCast { public: - SamplerVK(SamplerDescriptor desc, vk::UniqueSampler sampler); + SamplerVK(const vk::Device& device, + SamplerDescriptor desc, + std::shared_ptr yuv_conversion = {}); // |Sampler| ~SamplerVK() override; vk::Sampler GetSampler() const; + std::shared_ptr CreateVariantForConversion( + std::shared_ptr conversion) const; + + const std::shared_ptr& GetYUVConversion() const; + private: friend SamplerLibraryVK; - std::shared_ptr> sampler_; + const vk::Device device_; + SharedHandleVK sampler_; + std::shared_ptr yuv_conversion_; bool is_valid_ = false; SamplerVK(const SamplerVK&) = delete; diff --git a/impeller/renderer/backend/vulkan/texture_source_vk.cc b/impeller/renderer/backend/vulkan/texture_source_vk.cc index 4a39bce151bd8..8fa29cbe8b9a6 100644 --- a/impeller/renderer/backend/vulkan/texture_source_vk.cc +++ b/impeller/renderer/backend/vulkan/texture_source_vk.cc @@ -14,6 +14,10 @@ const TextureDescriptor& TextureSourceVK::GetTextureDescriptor() const { return desc_; } +std::shared_ptr TextureSourceVK::GetYUVConversion() const { + return nullptr; +} + vk::ImageLayout TextureSourceVK::GetLayout() const { ReaderLock lock(layout_mutex_); return layout_; diff --git a/impeller/renderer/backend/vulkan/texture_source_vk.h b/impeller/renderer/backend/vulkan/texture_source_vk.h index 4f76a8067cf4f..bbe5552870ccf 100644 --- a/impeller/renderer/backend/vulkan/texture_source_vk.h +++ b/impeller/renderer/backend/vulkan/texture_source_vk.h @@ -12,57 +12,117 @@ #include "impeller/renderer/backend/vulkan/formats_vk.h" #include "impeller/renderer/backend/vulkan/shared_object_vk.h" #include "impeller/renderer/backend/vulkan/vk.h" +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" #include "vulkan/vulkan_handles.hpp" namespace impeller { -/// Abstract base class that represents a vkImage and an vkImageView. +//------------------------------------------------------------------------------ +/// @brief Abstract base class that represents a vkImage and an +/// vkImageView. +/// +/// This is intended to be used with an impeller::TextureVK. Example +/// implementations represent swapchain images, uploaded textures, +/// Android Hardware Buffer backend textures, etc... /// -/// This is intended to be used with an impeller::TextureVK. Example -/// implementations represent swapchain images or uploaded textures. class TextureSourceVK { public: virtual ~TextureSourceVK(); + //---------------------------------------------------------------------------- + /// @brief Gets the texture descriptor for this image source. + /// + /// @warning Texture descriptors from texture sources whose capabilities + /// are a superset of those that can be expressed with Vulkan + /// (like Android Hardware Buffer) are inferred. Stuff like size, + /// mip-counts, types is reliable. So use these descriptors as + /// advisory. Creating copies of texture sources from these + /// descriptors is usually not possible and depends on the + /// allocator used. + /// + /// @return The texture descriptor. + /// const TextureDescriptor& GetTextureDescriptor() const; + //---------------------------------------------------------------------------- + /// @brief Get the image handle for this texture source. + /// + /// @return The image. + /// virtual vk::Image GetImage() const = 0; - /// @brief Retrieve the image view used for sampling/blitting/compute with - /// this texture source. + //---------------------------------------------------------------------------- + /// @brief Retrieve the image view used for sampling/blitting/compute + /// with this texture source. + /// + /// @return The image view. + /// virtual vk::ImageView GetImageView() const = 0; - /// @brief Retrieve the image view used for render target attachments - /// with this texture source. + //---------------------------------------------------------------------------- + /// @brief Retrieve the image view used for render target attachments + /// with this texture source. + /// + /// ImageViews used as render target attachments cannot have any + /// mip levels. In cases where we want to generate mipmaps with + /// the result of this texture, we need to create multiple image + /// views. + /// + /// @return The render target view. /// - /// ImageViews used as render target attachments cannot have any mip levels. - /// In cases where we want to generate mipmaps with the result of this - /// texture, we need to create multiple image views. virtual vk::ImageView GetRenderTargetView() const = 0; - /// Encodes the layout transition `barrier` to `barrier.cmd_buffer` for the - /// image. + //---------------------------------------------------------------------------- + /// @brief Encodes the layout transition `barrier` to + /// `barrier.cmd_buffer` for the image. + /// + /// The transition is from the layout stored via + /// `SetLayoutWithoutEncoding` to `barrier.new_layout`. + /// + /// @param[in] barrier The barrier. + /// + /// @return If the layout transition was successfully made. /// - /// The transition is from the layout stored via `SetLayoutWithoutEncoding` to - /// `barrier.new_layout`. fml::Status SetLayout(const BarrierVK& barrier) const; - /// Store the layout of the image. + //---------------------------------------------------------------------------- + /// @brief Store the layout of the image. + /// + /// This just is bookkeeping on the CPU, to actually set the + /// layout use `SetLayout`. /// - /// This just is bookkeeping on the CPU, to actually set the layout use - /// `SetLayout`. + /// @param[in] layout The new layout. + /// + /// @return The old layout. /// - /// @param layout The new layout. - /// @return The old layout. vk::ImageLayout SetLayoutWithoutEncoding(vk::ImageLayout layout) const; - /// Get the last layout assigned to the TextureSourceVK. + //---------------------------------------------------------------------------- + /// @brief Get the last layout assigned to the TextureSourceVK. + /// + /// This value is synchronized with the GPU via SetLayout so it + /// may not reflect the actual layout. + /// + /// @return The last known layout of the texture source. /// - /// This value is synchronized with the GPU via SetLayout so it may not - /// reflect the actual layout. vk::ImageLayout GetLayout() const; - /// Whether or not this is a swapchain image. + //---------------------------------------------------------------------------- + /// @brief When sampling from textures whose formats are not known to + /// Vulkan, a custom conversion is necessary to setup custom + /// samplers. This accessor provides this conversion if one is + /// present. Most texture source have none. + /// + /// @return The sampler conversion. + /// + virtual std::shared_ptr GetYUVConversion() const; + + //---------------------------------------------------------------------------- + /// @brief Determines if swapchain image. That is, an image used as the + /// root render target. + /// + /// @return Whether or not this is a swapchain image. + /// virtual bool IsSwapchainImage() const = 0; protected: diff --git a/impeller/renderer/backend/vulkan/texture_vk.cc b/impeller/renderer/backend/vulkan/texture_vk.cc index b105d31b96e85..aa682b1c5335a 100644 --- a/impeller/renderer/backend/vulkan/texture_vk.cc +++ b/impeller/renderer/backend/vulkan/texture_vk.cc @@ -7,6 +7,7 @@ #include "impeller/renderer/backend/vulkan/command_buffer_vk.h" #include "impeller/renderer/backend/vulkan/command_encoder_vk.h" #include "impeller/renderer/backend/vulkan/formats_vk.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" namespace impeller { @@ -191,4 +192,26 @@ SharedHandleVK TextureVK::GetRenderPass() const { return render_pass_; } +void TextureVK::SetMipMapGenerated() { + mipmap_generated_ = true; +} + +bool TextureVK::IsSwapchainImage() const { + return source_->IsSwapchainImage(); +} + +std::shared_ptr TextureVK::GetImmutableSamplerVariant( + const SamplerVK& sampler) const { + if (!source_) { + return nullptr; + } + auto conversion = source_->GetYUVConversion(); + if (!conversion) { + // Most textures don't need a sampler conversion and will go down this path. + // Only needed for YUV sampling from external textures. + return nullptr; + } + return sampler.CreateVariantForConversion(std::move(conversion)); +} + } // namespace impeller diff --git a/impeller/renderer/backend/vulkan/texture_vk.h b/impeller/renderer/backend/vulkan/texture_vk.h index 5826e3df78174..520759e8a6ca3 100644 --- a/impeller/renderer/backend/vulkan/texture_vk.h +++ b/impeller/renderer/backend/vulkan/texture_vk.h @@ -10,6 +10,7 @@ #include "impeller/renderer/backend/vulkan/context_vk.h" #include "impeller/renderer/backend/vulkan/device_buffer_vk.h" #include "impeller/renderer/backend/vulkan/formats_vk.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" #include "impeller/renderer/backend/vulkan/texture_source_vk.h" #include "impeller/renderer/backend/vulkan/vk.h" @@ -40,9 +41,12 @@ class TextureVK final : public Texture, public BackendCast { // |Texture| ISize GetSize() const override; - void SetMipMapGenerated() { mipmap_generated_ = true; } + void SetMipMapGenerated(); - bool IsSwapchainImage() const { return source_->IsSwapchainImage(); } + bool IsSwapchainImage() const; + + std::shared_ptr GetImmutableSamplerVariant( + const SamplerVK& sampler) const; // These methods should only be used by render_pass_vk.h diff --git a/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc b/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc new file mode 100644 index 0000000000000..43417815b2554 --- /dev/null +++ b/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.cc @@ -0,0 +1,34 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h" + +#include "impeller/base/validation.h" +#include "impeller/renderer/backend/vulkan/device_holder.h" + +namespace impeller { + +YUVConversionLibraryVK::YUVConversionLibraryVK( + std::weak_ptr device_holder) + : device_holder_(std::move(device_holder)) {} + +YUVConversionLibraryVK::~YUVConversionLibraryVK() = default; + +std::shared_ptr YUVConversionLibraryVK::GetConversion( + const YUVConversionDescriptorVK& desc) { + Lock lock(conversions_mutex_); + auto found = conversions_.find(desc); + if (found != conversions_.end()) { + return found->second; + } + auto device_holder = device_holder_.lock(); + if (!device_holder) { + VALIDATION_LOG << "Context loss during creation of YUV conversion."; + return nullptr; + } + return (conversions_[desc] = std::shared_ptr( + new YUVConversionVK(device_holder->GetDevice(), desc))); +} + +} // namespace impeller diff --git a/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h b/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h new file mode 100644 index 0000000000000..27669ec7c703a --- /dev/null +++ b/impeller/renderer/backend/vulkan/yuv_conversion_library_vk.h @@ -0,0 +1,65 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_LIBRARY_VK_H_ +#define FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_LIBRARY_VK_H_ + +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" + +namespace impeller { + +class DeviceHolder; + +//------------------------------------------------------------------------------ +/// @brief Due the way the Vulkan spec. treats "identically defined" +/// conversions, creating two conversion with identical descriptors, +/// using one with the image and the other with the sampler, is +/// invalid use. +/// +/// A conversion library hashes and caches identical descriptors to +/// de-duplicate conversions. +/// +/// There can only be one conversion library (the constructor is +/// private to force this) and it found in the context. +/// +class YUVConversionLibraryVK { + public: + ~YUVConversionLibraryVK(); + + YUVConversionLibraryVK(const YUVConversionLibraryVK&) = delete; + + YUVConversionLibraryVK& operator=(const YUVConversionLibraryVK&) = delete; + + //---------------------------------------------------------------------------- + /// @brief Get a conversion for the given descriptor. If there is already + /// a conversion created for an equivalent descriptor, a reference + /// to that descriptor is returned instead. + /// + /// @param[in] desc The descriptor. + /// + /// @return The conversion. A previously created conversion if one was + /// present and a new one if not. A newly created conversion is + /// cached for subsequent accesses. + /// + std::shared_ptr GetConversion( + const YUVConversionDescriptorVK& chain); + + private: + friend class ContextVK; + + using ConversionsMap = std::unordered_map, + YUVConversionDescriptorVKHash, + YUVConversionDescriptorVKEqual>; + + std::weak_ptr device_holder_; + Mutex conversions_mutex_; + ConversionsMap conversions_ IPLR_GUARDED_BY(conversions_mutex_); + + explicit YUVConversionLibraryVK(std::weak_ptr device_holder); +}; + +} // namespace impeller + +#endif // FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_LIBRARY_VK_H_ diff --git a/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc b/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc new file mode 100644 index 0000000000000..7ed74fb78da07 --- /dev/null +++ b/impeller/renderer/backend/vulkan/yuv_conversion_vk.cc @@ -0,0 +1,118 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "impeller/renderer/backend/vulkan/yuv_conversion_vk.h" + +#include "flutter/fml/hash_combine.h" +#include "impeller/base/validation.h" +#include "impeller/renderer/backend/vulkan/device_holder.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" + +namespace impeller { + +YUVConversionVK::YUVConversionVK(const vk::Device& device, + const YUVConversionDescriptorVK& chain) + : chain_(chain) { + auto conversion = device.createSamplerYcbcrConversionUnique(chain_.get()); + if (conversion.result != vk::Result::eSuccess) { + VALIDATION_LOG << "Could not create YUV conversion: " + << vk::to_string(conversion.result); + return; + } + conversion_ = std::move(conversion.value); +} + +YUVConversionVK::~YUVConversionVK() = default; + +bool YUVConversionVK::IsValid() const { + return conversion_ && !!conversion_.get(); +} + +vk::SamplerYcbcrConversion YUVConversionVK::GetConversion() const { + return conversion_ ? conversion_.get() : VK_NULL_HANDLE; +} + +const YUVConversionDescriptorVK& YUVConversionVK::GetDescriptor() const { + return chain_; +} + +std::size_t YUVConversionDescriptorVKHash::operator()( + const YUVConversionDescriptorVK& desc) const { + // Hashers in Vulkan HPP hash the pNext member which isn't what we want for + // these to be stable. + const auto& conv = desc.get(); + + std::size_t hash = fml::HashCombine(conv.format, // + conv.ycbcrModel, // + conv.ycbcrRange, // + conv.components.r, // + conv.components.g, // + conv.components.b, // + conv.components.a, // + conv.xChromaOffset, // + conv.yChromaOffset, // + conv.chromaFilter, // + conv.forceExplicitReconstruction // + ); +#if FML_OS_ANDROID + const auto external_format = desc.get(); + fml::HashCombineSeed(hash, external_format.externalFormat); +#endif // FML_OS_ANDROID + + return hash; +}; + +bool YUVConversionDescriptorVKEqual::operator()( + const YUVConversionDescriptorVK& lhs_desc, + const YUVConversionDescriptorVK& rhs_desc) const { + // Default equality checks in Vulkan HPP checks pNext member members by + // pointer which isn't what we want. + { + const auto& lhs = lhs_desc.get(); + const auto& rhs = rhs_desc.get(); + + if (lhs.format != rhs.format || // + lhs.ycbcrModel != rhs.ycbcrModel || // + lhs.ycbcrRange != rhs.ycbcrRange || // + lhs.components.r != rhs.components.r || // + lhs.components.g != rhs.components.g || // + lhs.components.b != rhs.components.b || // + lhs.components.a != rhs.components.a || // + lhs.xChromaOffset != rhs.xChromaOffset || // + lhs.yChromaOffset != rhs.yChromaOffset || // + lhs.chromaFilter != rhs.chromaFilter || // + lhs.forceExplicitReconstruction != rhs.forceExplicitReconstruction // + ) { + return false; + } + } +#if FML_OS_ANDROID + { + const auto lhs = lhs_desc.get(); + const auto rhs = rhs_desc.get(); + return lhs.externalFormat == rhs.externalFormat; + } +#else // FML_OS_ANDROID + return true; +#endif // FML_OS_ANDROID +} + +ImmutableSamplerKeyVK::ImmutableSamplerKeyVK(const SamplerVK& sampler) + : sampler(sampler.GetDescriptor()) { + if (const auto& conversion = sampler.GetYUVConversion()) { + yuv_conversion = conversion->GetDescriptor(); + } +} + +bool ImmutableSamplerKeyVK::IsEqual(const ImmutableSamplerKeyVK& other) const { + return sampler.IsEqual(other.sampler) && + YUVConversionDescriptorVKEqual{}(yuv_conversion, other.yuv_conversion); +} + +std::size_t ImmutableSamplerKeyVK::GetHash() const { + return fml::HashCombine(sampler.GetHash(), + YUVConversionDescriptorVKHash{}(yuv_conversion)); +} + +} // namespace impeller diff --git a/impeller/renderer/backend/vulkan/yuv_conversion_vk.h b/impeller/renderer/backend/vulkan/yuv_conversion_vk.h new file mode 100644 index 0000000000000..88c17ec6b9ad3 --- /dev/null +++ b/impeller/renderer/backend/vulkan/yuv_conversion_vk.h @@ -0,0 +1,109 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_VK_H_ +#define FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_VK_H_ + +#include + +#include "flutter/fml/build_config.h" +#include "impeller/base/comparable.h" +#include "impeller/base/thread.h" +#include "impeller/core/sampler.h" +#include "impeller/renderer/backend/vulkan/sampler_vk.h" +#include "impeller/renderer/backend/vulkan/shared_object_vk.h" +#include "impeller/renderer/backend/vulkan/vk.h" + +namespace impeller { + +//------------------------------------------------------------------------------ +/// A descriptor used to create a new YUV conversion in a conversion library. +/// +using YUVConversionDescriptorVK = + vk::StructureChain; + +class YUVConversionLibraryVK; + +//------------------------------------------------------------------------------ +/// @brief It is sometimes necessary to deal with formats not native to +/// Vulkan. In such cases, extra information is necessary to access +/// images. A YUV conversion object is needed in such instances. +/// +/// There are usually only a handful of viable conversions in a +/// given context. However, due to the way the Vulkan spec. treats +/// "identically defined" conversions, only a single conversion +/// object is valid for an equivalent `YUVConversionDescriptorVK`. +/// Because of this restriction, it is not possible to just create a +/// conversion from a descriptor (as the underlying handles will be +/// equivalent but different). Instead, a conversion may only be +/// obtained from a conversion library. Libraries handle hashing and +/// caching conversions by descriptor. Caller can find a library on +/// the top-level context. They may not create their own (the +/// constructor is private). +/// +class YUVConversionVK final { + public: + ~YUVConversionVK(); + + YUVConversionVK(const YUVConversionVK&) = delete; + + YUVConversionVK& operator=(const YUVConversionVK&) = delete; + + //---------------------------------------------------------------------------- + /// @return `true` if this conversion is valid for use with images and + /// samplers. + /// + bool IsValid() const; + + //---------------------------------------------------------------------------- + /// @brief Get the descriptor used to create this conversion. + /// + const YUVConversionDescriptorVK& GetDescriptor() const; + + //---------------------------------------------------------------------------- + /// @return The Vulkan handle of the YUV conversion. + /// + vk::SamplerYcbcrConversion GetConversion() const; + + private: + friend class YUVConversionLibraryVK; + + YUVConversionDescriptorVK chain_; + vk::UniqueSamplerYcbcrConversion conversion_; + + YUVConversionVK(const vk::Device& device, + const YUVConversionDescriptorVK& chain); +}; + +struct YUVConversionDescriptorVKHash { + std::size_t operator()(const YUVConversionDescriptorVK& object) const; +}; + +struct YUVConversionDescriptorVKEqual { + bool operator()(const YUVConversionDescriptorVK& lhs, + const YUVConversionDescriptorVK& rhs) const; +}; + +struct ImmutableSamplerKeyVK : public Comparable { + SamplerDescriptor sampler; + YUVConversionDescriptorVK yuv_conversion; + + explicit ImmutableSamplerKeyVK(const SamplerVK& sampler); + + // |Comparable| + std::size_t GetHash() const override; + + // |Comparable| + bool IsEqual(const ImmutableSamplerKeyVK& other) const override; +}; + +} // namespace impeller + +#endif // FLUTTER_IMPELLER_RENDERER_BACKEND_VULKAN_YUV_CONVERSION_VK_H_ diff --git a/shell/platform/android/image_external_texture_vk.cc b/shell/platform/android/image_external_texture_vk.cc index b5cf834aa9314..906f4a5ee0561 100644 --- a/shell/platform/android/image_external_texture_vk.cc +++ b/shell/platform/android/image_external_texture_vk.cc @@ -54,18 +54,9 @@ void ImageExternalTextureVK::ProcessFrame(PaintContext& context, return; } - impeller::TextureDescriptor desc; - desc.storage_mode = impeller::StorageMode::kDevicePrivate; - desc.size = {static_cast(bounds.width()), - static_cast(bounds.height())}; - // TODO(johnmccutchan): Use hb_desc to compute the correct format at runtime. - desc.format = impeller::PixelFormat::kR8G8B8A8UNormInt; - desc.mip_count = 1; - auto texture_source = std::make_shared( - desc, impeller_context_->GetDevice(), latest_hardware_buffer, - hb_desc); + impeller_context_, latest_hardware_buffer, hb_desc); auto texture = std::make_shared(impeller_context_, texture_source);