Masked depthstencil clears

src/video_core/host_shaders/CMakeLists.txt

@@ -50,6 +50,7 @@ set(SHADER_FILES
     vulkan_blit_depth_stencil.frag
     vulkan_color_clear.frag
     vulkan_color_clear.vert
+    vulkan_depthstencil_clear.frag
     vulkan_fidelityfx_fsr_easu_fp16.comp
     vulkan_fidelityfx_fsr_easu_fp32.comp
     vulkan_fidelityfx_fsr_rcas_fp16.comp

src/video_core/host_shaders/vulkan_depthstencil_clear.frag

@@ -0,0 +1,12 @@
+// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#version 460 core
+
+layout (push_constant) uniform PushConstants {
+    vec4 clear_depth;
+};
+
+void main() {
+    gl_FragDepth = clear_depth.x;
+}

src/video_core/renderer_vulkan/blit_image.cpp

@@ -16,6 +16,7 @@
 #include "video_core/host_shaders/vulkan_blit_depth_stencil_frag_spv.h"
 #include "video_core/host_shaders/vulkan_color_clear_frag_spv.h"
 #include "video_core/host_shaders/vulkan_color_clear_vert_spv.h"
+#include "video_core/host_shaders/vulkan_depthstencil_clear_frag_spv.h"
 #include "video_core/renderer_vulkan/blit_image.h"
 #include "video_core/renderer_vulkan/maxwell_to_vk.h"
 #include "video_core/renderer_vulkan/vk_scheduler.h"
@@ -428,6 +429,7 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_,
       blit_depth_stencil_frag(BuildShader(device, VULKAN_BLIT_DEPTH_STENCIL_FRAG_SPV)),
       clear_color_vert(BuildShader(device, VULKAN_COLOR_CLEAR_VERT_SPV)),
       clear_color_frag(BuildShader(device, VULKAN_COLOR_CLEAR_FRAG_SPV)),
+      clear_stencil_frag(BuildShader(device, VULKAN_DEPTHSTENCIL_CLEAR_FRAG_SPV)),
       convert_depth_to_float_frag(BuildShader(device, CONVERT_DEPTH_TO_FLOAT_FRAG_SPV)),
       convert_float_to_depth_frag(BuildShader(device, CONVERT_FLOAT_TO_DEPTH_FRAG_SPV)),
       convert_abgr8_to_d24s8_frag(BuildShader(device, CONVERT_ABGR8_TO_D24S8_FRAG_SPV)),
@@ -593,6 +595,28 @@ void BlitImageHelper::ClearColor(const Framebuffer* dst_framebuffer, u8 color_ma
     scheduler.InvalidateState();
 }
 
+void BlitImageHelper::ClearDepthStencil(const Framebuffer* dst_framebuffer, bool depth_clear,
+                                        f32 clear_depth, u8 stencil_mask, u32 stencil_ref,
+                                        u32 stencil_compare_mask, const Region2D& dst_region) {
+    const BlitDepthStencilPipelineKey key{
+        .renderpass = dst_framebuffer->RenderPass(),
+        .depth_clear = depth_clear,
+        .stencil_mask = stencil_mask,
+        .stencil_compare_mask = stencil_compare_mask,
+        .stencil_ref = stencil_ref,
+    };
+    const VkPipeline pipeline = FindOrEmplaceClearStencilPipeline(key);
+    const VkPipelineLayout layout = *clear_color_pipeline_layout;
+    scheduler.RequestRenderpass(dst_framebuffer);
+    scheduler.Record([pipeline, layout, clear_depth, dst_region](vk::CommandBuffer cmdbuf) {
+        cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
+        BindBlitState(cmdbuf, dst_region);
+        cmdbuf.PushConstants(layout, VK_SHADER_STAGE_FRAGMENT_BIT, clear_depth);
+        cmdbuf.Draw(3, 1, 0, 0);
+    });
+    scheduler.InvalidateState();
+}
+
 void BlitImageHelper::Convert(VkPipeline pipeline, const Framebuffer* dst_framebuffer,
                               const ImageView& src_image_view) {
     const VkPipelineLayout layout = *one_texture_pipeline_layout;
@@ -820,6 +844,61 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipel
     return *clear_color_pipelines.back();
 }
 
+VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline(
+    const BlitDepthStencilPipelineKey& key) {
+    const auto it = std::ranges::find(clear_stencil_keys, key);
+    if (it != clear_stencil_keys.end()) {
+        return *clear_stencil_pipelines[std::distance(clear_stencil_keys.begin(), it)];
+    }
+    clear_stencil_keys.push_back(key);
+    const std::array stages = MakeStages(*clear_color_vert, *clear_stencil_frag);
+    const auto stencil = VkStencilOpState{
+        .failOp = VK_STENCIL_OP_KEEP,
+        .passOp = VK_STENCIL_OP_REPLACE,
+        .depthFailOp = VK_STENCIL_OP_KEEP,
+        .compareOp = VK_COMPARE_OP_ALWAYS,
+        .compareMask = key.stencil_compare_mask,
+        .writeMask = key.stencil_mask,
+        .reference = key.stencil_ref,
+    };
+    const VkPipelineDepthStencilStateCreateInfo depth_stencil_ci{
+        .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
+        .pNext = nullptr,
+        .flags = 0,
+        .depthTestEnable = VK_FALSE,
+        .depthWriteEnable = key.depth_clear,
+        .depthCompareOp = VK_COMPARE_OP_ALWAYS,
+        .depthBoundsTestEnable = VK_FALSE,
+        .stencilTestEnable = VK_TRUE,
+        .front = stencil,
+        .back = stencil,
+        .minDepthBounds = 0.0f,
+        .maxDepthBounds = 0.0f,
+    };
+    clear_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({
+        .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
+        .pNext = nullptr,
+        .flags = 0,
+        .stageCount = static_cast<u32>(stages.size()),
+        .pStages = stages.data(),
+        .pVertexInputState = &PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
+        .pInputAssemblyState = &PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
+        .pTessellationState = nullptr,
+        .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO,
+        .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
+        .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
+        .pDepthStencilState = &depth_stencil_ci,
+        .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO,
+        .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO,
+        .layout = *clear_color_pipeline_layout,
+        .renderPass = key.renderpass,
+        .subpass = 0,
+        .basePipelineHandle = VK_NULL_HANDLE,
+        .basePipelineIndex = 0,
+    }));
+    return *clear_stencil_pipelines.back();
+}
+
 void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass,
                                       bool is_target_depth) {
     if (pipeline) {

src/video_core/renderer_vulkan/blit_image.h

@@ -27,6 +27,16 @@ struct BlitImagePipelineKey {
     Tegra::Engines::Fermi2D::Operation operation;
 };
 
+struct BlitDepthStencilPipelineKey {
+    constexpr auto operator<=>(const BlitDepthStencilPipelineKey&) const noexcept = default;
+
+    VkRenderPass renderpass;
+    bool depth_clear;
+    u8 stencil_mask;
+    u32 stencil_compare_mask;
+    u32 stencil_ref;
+};
+
 class BlitImageHelper {
 public:
     explicit BlitImageHelper(const Device& device, Scheduler& scheduler,
@@ -64,6 +74,10 @@ public:
     void ClearColor(const Framebuffer* dst_framebuffer, u8 color_mask,
                     const std::array<f32, 4>& clear_color, const Region2D& dst_region);
 
+    void ClearDepthStencil(const Framebuffer* dst_framebuffer, bool depth_clear, f32 clear_depth,
+                           u8 stencil_mask, u32 stencil_ref, u32 stencil_compare_mask,
+                           const Region2D& dst_region);
+
 private:
     void Convert(VkPipeline pipeline, const Framebuffer* dst_framebuffer,
                  const ImageView& src_image_view);
@@ -76,6 +90,8 @@ private:
     [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key);
 
     [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key);
+    [[nodiscard]] VkPipeline FindOrEmplaceClearStencilPipeline(
+        const BlitDepthStencilPipelineKey& key);
 
     void ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass, bool is_target_depth);
 
@@ -108,6 +124,7 @@ private:
     vk::ShaderModule blit_depth_stencil_frag;
     vk::ShaderModule clear_color_vert;
     vk::ShaderModule clear_color_frag;
+    vk::ShaderModule clear_stencil_frag;
     vk::ShaderModule convert_depth_to_float_frag;
     vk::ShaderModule convert_float_to_depth_frag;
     vk::ShaderModule convert_abgr8_to_d24s8_frag;
@@ -122,6 +139,8 @@ private:
     std::vector<vk::Pipeline> blit_depth_stencil_pipelines;
     std::vector<BlitImagePipelineKey> clear_color_keys;
     std::vector<vk::Pipeline> clear_color_pipelines;
+    std::vector<BlitDepthStencilPipelineKey> clear_stencil_keys;
+    std::vector<vk::Pipeline> clear_stencil_pipelines;
     vk::Pipeline convert_d32_to_r32_pipeline;
     vk::Pipeline convert_r32_to_d32_pipeline;
     vk::Pipeline convert_d16_to_r16_pipeline;

src/video_core/renderer_vulkan/vk_rasterizer.cpp

@@ -428,6 +428,17 @@ void RasterizerVulkan::Clear(u32 layer_count) {
     if (aspect_flags == 0) {
         return;
     }
+
+    if (use_stencil && regs.stencil_front_mask != 0xFF && regs.stencil_front_mask != 0) {
+        Region2D dst_region = {
+            Offset2D{.x = clear_rect.rect.offset.x, .y = clear_rect.rect.offset.y},
+            Offset2D{.x = clear_rect.rect.offset.x + static_cast<s32>(clear_rect.rect.extent.width),
+                     .y = clear_rect.rect.offset.y +
+                          static_cast<s32>(clear_rect.rect.extent.height)}};
+        blit_image.ClearDepthStencil(framebuffer, use_depth, regs.clear_depth,
+                                     static_cast<u8>(regs.stencil_front_mask), regs.clear_stencil,
+                                     regs.stencil_front_func_mask, dst_region);
+    } else {
         scheduler.Record([clear_depth = regs.clear_depth, clear_stencil = regs.clear_stencil,
                           clear_rect, aspect_flags](vk::CommandBuffer cmdbuf) {
             VkClearAttachment attachment;
@@ -437,6 +448,7 @@ void RasterizerVulkan::Clear(u32 layer_count) {
             attachment.clearValue.depthStencil.stencil = clear_stencil;
             cmdbuf.ClearAttachments(attachment, clear_rect);
         });
+    }
 }
 
 void RasterizerVulkan::DispatchCompute() {