rsx: Disable early-Z optimizations if a loopback on the depth texture is active

- The optimizations worked a bit too well and allowed early-Z to kick in.
- Early Z inverts the logic making reads happen after writes and breaking loopback sampling.

rpcs3/Emu/RSX/GL/GLFragmentProgram.cpp

@@ -335,20 +335,31 @@ void GLFragmentDecompilerThread::insertMainEnd(std::stringstream & OS)
 
 	OS << "\n" << "	fs_main();\n\n";
 
+	if ((m_ctrl & RSX_SHADER_CONTROL_DISABLE_EARLY_Z) &&
+		!(m_ctrl & (CELL_GCM_SHADER_CONTROL_DEPTH_EXPORT | RSX_SHADER_CONTROL_META_USES_DISCARD)) &&
+		g_cfg.video.shader_precision != gpu_preset_level::low)
+	{
+		// This is effectively unreachable code, but good enough to trick the GPU to skip early Z
+		OS <<
+			"// Insert NOP sequence to disable early-Z\n"
+			"if (isnan(gl_FragCoord.z))\n"
+			"	discard;\n\n";
+	}
+
 	glsl::insert_rop(OS, m_shader_props);
 
 	if (m_ctrl & CELL_GCM_SHADER_CONTROL_DEPTH_EXPORT)
 	{
 		if (m_parr.HasParam(PF_PARAM_NONE, "vec4", "r1"))
 		{
-			//Depth writes are always from a fp32 register. See issues section on nvidia's NV_fragment_program spec
+			// Depth writes are always from a fp32 register. See issues section on nvidia's NV_fragment_program spec
-			//https://www.khronos.org/registry/OpenGL/extensions/NV/NV_fragment_program.txt
+			// https://www.khronos.org/registry/OpenGL/extensions/NV/NV_fragment_program.txt
 			OS << "	gl_FragDepth = r1.z;\n";
 		}
 		else
 		{
-			//Input not declared. Leave commented to assist in debugging the shader
+			// Input not declared. Leave commented to assist in debugging the shader
-			OS << "	//gl_FragDepth = r1.z;\n";
+			OS << "	// gl_FragDepth = r1.z;\n";
 		}
 	}
 

rpcs3/Emu/RSX/RSXThread.cpp

@@ -2240,6 +2240,12 @@ namespace rsx
 				default:
 					rsx_log.error("Depth texture bound to pipeline with unexpected format 0x%X", format);
 				}
+
+				if (sampler_descriptors[i]->is_cyclic_reference &&
+					g_cfg.video.shader_precision != gpu_preset_level::low)
+				{
+					current_fragment_program.ctrl |= RSX_SHADER_CONTROL_DISABLE_EARLY_Z;
+				}
 			}
 			else if (!backend_config.supports_hw_renormalization /* &&
 				tex.min_filter() == rsx::texture_minify_filter::nearest &&

rpcs3/Emu/RSX/VK/VKFragmentProgram.cpp

@@ -471,6 +471,17 @@ void VKFragmentDecompilerThread::insertMainEnd(std::stringstream & OS)
 
 	OS << "\n" << "	fs_main();\n\n";
 
+	if ((m_ctrl & RSX_SHADER_CONTROL_DISABLE_EARLY_Z) &&
+		!(m_ctrl & (CELL_GCM_SHADER_CONTROL_DEPTH_EXPORT | RSX_SHADER_CONTROL_META_USES_DISCARD)) &&
+		g_cfg.video.shader_precision != gpu_preset_level::low)
+	{
+		// This is effectively unreachable code, but good enough to trick the GPU to skip early Z
+		OS <<
+			"// Insert NOP sequence to disable early-Z\n"
+			"if (isnan(gl_FragCoord.z))\n"
+			"	discard;\n\n";
+	}
+
 	glsl::insert_rop(OS, m_shader_props);
 
 	if (m_ctrl & CELL_GCM_SHADER_CONTROL_DEPTH_EXPORT)

rpcs3/Emu/RSX/gcm_enums.h

@@ -465,6 +465,11 @@ namespace gcm
 		RSX_SHADER_CONTROL_ALPHA_TEST              = 0x0400000, // Uses alpha test on the outputs
 		RSX_SHADER_CONTROL_POLYGON_STIPPLE         = 0x0800000, // Uses polygon stipple for dithered rendering
 		RSX_SHADER_CONTROL_ALPHA_TO_COVERAGE       = 0x1000000, // Alpha to coverage
+
+		RSX_SHADER_CONTROL_DISABLE_EARLY_Z         = 0x2000000, // Do not allow early-Z optimizations on this shader
+
+		// Meta
+		RSX_SHADER_CONTROL_META_USES_DISCARD       = (RSX_SHADER_CONTROL_USES_KIL | RSX_SHADER_CONTROL_TEXTURE_ALPHA_KILL | RSX_SHADER_CONTROL_ALPHA_TEST | RSX_SHADER_CONTROL_POLYGON_STIPPLE | RSX_SHADER_CONTROL_ALPHA_TO_COVERAGE)
 	};
 
 	// GCM Reports