rsx/fp: Rework FP prepass analyzer to capture more information about bx2 expansion

rpcs3/Emu/RSX/Program/ProgramStateCache.cpp

@@ -1,5 +1,6 @@
 #include "stdafx.h"
 #include "ProgramStateCache.h"
+#include "FragmentProgramDecompiler.h"
 #include "Emu/system_config.h"
 #include "Emu/RSX/Core/RSXDriverState.h"
 #include "util/sysinfo.hpp"
@@ -637,19 +638,9 @@ fragment_program_utils::fragment_program_metadata fragment_program_utils::analys
 	while (true)
 	{
 		const auto inst = v128::loadu(instBuffer, index);
+		const auto d0 = OPDEST::from_be32(inst._u32[0]);
+		const auto opcode = static_cast<rsx::assembler::FP_opcode>(d0.opcode);
 
-		// Check for opcode high bit which indicates a branch instructions (opcode 0x40...0x45)
-		if (inst._u32[2] & (1 << 23))
-		{
-			// NOTE: Jump instructions are not yet proved to work outside of loops and if/else blocks
-			// Otherwise we would need to follow the execution chain
-			result.has_branch_instructions = true;
-		}
-		else
-		{
-			const u32 opcode = (inst._u32[0] >> 16) & 0x3F;
-			if (opcode)
-			{
 		switch (opcode)
 		{
 		case RSX_FP_OPCODE_TEX:
@@ -659,13 +650,9 @@ fragment_program_utils::fragment_program_metadata fragment_program_utils::analys
 		case RSX_FP_OPCODE_TXD:
 		case RSX_FP_OPCODE_TXB:
 		case RSX_FP_OPCODE_TXL:
-				{
+			result.referenced_textures_mask |= (1 << d0.tex_num);
-					//Bits 17-20 of word 1, swapped within u16 sections
+			result.has_tex_bx2_conv |= !!d0.exp_tex;
-					//Bits 16-23 are swapped into the upper 8 bits (24-31)
-					const u32 tex_num = (inst._u32[0] >> 25) & 15;
-					result.referenced_textures_mask |= (1 << tex_num);
 			break;
-				}
 		case RSX_FP_OPCODE_PK4:
 		case RSX_FP_OPCODE_UP4:
 		case RSX_FP_OPCODE_PK2:
@@ -676,20 +663,27 @@ fragment_program_utils::fragment_program_metadata fragment_program_utils::analys
 		case RSX_FP_OPCODE_UP16:
 		case RSX_FP_OPCODE_PKG:
 		case RSX_FP_OPCODE_UPG:
-				{
 			result.has_pack_instructions = true;
 			break;
-				}
+		case RSX_FP_OPCODE_BRK:
-				}
+		case RSX_FP_OPCODE_CAL:
+		case RSX_FP_OPCODE_IFE:
+		case RSX_FP_OPCODE_LOOP:
+		case RSX_FP_OPCODE_REP:
+		case RSX_FP_OPCODE_RET:
+			// NOTE: Jump instructions are not yet proved to work outside of loops and if/else blocks
+			// Otherwise we would need to follow the execution chain
+			result.has_branch_instructions = true;
+			break;
 		}
 
-			if (is_any_src_constant(inst))
+		if (rsx::assembler::FP::get_operand_count(opcode) > 0 &&
+			is_any_src_constant(inst))
 		{
 			// Instruction references constant, skip one slot occupied by data
 			index++;
 			result.program_constants_buffer_length += 16;
 		}
-		}
 
 		index++;
 

rpcs3/Emu/RSX/Program/ProgramStateCache.h

@@ -59,6 +59,7 @@ namespace program_hash_util
 
 			bool has_pack_instructions;
 			bool has_branch_instructions;
+			bool has_tex_bx2_conv;
 			bool is_nop_shader;           // Does this affect Z-pass testing???
 		};
 

rpcs3/Emu/RSX/Program/RSXFragmentProgram.h

@@ -54,6 +54,14 @@ union OPDEST
 		u32                  : 9;
 		u32 write_mask       : 4;
 	};
+
+	static OPDEST from_be32(u32 be_word)
+	{
+		const u32 _hex =
+			((be_word & 0x00FF00FF) << 8) |
+			((be_word & 0xFF00FF00) >> 8);
+		return OPDEST{ .HEX = _hex };
+	}
 };
 
 union SRC0