rsx/fp: Harden the FP decompiler a bit when bogus inputs are passed in.

rpcs3/Emu/RSX/Program/Assembler/CFG.h

@@ -36,7 +36,7 @@ namespace rsx::assembler
 
 	struct CFGPass
 	{
-		virtual void run(FlowGraph& graph) = 0;
+		virtual bool run(FlowGraph& graph) = 0;
 	};
 
 	FlowGraph deconstruct_fragment_program(const RSXFragmentProgram& prog);

rpcs3/Emu/RSX/Program/Assembler/FPOpcodes.cpp

@@ -8,6 +8,42 @@
 
 namespace rsx::assembler::FP
 {
+	static const char* s_opcode_names[RSX_FP_OPCODE_ENUM_MAX + 1] =
+	{
+		"NOP", "MOV", "MUL", "ADD", "MAD", "DP3", "DP4", "DST", "MIN", "MAX", "SLT", "SGE", "SLE", "SGT", "SNE", "SEQ",                                   // 0x00 - 0x0F
+		"FRC", "FLR", "KIL", "PK4", "UP4", "DDX", "DDY", "TEX", "TXP", "TXD", "RCP", "RSQ", "EX2", "LG2", "LIT", "LRP",                                   // 0x10 - 0x1F
+		"STR", "SFL", "COS", "SIN", "PK2", "UP2", "POW", "PKB", "UPB", "PK16", "UP16", "BEM", "PKG", "UPG", "DP2A", "TXL",                                // 0x20 - 0x2F
+		"UNK_30", "TXB", "UNK_32", "TEXBEM", "TXPBEM", "BEMLUM", "REFL", "TIMESWTEX", "DP2", "NRM", "DIV", "DIVSQ", "LIF", "FENCT", "FENCB", "UNK_3F",    // 0x30 - 0x3F
+		"BRK", "CAL", "IFE", "LOOP", "REP", "RET",                                                                                                        // 0x40 - 0x45 (Flow control)
+		"OR16_LO", "OR16_HI"                                                                                                                              // Custom instructions for RPCS3 use
+	};
+
+	const char* get_opcode_name(FP_opcode opcode)
+	{
+		if (opcode > RSX_FP_OPCODE_ENUM_MAX)
+		{
+			return "invalid";
+		}
+		return s_opcode_names[opcode];
+	}
+
+	bool is_instruction_valid(FP_opcode opcode)
+	{
+		switch (opcode)
+		{
+		case RSX_FP_OPCODE_POW:
+		case RSX_FP_OPCODE_BEM:
+		case RSX_FP_OPCODE_TEXBEM:
+		case RSX_FP_OPCODE_TXPBEM:
+		case RSX_FP_OPCODE_BEMLUM:
+		case RSX_FP_OPCODE_TIMESWTEX:
+			return false;
+		default:
+			// This isn't necessarily true
+			return opcode <= RSX_FP_OPCODE_ENUM_MAX;
+		}
+	}
+
 	u8 get_operand_count(FP_opcode opcode)
 	{
 		switch (opcode)
@@ -90,6 +126,8 @@ namespace rsx::assembler::FP
 			return 2;
 		case RSX_FP_OPCODE_LIF:
 			return 1;
+		case RSX_FP_OPCODE_REFL:
+			return 2;
 		case RSX_FP_OPCODE_FENCT:
 		case RSX_FP_OPCODE_FENCB:
 		case RSX_FP_OPCODE_BRK:
@@ -110,8 +148,6 @@ namespace rsx::assembler::FP
 		case RSX_FP_OPCODE_TXPBEM:
 		case RSX_FP_OPCODE_BEMLUM:
 			fmt::throw_exception("Unimplemented BEM class instruction"); // Unused
-		case RSX_FP_OPCODE_REFL:
-			return 2;
 		case RSX_FP_OPCODE_TIMESWTEX:
 			fmt::throw_exception("Unimplemented TIMESWTEX instruction"); // Unused
 		default:
@@ -397,7 +433,7 @@ namespace rsx::assembler::FP
 	// Convert vector mask to file range
 	rsx::simple_array<u32> get_register_file_range(const RegisterRef& reg)
 	{
-		if (!reg.mask)
+		if (!reg.mask || reg.reg.id >= 48)
 		{
 			return {};
 		}

rpcs3/Emu/RSX/Program/Assembler/FPOpcodes.h

@@ -81,6 +81,9 @@ namespace rsx::assembler
 		// Custom opcodes for dependency injection
 		RSX_FP_OPCODE_OR16_LO = 0x46,  // Performs a 16-bit OR, taking one register channel as input and overwriting low 16 bits of the output
 		RSX_FP_OPCODE_OR16_HI = 0x47,  // Same as the lo variant but now overwrites the high 16-bit block
+
+		// Meta
+		RSX_FP_OPCODE_ENUM_MAX = RSX_FP_OPCODE_OR16_HI
 	};
 
 	namespace FP
@@ -100,6 +103,12 @@ namespace rsx::assembler
 
 		using register_file_t = std::array<char, constants::register_file_max_len>;
 
+		// Convert opcode to human-readable string
+		const char* get_opcode_name(FP_opcode opcode);
+
+		// Returns true if the instruction is implemented by RSX HW
+		bool is_instruction_valid(FP_opcode opcode);
+
 		// Returns number of operands consumed by an instruction
 		u8 get_operand_count(FP_opcode opcode);
 

rpcs3/Emu/RSX/Program/Assembler/Passes/FP/RegisterAnnotationPass.cpp

@@ -126,8 +126,9 @@ namespace rsx::assembler::FP
 	}
 
 	// Decay instructions into register references
-	void annotate_instructions(BasicBlock* block, const RSXFragmentProgram& prog, bool skip_delay_slots)
+	bool annotate_instructions(BasicBlock* block, const RSXFragmentProgram& prog, bool skip_delay_slots)
 	{
+		bool result = true;
 		for (auto& instruction : block->instructions)
 		{
 			if (skip_delay_slots && is_delay_slot(instruction))
@@ -135,7 +136,15 @@ namespace rsx::assembler::FP
 				continue;
 			}
 
-			const u32 operand_count = get_operand_count(static_cast<FP_opcode>(instruction.opcode));
+			const auto opcode = static_cast<FP_opcode>(instruction.opcode);
+			if (!is_instruction_valid(opcode))
+			{
+				rsx_log.error("[CFG] Annotation: Unexpected instruction '%s'", get_opcode_name(opcode));
+				result = false;
+				continue;
+			}
+
+			const u32 operand_count = get_operand_count(opcode);
 			for (u32 i = 0; i < operand_count; i++)
 			{
 				RegisterRef reg = get_src_register(prog, &instruction, i);
@@ -145,15 +154,29 @@ namespace rsx::assembler::FP
 					continue;
 				}
 
+				if (reg.reg.id >= 48)
+				{
+					rsx_log.error("[CFG] Annotation: Instruction references invalid register %s", reg.reg.to_string());
+					result = false;
+				}
+
 				instruction.srcs.push_back(std::move(reg));
 			}
 
 			RegisterRef dst = get_dst_register(&instruction);
 			if (dst)
 			{
+				if (dst.reg.id >= 48)
+				{
+					rsx_log.error("[CFG] Annotation: Instruction references invalid register %s", dst.reg.to_string());
+					result = false;
+				}
+
 				instruction.dsts.push_back(std::move(dst));
 			}
 		}
+
+		return result;
 	}
 
 	// Annotate each block with input and output lanes (read and clobber list)
@@ -215,12 +238,19 @@ namespace rsx::assembler::FP
 		block->input_list = compile_register_file(input_register_file);
 	}
 
-	void RegisterAnnotationPass::run(FlowGraph& graph)
+	bool RegisterAnnotationPass::run(FlowGraph& graph)
 	{
+		bool result = true;
 		for (auto& block : graph.blocks)
 		{
-			annotate_instructions(&block, m_prog, m_config.skip_delay_slots);
+			if (!annotate_instructions(&block, m_prog, m_config.skip_delay_slots))
+			{
+				result = false;
+			}
+
 			annotate_block_io(&block);
 		}
+
+		return result;
 	}
 }

rpcs3/Emu/RSX/Program/Assembler/Passes/FP/RegisterAnnotationPass.h

@@ -25,7 +25,7 @@ namespace rsx::assembler::FP
 			: m_prog(prog), m_config(options)
 		{}
 
-		void run(FlowGraph& graph) override;
+		bool run(FlowGraph& graph) override;
 
 	private:
 		const RSXFragmentProgram& m_prog;

rpcs3/Emu/RSX/Program/Assembler/Passes/FP/RegisterDependencyPass.cpp

@@ -464,7 +464,7 @@ namespace rsx::assembler::FP
 		}
 	}
 
-	void RegisterDependencyPass::run(FlowGraph& graph)
+	bool RegisterDependencyPass::run(FlowGraph& graph)
 	{
 		DependencyPassContext ctx{};
 
@@ -480,5 +480,7 @@ namespace rsx::assembler::FP
 		{
 			insert_block_dependencies(ctx, &(*it));
 		}
+
+		return true;
 	}
 }

rpcs3/Emu/RSX/Program/Assembler/Passes/FP/RegisterDependencyPass.h

@@ -10,6 +10,6 @@ namespace rsx::assembler::FP
 	class RegisterDependencyPass : public CFGPass
 	{
 	public:
-		void run(FlowGraph& graph) override;
+		bool run(FlowGraph& graph) override;
 	};
 }

rpcs3/Emu/RSX/Program/FragmentProgramDecompiler.cpp

@@ -761,7 +761,7 @@ template<typename T> std::string FragmentProgramDecompiler::GetSRC(T src)
 		break;
 
 	case RSX_FP_REGISTER_TYPE_UNKNOWN: // ??? Used by a few games, what is it?
-		rsx_log.error("Src type 3 used, opcode=0x%X, dst=0x%X s0=0x%X s1=0x%X s2=0x%X",
+		rsx_log.error("[FP] Invalid Src type 3 used, opcode=0x%X, dst=0x%X s0=0x%X s1=0x%X s2=0x%X",
 				dst.opcode, dst.HEX, src0.HEX, src1.HEX, src2.HEX);
 
 		// This is not some special type, it is a bug indicating memory corruption
@@ -1289,6 +1289,7 @@ bool FragmentProgramDecompiler::handle_tex_srb(u32 opcode)
 std::string FragmentProgramDecompiler::Decompile()
 {
 	auto graph = deconstruct_fragment_program(m_prog);
+	m_is_valid_ucode = true;
 
 	if (!graph.blocks.empty())
 	{
@@ -1315,15 +1316,14 @@ std::string FragmentProgramDecompiler::Decompile()
 		FP::RegisterAnnotationPass annotation_pass{ m_prog, { .skip_delay_slots = true } };
 		FP::RegisterDependencyPass dependency_pass{};
 
-		annotation_pass.run(graph);
+		m_is_valid_ucode = m_is_valid_ucode && annotation_pass.run(graph);
-		dependency_pass.run(graph);
+		m_is_valid_ucode = m_is_valid_ucode && dependency_pass.run(graph);
 	}
 
 	m_size = 0;
 	m_location = 0;
 	m_loop_count = 0;
 	m_code_level = 1;
-	m_is_valid_ucode = true;
 	m_constant_offsets.clear();
 
 	// For GLSL scope wind/unwind. We store the min scope depth and loop count for each block and "unwind" to it.