RSX: Add hardware spin_wait helper, use it in flush requests

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -1307,11 +1307,11 @@ void GLGSRender::do_local_task(rsx::FIFO::state state)
 	{
 		std::lock_guard lock(queue_guard);
 
-		work_queue.remove_if([](auto &q) { return q.received; });
+		work_queue.remove_if([](auto &q) { return q.received.load(); });
 
 		for (auto& q : work_queue)
 		{
-			if (q.processed) continue;
+			if (q.processed.load()) continue;
 
 			gl::command_context cmd{ gl_state };
 			q.result = m_gl_texture_cache.flush_all(cmd, q.section_data);

rpcs3/Emu/RSX/GL/GLGSRender.h

@@ -6,6 +6,7 @@
 #include "GLOverlays.h"
 #include "GLShaderInterpreter.h"
 #include "Emu/RSX/rsx_cache.h"
+#include "util/asm.hpp"
 
 #include <optional>
 #include <unordered_map>
@@ -45,16 +46,16 @@ namespace gl
 		u32  address_to_flush = 0;
 		gl::texture_cache::thrashed_set section_data;
 
-		volatile bool processed = false;
+		atomic_t<bool> processed = false;
 		volatile bool result = false;
-		volatile bool received = false;
+		atomic_t<bool> received = false;
 
 		void producer_wait()
 		{
-			while (!processed)
+			utils::spin_wait(processed, [](auto v)
 			{
-				std::this_thread::yield();
+				return v;
-			}
+			});
 
 			received = true;
 		}

rpcs3/Emu/RSX/VK/VKGSRenderTypes.hpp

@@ -254,18 +254,18 @@ namespace vk
 
 		void consumer_wait() const
 		{
-			while (num_waiters.load() != 0)
+			utils::spin_wait(num_waiters, [](auto v)
 			{
-				utils::pause();
+				return v == 0;
-			}
+			});
 		}
 
 		void producer_wait() const
 		{
-			while (pending_state.load())
+			utils::spin_wait(pending_state, [](auto v)
 			{
-				std::this_thread::yield();
+				return !v;
-			}
+			});
 		}
 	};
 

rpcs3/util/asm.hpp

@@ -3,11 +3,13 @@
 #include "util/types.hpp"
 #include "util/tsc.hpp"
 #include "util/atomic.hpp"
+#include "util/sysinfo.hpp"
 #include <functional>
 
 #ifdef ARCH_X64
 #ifdef _MSC_VER
 #include <intrin.h>
+#include <immintrin.h>
 #else
 #include <immintrin.h>
 #include <x86intrin.h>
@@ -216,6 +218,84 @@ namespace utils
 		while (get_tsc() < stop);
 	}
 
+	template <typename T, usz Align, typename Pred>
+#if defined(ARCH_X64) && !defined(_MSC_VER)
+	__attribute__((target("waitpkg,mwaitx")))
+#endif
+	inline void spin_wait(const atomic_t<T, Align>& var, Pred predicate)
+	{
+#ifdef ARCH_X64
+		static const bool use_umwait = has_waitpkg();
+		static const bool use_waitx = has_waitx();
+#endif
+
+		const auto read_mem = [&]()
+		{
+			return var.load();
+		};
+
+		const void* addr = &var.raw();
+
+		while (true)
+		{
+			if (predicate(read_mem()))
+			{
+				return;
+			}
+
+#if defined(ARCH_ARM64)
+			using value_type = decltype(read_mem());
+			using wait_type = std::remove_cvref_t<decltype(var.raw())>;
+
+			wait_type value{};
+			const auto* wait_addr = static_cast<const volatile wait_type*>(addr);
+
+			if constexpr (sizeof(wait_type) == 1) __asm__ volatile("ldaxrb %w0, %1" : "=r"(value) : "Q"(*wait_addr) : "memory");
+			else if constexpr (sizeof(wait_type) == 2) __asm__ volatile("ldaxrh %w0, %1" : "=r"(value) : "Q"(*wait_addr) : "memory");
+			else if constexpr (sizeof(wait_type) == 4) __asm__ volatile("ldaxr %w0, %1" : "=r"(value) : "Q"(*wait_addr) : "memory");
+			else if constexpr (sizeof(wait_type) == 8) __asm__ volatile("ldaxr %x0, %1" : "=r"(value) : "Q"(*wait_addr) : "memory");
+			else static_assert(sizeof(wait_type) <= 8, "Unsupported atomic size for spin_wait");
+
+			if (predicate(static_cast<value_type>(value)))
+			{
+				__asm__ volatile("clrex" ::: "memory");
+				return;
+			}
+
+			__asm__ volatile("wfe" ::: "memory");
+			__asm__ volatile("clrex" ::: "memory");
+
+#elif defined(ARCH_X64)
+			if (use_umwait)
+			{
+				_umonitor(const_cast<void*>(addr));
+				if (predicate(read_mem()))
+				{
+					return;
+				}
+
+				_umwait(0, ~0ULL);
+			}
+			else if (use_waitx)
+			{
+				_mm_monitorx(const_cast<void*>(addr), 0, 0);
+				if (predicate(read_mem()))
+				{
+					return;
+				}
+
+				_mm_mwaitx(0, 0, 0);
+			}
+			else
+			{
+				pause();
+			}
+#else
+			pause();
+#endif
+		}
+	}
+
 	// Align to power of 2
 	template <typename T, typename U>
 		requires std::is_unsigned_v<T>

rpcs3/util/atomic.hpp

@@ -1742,6 +1742,11 @@ public:
 		return base::load() != 0;
 	}
 
+	const uchar& raw() const
+	{
+		return base::raw();
+	}
+
 	// Override implicit conversion from the parent type
 	explicit operator uchar() const = delete;