Use attributes for LIKELY/UNLIKELY

Remove LIKELY/UNLIKELY macro.

Utilities/JIT.cpp

@@ -45,7 +45,7 @@ static u8* add_jit_memory(std::size_t size, uint align)
 	// Select subrange
 	u8* pointer = get_jit_memory() + Off;
 
-	if (UNLIKELY(!size && !align))
+	if (!size && !align) [[unlikely]]
 	{
 		// Return subrange info
 		return pointer;
@@ -59,7 +59,7 @@ static u8* add_jit_memory(std::size_t size, uint align)
 		const u64 _pos = ::align(ctr & 0xffff'ffff, align);
 		const u64 _new = ::align(_pos + size, align);
 
-		if (UNLIKELY(_new > 0x40000000))
+		if (_new > 0x40000000) [[unlikely]]
 		{
 			// Sorry, we failed, and further attempts should fail too.
 			ctr |= 0x40000000;
@@ -71,7 +71,7 @@ static u8* add_jit_memory(std::size_t size, uint align)
 		newa = olda;
 
 		// Check the necessity to commit more memory
-		if (UNLIKELY(_new > olda))
+		if (_new > olda) [[unlikely]]
 		{
 			newa = ::align(_new, 0x100000);
 		}
@@ -80,13 +80,13 @@ static u8* add_jit_memory(std::size_t size, uint align)
 		return _pos;
 	});
 
-	if (UNLIKELY(pos == -1))
+	if (pos == -1) [[unlikely]]
 	{
 		jit_log.warning("JIT: Out of memory (size=0x%x, align=0x%x, off=0x%x)", size, align, Off);
 		return nullptr;
 	}
 
-	if (UNLIKELY(olda != newa))
+	if (olda != newa) [[unlikely]]
 	{
 #ifdef CAN_OVERCOMMIT
 		madvise(pointer + olda, newa - olda, MADV_WILLNEED);
@@ -119,21 +119,21 @@ jit_runtime::~jit_runtime()
 asmjit::Error jit_runtime::_add(void** dst, asmjit::CodeHolder* code) noexcept
 {
 	std::size_t codeSize = code->getCodeSize();
-	if (UNLIKELY(!codeSize))
+	if (!codeSize) [[unlikely]]
 	{
 		*dst = nullptr;
 		return asmjit::kErrorNoCodeGenerated;
 	}
 
 	void* p = jit_runtime::alloc(codeSize, 16);
-	if (UNLIKELY(!p))
+	if (!p) [[unlikely]]
 	{
 		*dst = nullptr;
 		return asmjit::kErrorNoVirtualMemory;
 	}
 
 	std::size_t relocSize = code->relocate(p);
-	if (UNLIKELY(!relocSize))
+	if (!relocSize) [[unlikely]]
 	{
 		*dst = nullptr;
 		return asmjit::kErrorInvalidState;

Utilities/Log.cpp

@@ -589,7 +589,7 @@ void logs::file_writer::log(logs::level sev, const char* text, std::size_t size)
 			const u64 v1 = v >> 24;
 			const u64 v2 = v & 0xffffff;
 
-			if (UNLIKELY(v2 + size > 0xffffff || v1 + v2 + size >= m_out + s_log_size))
+			if (v2 + size > 0xffffff || v1 + v2 + size >= m_out + s_log_size) [[unlikely]]
 			{
 				bufv = v;
 				return nullptr;
@@ -599,7 +599,7 @@ void logs::file_writer::log(logs::level sev, const char* text, std::size_t size)
 			return m_fptr + (v1 + v2) % s_log_size;
 		});
 
-		if (UNLIKELY(!pos))
+		if (!pos) [[unlikely]]
 		{
 			if ((bufv & 0xffffff) + size > 0xffffff || bufv & 0xffffff)
 			{

Utilities/Log.h

@@ -80,7 +80,7 @@ namespace logs
 		template <std::size_t N, typename... Args>\
 		void _sev(const char(&fmt)[N], const Args&... args)\
 		{\
-			if (UNLIKELY(level::_sev <= enabled.load(std::memory_order_relaxed)))\
+			if (level::_sev <= enabled.load(std::memory_order_relaxed)) [[unlikely]]\
 			{\
 				static constexpr fmt_type_info type_list[sizeof...(Args) + 1]{fmt_type_info::make<fmt_unveil_t<Args>>()...};\
 				msg_##_sev.broadcast(fmt, type_list, u64{fmt_unveil<Args>::get(args)}...);\
@@ -89,7 +89,7 @@ namespace logs
 		template <std::size_t N, typename... Args>\
 		void _sev(const char2(&fmt)[N], const Args&... args)\
 		{\
-			if (UNLIKELY(level::_sev <= enabled.load(std::memory_order_relaxed)))\
+			if (level::_sev <= enabled.load(std::memory_order_relaxed)) [[unlikely]]\
 			{\
 				static constexpr fmt_type_info type_list[sizeof...(Args) + 1]{fmt_type_info::make<fmt_unveil_t<Args>>()...};\
 				msg_##_sev.broadcast(reinterpret_cast<const char*>(+fmt), type_list, u64{fmt_unveil<Args>::get(args)}...);\

Utilities/cfmt.h

@@ -136,7 +136,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 	case '8':
 	case '9':
 	{
-		if (UNLIKELY(ctx.width))
+		if (ctx.width) [[unlikely]]
 		{
 			drop_sequence();
 		}
@@ -150,7 +150,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case '*':
 	{
-		if (UNLIKELY(ctx.width || !src.test(ctx.args)))
+		if (ctx.width || !src.test(ctx.args)) [[unlikely]]
 		{
 			drop_sequence();
 		}
@@ -166,7 +166,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case '.':
 	{
-		if (UNLIKELY(ctx.dot || ctx.prec))
+		if (ctx.dot || ctx.prec) [[unlikely]]
 		{
 			drop_sequence();
 		}
@@ -177,7 +177,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 		}
 		else if (*fmt == '*')
 		{
-			if (UNLIKELY(!src.test(ctx.args)))
+			if (!src.test(ctx.args)) [[unlikely]]
 			{
 				drop_sequence();
 			}
@@ -200,7 +200,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 'h':
 	{
-		if (UNLIKELY(ctx.type))
+		if (ctx.type) [[unlikely]]
 		{
 			drop_sequence();
 		}
@@ -219,7 +219,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 'l':
 	{
-		if (UNLIKELY(ctx.type))
+		if (ctx.type) [[unlikely]]
 		{
 			drop_sequence();
 		}
@@ -238,7 +238,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 'z':
 	{
-		if (UNLIKELY(ctx.type))
+		if (ctx.type) [[unlikely]]
 		{
 			drop_sequence();
 		}
@@ -252,7 +252,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 'j':
 	{
-		if (UNLIKELY(ctx.type))
+		if (ctx.type) [[unlikely]]
 		{
 			drop_sequence();
 		}
@@ -266,7 +266,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 't':
 	{
-		if (UNLIKELY(ctx.type))
+		if (ctx.type) [[unlikely]]
 		{
 			drop_sequence();
 		}
@@ -280,7 +280,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 'c':
 	{
-		if (UNLIKELY(ctx.type || !src.test(ctx.args)))
+		if (ctx.type || !src.test(ctx.args)) [[unlikely]]
 		{
 			drop_sequence();
 			break;
@@ -302,7 +302,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 's':
 	{
-		if (UNLIKELY(ctx.type || !src.test(ctx.args)))
+		if (ctx.type || !src.test(ctx.args)) [[unlikely]]
 		{
 			drop_sequence();
 			break;
@@ -333,7 +333,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 	case 'd':
 	case 'i':
 	{
-		if (UNLIKELY(!src.test(ctx.args)))
+		if (!src.test(ctx.args)) [[unlikely]]
 		{
 			drop_sequence();
 			break;
@@ -395,7 +395,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 'o':
 	{
-		if (UNLIKELY(!src.test(ctx.args)))
+		if (!src.test(ctx.args)) [[unlikely]]
 		{
 			drop_sequence();
 			break;
@@ -452,7 +452,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 	case 'x':
 	case 'X':
 	{
-		if (UNLIKELY(!src.test(ctx.args)))
+		if (!src.test(ctx.args)) [[unlikely]]
 		{
 			drop_sequence();
 			break;
@@ -516,7 +516,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 'u':
 	{
-		if (UNLIKELY(!src.test(ctx.args)))
+		if (!src.test(ctx.args)) [[unlikely]]
 		{
 			drop_sequence();
 			break;
@@ -563,7 +563,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 
 	case 'p':
 	{
-		if (UNLIKELY(!src.test(ctx.args) || ctx.type))
+		if (!src.test(ctx.args) || ctx.type) [[unlikely]]
 		{
 			drop_sequence();
 			break;
@@ -597,7 +597,7 @@ std::size_t cfmt_append(Dst& out, const Char* fmt, Src&& src)
 	case 'g':
 	case 'G':
 	{
-		if (UNLIKELY(!src.test(ctx.args) || ctx.type))
+		if (!src.test(ctx.args) || ctx.type) [[unlikely]]
 		{
 			drop_sequence();
 			break;

Utilities/lockless.h

@@ -22,7 +22,7 @@ public:
 
 	~lf_array()
 	{
-		for (auto ptr = m_next.raw(); UNLIKELY(ptr);)
+		for (auto ptr = m_next.raw(); ptr;)
 		{
 			delete std::exchange(ptr, std::exchange(ptr->m_next.raw(), nullptr));
 		}
@@ -30,14 +30,14 @@ public:
 
 	T& operator [](std::size_t index)
 	{
-		if (LIKELY(index < N))
+		if (index < N) [[likely]]
 		{
 			return m_data[index];
 		}
-		else if (UNLIKELY(!m_next))
+		else if (!m_next) [[unlikely]]
 		{
 			// Create new array block. It's not a full-fledged once-synchronization, unlikely needed.
-			for (auto _new = new lf_array, ptr = this; UNLIKELY(ptr);)
+			for (auto _new = new lf_array, ptr = this; ptr;)
 			{
 				// Install the pointer. If failed, go deeper.
 				ptr = ptr->m_next.compare_and_swap(nullptr, _new);
@@ -121,7 +121,7 @@ public:
 	// Access element (added implicitly)
 	T& operator [](const K& key)
 	{
-		if (UNLIKELY(key == K{}))
+		if (key == K{}) [[unlikely]]
 		{
 			return m_default_key_data;
 		}
@@ -133,7 +133,7 @@ public:
 			auto& pair = m_data[pos];
 
 			// Check the key value (optimistic)
-			if (LIKELY(pair.key == key) || pair.key.compare_and_swap_test(K{}, key))
+			if (pair.key == key || pair.key.compare_and_swap_test(K{}, key)) [[likely]]
 			{
 				return pair.value;
 			}

Utilities/mutex.h

@@ -45,7 +45,7 @@ public:
 	{
 		const u32 value = m_value.load();
 
-		if (UNLIKELY(value >= c_one - 1 || !m_value.compare_and_swap_test(value, value + 1)))
+		if (value >= c_one - 1 || !m_value.compare_and_swap_test(value, value + 1)) [[unlikely]]
 		{
 			imp_lock_shared(value);
 		}
@@ -55,10 +55,10 @@ public:
 	{
 		const u32 value = m_value.load();
 
-		if (LIKELY(value < c_one - 1))
+		if (value < c_one - 1) [[likely]]
 		{
 			u32 old = value;
-			if (LIKELY(atomic_storage<u32>::compare_exchange_hle_acq(m_value.raw(), old, value + 1)))
+			if (atomic_storage<u32>::compare_exchange_hle_acq(m_value.raw(), old, value + 1)) [[likely]]
 			{
 				return;
 			}
@@ -72,7 +72,7 @@ public:
 		// Unconditional decrement (can result in broken state)
 		const u32 value = m_value.fetch_sub(1);
 
-		if (UNLIKELY(value >= c_one))
+		if (value >= c_one) [[unlikely]]
 		{
 			imp_unlock_shared(value);
 		}
@@ -82,7 +82,7 @@ public:
 	{
 		const u32 value = atomic_storage<u32>::fetch_add_hle_rel(m_value.raw(), -1);
 
-		if (UNLIKELY(value >= c_one))
+		if (value >= c_one) [[unlikely]]
 		{
 			imp_unlock_shared(value);
 		}
@@ -100,7 +100,7 @@ public:
 	{
 		const u32 value = m_value.load();
 
-		if (UNLIKELY(value >= c_vip - 1 || !m_value.compare_and_swap_test(value, value + 1)))
+		if (value >= c_vip - 1 || !m_value.compare_and_swap_test(value, value + 1)) [[unlikely]]
 		{
 			imp_lock_low(value);
 		}
@@ -111,7 +111,7 @@ public:
 		// Unconditional decrement (can result in broken state)
 		const u32 value = m_value.fetch_sub(1);
 
-		if (UNLIKELY(value >= c_one))
+		if (value >= c_one) [[unlikely]]
 		{
 			imp_unlock_low(value);
 		}
@@ -129,7 +129,7 @@ public:
 	{
 		const u32 value = m_value.load();
 
-		if (UNLIKELY((value >= c_one - 1 && !(value & (c_one - c_vip))) || (value % c_vip) || !m_value.compare_and_swap_test(value, value + c_vip)))
+		if ((value >= c_one - 1 && !(value & (c_one - c_vip))) || (value % c_vip) || !m_value.compare_and_swap_test(value, value + c_vip)) [[unlikely]]
 		{
 			imp_lock_vip(value);
 		}
@@ -140,7 +140,7 @@ public:
 		// Unconditional decrement (can result in broken state)
 		const u32 value = m_value.fetch_sub(c_vip);
 
-		if (UNLIKELY(value >= c_one))
+		if (value >= c_one) [[unlikely]]
 		{
 			imp_unlock_vip(value);
 		}
@@ -155,7 +155,7 @@ public:
 	{
 		const u32 value = m_value.compare_and_swap(0, c_one);
 
-		if (UNLIKELY(value))
+		if (value) [[unlikely]]
 		{
 			imp_lock(value);
 		}
@@ -165,7 +165,7 @@ public:
 	{
 		u32 value = 0;
 
-		if (UNLIKELY(!atomic_storage<u32>::compare_exchange_hle_acq(m_value.raw(), value, c_one)))
+		if (!atomic_storage<u32>::compare_exchange_hle_acq(m_value.raw(), value, c_one)) [[unlikely]]
 		{
 			imp_lock(value);
 		}
@@ -176,7 +176,7 @@ public:
 		// Unconditional decrement (can result in broken state)
 		const u32 value = m_value.fetch_sub(c_one);
 
-		if (UNLIKELY(value != c_one))
+		if (value != c_one) [[unlikely]]
 		{
 			imp_unlock(value);
 		}
@@ -186,7 +186,7 @@ public:
 	{
 		const u32 value = atomic_storage<u32>::fetch_add_hle_rel(m_value.raw(), 0u - c_one);
 
-		if (UNLIKELY(value != c_one))
+		if (value != c_one) [[unlikely]]
 		{
 			imp_unlock(value);
 		}
@@ -202,7 +202,7 @@ public:
 
 	void lock_upgrade()
 	{
-		if (UNLIKELY(!try_lock_upgrade()))
+		if (!try_lock_upgrade()) [[unlikely]]
 		{
 			imp_lock_upgrade();
 		}
@@ -223,7 +223,7 @@ public:
 	// Optimized wait for lockability without locking, relaxed
 	void lock_unlock()
 	{
-		if (UNLIKELY(m_value != 0))
+		if (m_value != 0) [[unlikely]]
 		{
 			imp_lock_unlock();
 		}

Utilities/sema.h

@@ -26,7 +26,7 @@ protected:
 		const s32 value = m_value.load();
 
 		// Conditional decrement
-		if (UNLIKELY(value <= 0 || !m_value.compare_and_swap_test(value, value - 1)))
+		if (value <= 0 || !m_value.compare_and_swap_test(value, value - 1)) [[unlikely]]
 		{
 			imp_wait();
 		}
@@ -42,7 +42,7 @@ protected:
 		// Unconditional increment
 		const s32 value = m_value.fetch_add(1);
 
-		if (UNLIKELY(value < 0 || value >= _max))
+		if (value < 0 || value >= _max) [[unlikely]]
 		{
 			imp_post(value);
 		}

Utilities/typemap.h

@@ -658,7 +658,7 @@ namespace utils
 					static_assert(!std::is_volatile_v<std::remove_reference_t<Type>>);
 
 					// Try to acquire the semaphore
-					if (UNLIKELY(!head->m_sema.try_inc(last + 1)))
+					if (!head->m_sema.try_inc(last + 1)) [[unlikely]]
 					{
 						block = nullptr;
 					}
@@ -671,7 +671,7 @@ namespace utils
 
 							if (block->m_type == 0 && block->m_mutex.try_lock())
 							{
-								if (LIKELY(block->m_type == 0))
+								if (block->m_type == 0) [[likely]]
 								{
 									break;
 								}
@@ -695,7 +695,7 @@ namespace utils
 						{
 							block = nullptr;
 						}
-						else if (UNLIKELY(block->m_type != 0))
+						else if (block->m_type != 0) [[unlikely]]
 						{
 							block->m_mutex.unlock();
 							block = nullptr;
@@ -737,13 +737,13 @@ namespace utils
 				block = reinterpret_cast<typemap_block*>(head->m_ptr + std::size_t{head->m_ssize} * unscaled);
 
 				// Check id range and type
-				if (UNLIKELY(unscaled >= typeinfo_count<Type>::max_count || unbiased % step))
+				if (unscaled >= typeinfo_count<Type>::max_count || unbiased % step) [[unlikely]]
 				{
 					block = nullptr;
 				}
 				else
 				{
-					if (UNLIKELY(block->m_type == 0))
+					if (block->m_type == 0) [[unlikely]]
 					{
 						block = nullptr;
 					}
@@ -785,12 +785,12 @@ namespace utils
 			}
 			else if constexpr (std::is_invocable_r_v<bool, const Arg&, const Type&>)
 			{
-				if (UNLIKELY(!block))
+				if (!block) [[unlikely]]
 				{
 					return;
 				}
 
-				if (LIKELY(block->m_type))
+				if (block->m_type) [[likely]]
 				{
 					if (std::invoke(std::forward<Arg>(id), std::as_const(*block->get_ptr<Type>())))
 					{
@@ -800,7 +800,7 @@ namespace utils
 			}
 			else if (block)
 			{
-				if (LIKELY(block->m_type))
+				if (block->m_type) [[likely]]
 				{
 					return;
 				}
@@ -848,7 +848,7 @@ namespace utils
 				}
 				else if constexpr (is_const || is_volatile)
 				{
-					if (LIKELY(block->m_mutex.is_lockable()))
+					if (block->m_mutex.is_lockable()) [[likely]]
 					{
 						return true;
 					}
@@ -858,7 +858,7 @@ namespace utils
 				}
 				else
 				{
-					if (LIKELY(block->m_mutex.is_free()))
+					if (block->m_mutex.is_free()) [[likely]]
 					{
 						return true;
 					}
@@ -878,7 +878,7 @@ namespace utils
 				if constexpr (I + 1 < N)
 				{
 					// Proceed recursively
-					if (LIKELY(try_lock<I + 1, Types...>(array, locked, std::integer_sequence<bool, Locks...>{})))
+					if (try_lock<I + 1, Types...>(array, locked, std::integer_sequence<bool, Locks...>{})) [[likely]]
 					{
 						return true;
 					}
@@ -973,7 +973,7 @@ namespace utils
 			while (true)
 			{
 				const uint locked = lock_array<decode_t<Types>...>(result, seq_t{}, locks_t{});
-				if (LIKELY(try_lock<0, decode_t<Types>...>(result, locked, locks_t{})))
+				if (try_lock<0, decode_t<Types>...>(result, locked, locks_t{})) [[likely]]
 					break;
 			}
 

Utilities/types.h

@@ -28,8 +28,6 @@
 #ifdef _MSC_VER
 
 #define ASSUME(...) __assume(__VA_ARGS__) // MSVC __assume ignores side-effects
-#define LIKELY
-#define UNLIKELY
 #define SAFE_BUFFERS __declspec(safebuffers)
 #define NEVER_INLINE __declspec(noinline)
 #define FORCE_INLINE __forceinline
@@ -48,8 +46,6 @@
 #define ASSUME(...) do { if (!(__VA_ARGS__)) __builtin_unreachable(); } while (0)  // note: the compiler will generate code to evaluate "cond" if the expression is opaque
 #endif
 
-#define LIKELY(...) __builtin_expect(!!(__VA_ARGS__), 1)
-#define UNLIKELY(...) __builtin_expect(!!(__VA_ARGS__), 0)
 #define SAFE_BUFFERS
 #define NEVER_INLINE __attribute__((noinline))
 #define FORCE_INLINE __attribute__((always_inline)) inline