Partial commit: Utilities

2016-02-02 00:55:43 +03:00
parent 5fc6f59821
commit 250ce63527
37 changed files with 4783 additions and 4007 deletions
@@ -1,49 +1,156 @@
 #pragma once

+#include <cstdint>
+#include <exception>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+
+#include "Atomic.h"
+
 //! An attempt to create effective implementation of "shared mutex", lock-free in optimistic case.
 //! All locking and unlocking may be done by single LOCK XADD or LOCK CMPXCHG instructions.
 //! MSVC implementation of std::shared_timed_mutex seems suboptimal.
 //! std::shared_mutex is not available until C++17.
 class shared_mutex final
 {
-	enum : u32
-	{
-		SM_WRITER_LOCK  = 1u << 31, // Exclusive lock flag, must be MSB
-		SM_WRITER_QUEUE = 1u << 30, // Flag set if m_wq_size != 0
-		SM_READER_QUEUE = 1u << 29, // Flag set if m_rq_size != 0
+	using ctrl_type = u32;

-		SM_READER_COUNT = SM_READER_QUEUE - 1, // Valid reader count bit mask
-		SM_READER_MAX   = 1u << 24, // Max reader count
+	enum : ctrl_type
+	{
+		SM_WRITER_LOCK = 1u << 31, // Exclusive lock flag, must be MSB
+		SM_WAITERS_BIT = 1u << 30, // Flag set if m_wq_size or m_rq_size is non-zero
+		SM_INVALID_BIT = 1u << 29, // Unreachable reader count bit (may be set by incorrect unlock_shared() call)
+
+		SM_READER_MASK = SM_WAITERS_BIT - 1, // Valid reader count bit mask
+		SM_READER_MAX  = 1u << 24, // Max reader count
 	};

-	std::atomic<u32> m_ctrl{}; // Control atomic variable: reader count | SM_* flags
-	std::thread::id m_owner{}; // Current exclusive owner (TODO: implement only for debug mode?)
+	atomic_t<ctrl_type> m_ctrl{}; // Control atomic variable: reader count | SM_* flags

 	std::mutex m_mutex;

-	u32 m_rq_size{}; // Reader queue size (threads waiting on m_rcv)
-	u32 m_wq_size{}; // Writer queue size (threads waiting on m_wcv+m_ocv)
+	std::size_t m_rq_size{}; // Reader queue size (threads waiting on m_rcv)
+	std::size_t m_wq_size{}; // Writer queue size (threads waiting on m_wcv and m_ocv)
 	
 	std::condition_variable m_rcv; // Reader queue
 	std::condition_variable m_wcv; // Writer queue
 	std::condition_variable m_ocv; // For current exclusive owner

-	static bool op_lock_shared(u32& ctrl)
+	void lock_shared_hard()
 	{
-		// Check writer flags and reader limit
-		return (ctrl & ~SM_READER_QUEUE) < SM_READER_MAX ? ctrl++, true : false;
+		std::unique_lock<std::mutex> lock(m_mutex);
+
+		// Validate
+		if ((m_ctrl & SM_INVALID_BIT) != 0) throw std::runtime_error("shared_mutex::lock_shared(): Invalid bit");
+		if ((m_ctrl & SM_READER_MASK) == 0) throw std::runtime_error("shared_mutex::lock_shared(): No readers");
+
+		// Notify non-zero reader queue size
+		m_ctrl |= SM_WAITERS_BIT, m_rq_size++;
+
+		// Fix excess reader count
+		if ((--m_ctrl & SM_READER_MASK) == 0 && m_wq_size)
+		{
+			// Notify exclusive owner
+			m_ocv.notify_one();
+		}
+
+		// Obtain the reader lock
+		while (true)
+		{
+			const auto ctrl = m_ctrl.load();
+
+			// Check writers and reader limit
+			if (m_wq_size || (ctrl & ~SM_WAITERS_BIT) >= SM_READER_MAX)
+			{
+				m_rcv.wait(lock);
+				continue;
+			}
+
+			if (m_ctrl.compare_and_swap_test(ctrl, ctrl + 1))
+			{
+				break;
+			}
+		}
+
+		if (!--m_rq_size && !m_wq_size)
+		{
+			m_ctrl &= ~SM_WAITERS_BIT;
+		}
 	}

-	static bool op_lock_excl(u32& ctrl)
+	void unlock_shared_notify()
 	{
-		// Test and set writer lock
-		return (ctrl & SM_WRITER_LOCK) == 0 ? ctrl |= SM_WRITER_LOCK, true : false;
+		// Mutex is locked for reliable notification because m_ctrl has been changed outside
+		std::lock_guard<std::mutex> lock(m_mutex);
+
+		if ((m_ctrl & SM_READER_MASK) == 0 && m_wq_size)
+		{
+			// Notify exclusive owner
+			m_ocv.notify_one();
+		}
+		else if (m_rq_size)
+		{
+			// Notify other readers
+			m_rcv.notify_one();
+		}
 	}

-	void impl_lock_shared(u32 old_ctrl);
-	void impl_unlock_shared(u32 new_ctrl);
-	void impl_lock_excl(u32 ctrl);
-	void impl_unlock_excl(u32 ctrl);
+	void lock_hard()
+	{
+		std::unique_lock<std::mutex> lock(m_mutex);
+
+		// Validate
+		if ((m_ctrl & SM_INVALID_BIT) != 0) throw std::runtime_error("shared_mutex::lock(): Invalid bit");
+
+		// Notify non-zero writer queue size
+		m_ctrl |= SM_WAITERS_BIT, m_wq_size++;
+
+		// Obtain the writer lock
+		while (true)
+		{
+			const auto ctrl = m_ctrl.load();
+
+			if (ctrl & SM_WRITER_LOCK)
+			{
+				m_wcv.wait(lock);
+				continue;
+			}
+
+			if (m_ctrl.compare_and_swap_test(ctrl, ctrl | SM_WRITER_LOCK))
+			{
+				break;
+			}
+		}
+
+		// Wait for remaining readers
+		while ((m_ctrl & SM_READER_MASK) != 0)
+		{
+			m_ocv.wait(lock);
+		}
+
+		if (!--m_wq_size && !m_rq_size)
+		{
+			m_ctrl &= ~SM_WAITERS_BIT;
+		}
+	}
+
+	void unlock_notify()
+	{
+		// Mutex is locked for reliable notification because m_ctrl has been changed outside
+		std::lock_guard<std::mutex> lock(m_mutex);
+
+		if (m_wq_size)
+		{
+			// Notify next exclusive owner
+			m_wcv.notify_one();
+		}
+		else if (m_rq_size)
+		{
+			// Notify all readers
+			m_rcv.notify_all();
+		}
+	}

 public:
 	shared_mutex() = default;
@@ -51,65 +158,49 @@ public:
 	// Lock in shared mode
 	void lock_shared()
 	{
-		const u32 old_ctrl = m_ctrl++;
-
-		// Check flags and reader limit
-		if (old_ctrl >= SM_READER_MAX)
+		if (m_ctrl++ >= SM_READER_MAX)
 		{
-			impl_lock_shared(old_ctrl);
+			lock_shared_hard();
 		}
 	}

 	// Try to lock in shared mode
 	bool try_lock_shared()
 	{
-		return atomic_op(m_ctrl, [](u32& ctrl)
-		{
-			// Check flags and reader limit
-			return ctrl < SM_READER_MAX ? ctrl++, true : false;
-		});
+		auto ctrl = m_ctrl.load();
+
+		return ctrl < SM_READER_MAX && m_ctrl.compare_and_swap_test(ctrl, ctrl + 1);
 	}

 	// Unlock in shared mode
 	void unlock_shared()
 	{
-		const u32 new_ctrl = --m_ctrl;
-
-		// Check if notification required
-		if (new_ctrl >= SM_READER_MAX)
+		if (m_ctrl-- >= SM_READER_MAX)
 		{
-			impl_unlock_shared(new_ctrl);
-		}
-	}
-
-	// Lock exclusively
-	void lock()
-	{
-		u32 value = 0;
-
-		if (!m_ctrl.compare_exchange_strong(value, SM_WRITER_LOCK))
-		{
-			impl_lock_excl(value);
+			unlock_shared_notify();
 		}
 	}

 	// Try to lock exclusively
 	bool try_lock()
 	{
-		u32 value = 0;
+		return m_ctrl.compare_and_swap_test(0, SM_WRITER_LOCK);
+	}

-		return m_ctrl.compare_exchange_strong(value, SM_WRITER_LOCK);
+	// Lock exclusively
+	void lock()
+	{
+		if (m_ctrl.compare_and_swap_test(0, SM_WRITER_LOCK)) return;
+
+		lock_hard();		
 	}

 	// Unlock exclusively
 	void unlock()
 	{
-		const u32 value = m_ctrl.fetch_add(SM_WRITER_LOCK);
-
-		// Check if notification required
-		if (value != SM_WRITER_LOCK)
+		if (m_ctrl.fetch_sub(SM_WRITER_LOCK) != SM_WRITER_LOCK)
 		{
-			impl_unlock_excl(value);
+			unlock_notify();
 		}
 	}
 };