From af309de438276f1ac723e2da90ccec5279e95968 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Thu, 13 Aug 2026 20:18:18 +1000 Subject: [PATCH] [idd] common: centralize atomic operations --- idd/LGCommon/Atomic.h | 188 ++++++++++++++++++ idd/LGCommon/CPipeEndpoint.cpp | 24 +-- idd/LGCommon/CPipeEndpoint.h | 6 +- idd/LGCommon/LGCommon.vcxproj | 1 + idd/LGCommon/LGCommon.vcxproj.filters | 3 + idd/LGIdd/capture/CFrameBufferResource.h | 11 +- idd/LGIdd/capture/CSwapChainProcessor.cpp | 7 +- idd/LGIdd/capture/CSwapChainProcessor.h | 2 +- idd/LGIdd/d3d/CD3D12CommandQueue.cpp | 63 +++--- idd/LGIdd/d3d/CD3D12CommandQueue.h | 8 +- idd/LGIdd/display/CDeviceContext.cpp | 17 +- idd/LGIdd/display/CDeviceContext.h | 3 +- idd/LGIdd/display/CMonitorContext.cpp | 7 +- idd/LGIdd/display/CMonitorContext.h | 5 +- idd/LGIdd/display/CMonitorManager.cpp | 13 +- idd/LGIdd/display/CMonitorManager.h | 2 +- idd/LGIdd/ipc/CInputPipeServer.cpp | 43 ++-- idd/LGIdd/ipc/CInputPipeServer.h | 4 +- idd/LGIdd/transport/CFrameHub.cpp | 165 ++++++++------- idd/LGIdd/transport/CFrameHub.h | 3 +- .../transport/lgmp/CLGMPFrameTransport.cpp | 73 ++++--- .../transport/lgmp/CLGMPFrameTransport.h | 4 +- .../transport/lgmp/CLGMPInputTransport.cpp | 12 +- .../transport/lgmp/CLGMPInputTransport.h | 2 +- idd/LGIdd/transport/lgmp/CLGMPTransport.cpp | 7 +- idd/LGIdd/transport/lgmp/CLGMPTransport.h | 3 +- idd/LGIdd/transport/lgmp/CRecovery.cpp | 72 ++----- 27 files changed, 477 insertions(+), 271 deletions(-) create mode 100644 idd/LGCommon/Atomic.h diff --git a/idd/LGCommon/Atomic.h b/idd/LGCommon/Atomic.h new file mode 100644 index 00000000..84ede1e6 --- /dev/null +++ b/idd/LGCommon/Atomic.h @@ -0,0 +1,188 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#pragma once + +#include "Seq.h" + +#include + +#include +#include + +namespace Atomic +{ + namespace Detail + { + inline volatile LONG * Ptr(uint32_t& value) + { + static_assert(sizeof(value) == sizeof(LONG), + "atomic value must match the Windows interlocked width"); + return reinterpret_cast(&value); + } + } + + template + T Load(const std::atomic& value, + std::memory_order order = std::memory_order_seq_cst) + { + return value.load(order); + } + + template + void Store(std::atomic& value, U data, + std::memory_order order = std::memory_order_seq_cst) + { + value.store(static_cast(data), order); + } + + template + T Swap(std::atomic& value, U data, + std::memory_order order = std::memory_order_seq_cst) + { + return value.exchange(static_cast(data), order); + } + + template + T FetchAdd(std::atomic& value, U data, + std::memory_order order = std::memory_order_seq_cst) + { + return value.fetch_add(static_cast(data), order); + } + + template + T FetchSub(std::atomic& value, U data, + std::memory_order order = std::memory_order_seq_cst) + { + return value.fetch_sub(static_cast(data), order); + } + + template + bool CAS(std::atomic& value, T& expected, U data, + std::memory_order order = std::memory_order_seq_cst) + { + return value.compare_exchange_strong( + expected, static_cast(data), order); + } + + template + bool CAS(std::atomic& value, T& expected, U data, + std::memory_order success, std::memory_order failure) + { + return value.compare_exchange_strong( + expected, static_cast(data), success, failure); + } + + template + bool CASWeak(std::atomic& value, T& expected, U data, + std::memory_order order = std::memory_order_seq_cst) + { + return value.compare_exchange_weak( + expected, static_cast(data), order); + } + + template + bool CASWeak(std::atomic& value, T& expected, U data, + std::memory_order success, std::memory_order failure) + { + return value.compare_exchange_weak( + expected, static_cast(data), success, failure); + } + + template + T Inc(std::atomic& value, + std::memory_order order = std::memory_order_seq_cst) + { + return FetchAdd(value, static_cast(1), order) + 1; + } + + template + T Next(std::atomic& value, + std::memory_order order = std::memory_order_relaxed) + { + T current = Load(value, std::memory_order_relaxed); + for (;;) + { + const T next = Seq::Next(current); + if (CASWeak(value, current, next, order)) + return next; + } + } + + inline uint32_t Load(uint32_t& value) + { + return static_cast(InterlockedCompareExchange( + Detail::Ptr(value), 0, 0)); + } + + inline void Store(uint32_t& value, uint32_t data) + { + InterlockedExchange(Detail::Ptr(value), static_cast(data)); + } + + inline uint32_t Swap(uint32_t& value, uint32_t data) + { + return static_cast( + InterlockedExchange(Detail::Ptr(value), static_cast(data))); + } + + inline uint32_t FetchAdd(uint32_t& value, uint32_t data) + { + return static_cast( + InterlockedExchangeAdd(Detail::Ptr(value), static_cast(data))); + } + + inline uint32_t FetchSub(uint32_t& value, uint32_t data) + { + return static_cast(InterlockedExchangeAdd( + Detail::Ptr(value), static_cast(0U - data))); + } + + inline bool CAS(uint32_t& value, uint32_t expected, uint32_t data) + { + const uint32_t actual = static_cast(InterlockedCompareExchange( + Detail::Ptr(value), static_cast(data), + static_cast(expected))); + return actual == expected; + } + + inline bool CASWeak(uint32_t& value, uint32_t expected, uint32_t data) + { + return CAS(value, expected, data); + } + + inline uint32_t Inc(uint32_t& value) + { + return static_cast(InterlockedIncrement(Detail::Ptr(value))); + } + + inline uint32_t Next(uint32_t& value, uint32_t step = 1) + { + uint32_t result = FetchAdd(value, step) + step; + if (!result) + result = FetchAdd(value, step) + step; + return result; + } + + inline void Fence() + { + MemoryBarrier(); + } +} diff --git a/idd/LGCommon/CPipeEndpoint.cpp b/idd/LGCommon/CPipeEndpoint.cpp index b8290b79..1b2003c4 100644 --- a/idd/LGCommon/CPipeEndpoint.cpp +++ b/idd/LGCommon/CPipeEndpoint.cpp @@ -217,12 +217,12 @@ bool CPipeEndpoint::Start( PublishPipe(pipe); } - m_running.store(true); + Atomic::Store(m_running, true); m_thread = CreateThread(nullptr, 0, ThreadProc, this, 0, nullptr); if (!m_thread) { DEBUG_ERROR_HR(GetLastError(), "Failed to create named pipe thread"); - m_running.store(false); + Atomic::Store(m_running, false); { CSRWExclusiveLock lock(m_pipeLock); @@ -245,8 +245,8 @@ bool CPipeEndpoint::Start( void CPipeEndpoint::Stop() { - m_running.store(false); - m_connected.store(false); + Atomic::Store(m_running, false); + Atomic::Store(m_connected, false); if (m_stopEvent) SetEvent(m_stopEvent); @@ -284,7 +284,7 @@ void CPipeEndpoint::Stop() m_writeEvent = nullptr; } - m_connected.store(false); + Atomic::Store(m_connected, false); } bool CPipeEndpoint::Send(const void * message, size_t size) @@ -303,7 +303,7 @@ bool CPipeEndpoint::Send(const void * message, size_t size) success = result == PipeIoResult::Success; if (!success) { - m_connected.store(false); + Atomic::Store(m_connected, false); CancelIoEx(m_pipe, nullptr); } } @@ -323,8 +323,8 @@ void CPipeEndpoint::Thread() else RunClient(); - m_running.store(false); - m_connected.store(false); + Atomic::Store(m_running, false); + Atomic::Store(m_connected, false); } HANDLE CPipeEndpoint::CreateServerPipe() @@ -421,14 +421,14 @@ void CPipeEndpoint::RunServer() WaitForSingleObject(m_stopEvent, 0) == WAIT_FIRST_OBJECT_VALUE) break; - m_connected.store(true); + Atomic::Store(m_connected, true); DEBUG_INFO("Named pipe client connected: %ls", m_pipeName.c_str()); if (m_handler) m_handler->OnPipeConnected(); ReadMessages(pipe); - m_connected.store(false); + Atomic::Store(m_connected, false); if (m_handler) m_handler->OnPipeDisconnected(); DEBUG_INFO("Named pipe client disconnected: %ls", m_pipeName.c_str()); @@ -504,7 +504,7 @@ void CPipeEndpoint::RunClient() } PublishPipe(pipe); - m_connected.store(true); + Atomic::Store(m_connected, true); retryDelay = CLIENT_RETRY_INITIAL_MS; lastConnectError = ERROR_SUCCESS; DEBUG_INFO("Named pipe connected: %ls", m_pipeName.c_str()); @@ -513,7 +513,7 @@ void CPipeEndpoint::RunClient() ReadMessages(pipe); - m_connected.store(false); + Atomic::Store(m_connected, false); if (m_handler) m_handler->OnPipeDisconnected(); DEBUG_INFO("Named pipe disconnected: %ls", m_pipeName.c_str()); diff --git a/idd/LGCommon/CPipeEndpoint.h b/idd/LGCommon/CPipeEndpoint.h index 761cf24f..2c0c07e2 100644 --- a/idd/LGCommon/CPipeEndpoint.h +++ b/idd/LGCommon/CPipeEndpoint.h @@ -20,11 +20,11 @@ #pragma once +#include "Atomic.h" #include "CSRWLock.h" #include -#include #include #include @@ -66,8 +66,8 @@ public: _In_reads_bytes_(size) const void * message, _In_ size_t size); - bool IsRunning() const { return m_running.load(); } - bool IsConnected() const { return m_connected.load(); } + bool IsRunning() const { return Atomic::Load(m_running); } + bool IsConnected() const { return Atomic::Load(m_connected); } void SetHandler(_In_opt_ IPipeEndpointHandler * handler) { diff --git a/idd/LGCommon/LGCommon.vcxproj b/idd/LGCommon/LGCommon.vcxproj index a8b5a342..c331f8ab 100644 --- a/idd/LGCommon/LGCommon.vcxproj +++ b/idd/LGCommon/LGCommon.vcxproj @@ -70,6 +70,7 @@ + diff --git a/idd/LGCommon/LGCommon.vcxproj.filters b/idd/LGCommon/LGCommon.vcxproj.filters index dee4af59..4a003b07 100644 --- a/idd/LGCommon/LGCommon.vcxproj.filters +++ b/idd/LGCommon/LGCommon.vcxproj.filters @@ -23,6 +23,9 @@ + + Header Files + Header Files diff --git a/idd/LGIdd/capture/CFrameBufferResource.h b/idd/LGIdd/capture/CFrameBufferResource.h index b9a587cc..15035080 100644 --- a/idd/LGIdd/capture/CFrameBufferResource.h +++ b/idd/LGIdd/capture/CFrameBufferResource.h @@ -20,11 +20,12 @@ #pragma once +#include "Atomic.h" + #include #include #include #include -#include #include #include "capture/CFrameScheduler.h" @@ -112,15 +113,17 @@ class CFrameBufferResource } void ResetCompletion() { - m_completionHandled.store(false, std::memory_order_release); + Atomic::Store( + m_completionHandled, false, std::memory_order_release); } void MarkCompletion() { - m_completionHandled.store(true, std::memory_order_release); + Atomic::Store( + m_completionHandled, true, std::memory_order_release); } bool CompletionHandled() const { - return m_completionHandled.load(std::memory_order_acquire); + return Atomic::Load(m_completionHandled, std::memory_order_acquire); } void SetCandidateIndex(unsigned index) { m_candidateIndex = index; } unsigned GetCandidateIndex() const { return m_candidateIndex; } diff --git a/idd/LGIdd/capture/CSwapChainProcessor.cpp b/idd/LGIdd/capture/CSwapChainProcessor.cpp index 76ebee78..70815bf3 100644 --- a/idd/LGIdd/capture/CSwapChainProcessor.cpp +++ b/idd/LGIdd/capture/CSwapChainProcessor.cpp @@ -20,6 +20,7 @@ #include "capture/CSwapChainProcessor.h" #include "capture/CFrameProcessorUtil.h" +#include "Atomic.h" #include "CSRWLock.h" #include "display/IddCxCompat.h" #include "display/CDeviceContext.h" @@ -355,7 +356,8 @@ void CSwapChainProcessor::SwapChainThreadCore() surface = buffer.MetaData.pSurface; colorSpace = buffer.MetaData.SurfaceColorSpace; sdrWhiteLevel = buffer.MetaData.SdrWhiteLevel; - m_sdrWhiteLevel.store(sdrWhiteLevel, std::memory_order_relaxed); + Atomic::Store( + m_sdrWhiteLevel, sdrWhiteLevel, std::memory_order_relaxed); UpdateHDRMetadata(buffer.MetaData); } } @@ -865,7 +867,8 @@ bool CSwapChainProcessor::QueryHWCursor() in.ShapeBufferSizeInBytes = 512 * 512 * 4; IDARG_OUT_QUERY_HWCURSOR out = {}; - UINT cursorWhiteLevel = m_sdrWhiteLevel.load(std::memory_order_relaxed); + UINT cursorWhiteLevel = + Atomic::Load(m_sdrWhiteLevel, std::memory_order_relaxed); NTSTATUS status; #ifdef HAS_IDDCX_110 if (m_devContext->HasIddCx110DDIs()) diff --git a/idd/LGIdd/capture/CSwapChainProcessor.h b/idd/LGIdd/capture/CSwapChainProcessor.h index 48964447..d33ce418 100644 --- a/idd/LGIdd/capture/CSwapChainProcessor.h +++ b/idd/LGIdd/capture/CSwapChainProcessor.h @@ -20,6 +20,7 @@ #pragma once +#include "Atomic.h" #include "d3d/CD3D11Device.h" #include "d3d/CD3D12Device.h" #include "display/IddCxCompat.h" @@ -30,7 +31,6 @@ #include #include -#include #include using namespace Microsoft::WRL; diff --git a/idd/LGIdd/d3d/CD3D12CommandQueue.cpp b/idd/LGIdd/d3d/CD3D12CommandQueue.cpp index 3dc12c5b..cba735a7 100644 --- a/idd/LGIdd/d3d/CD3D12CommandQueue.cpp +++ b/idd/LGIdd/d3d/CD3D12CommandQueue.cpp @@ -147,11 +147,12 @@ void CD3D12CommandSlot::DeInit() bool CD3D12CommandSlot::Acquire() { - if (!m_queue || m_queue->m_failed.load(std::memory_order_acquire)) + if (!m_queue || Atomic::Load( + m_queue->m_failed, std::memory_order_acquire)) return false; State expected = STATE_FREE; - if (!m_state.compare_exchange_strong(expected, STATE_RECORDING, + if (!Atomic::CAS(m_state, expected, STATE_RECORDING, std::memory_order_acq_rel)) return false; @@ -165,7 +166,7 @@ bool CD3D12CommandSlot::Acquire() m_timestampFrequency = 0; m_calibrationGPU = 0; m_calibrationCPU = 0; - m_submitted.store(false, std::memory_order_release); + Atomic::Store(m_submitted, false, std::memory_order_release); for (UINT i = 0; i < MAX_FENCE_WAITS; ++i) { @@ -180,8 +181,8 @@ bool CD3D12CommandSlot::Acquire() if (FAILED(hr)) { DEBUG_ERROR_HR(hr, "Failed to reset the CommandAllocator (%ls)", m_name); - m_queue->m_failed.store(true, std::memory_order_release); - m_state.store(STATE_FAILED, std::memory_order_release); + Atomic::Store(m_queue->m_failed, true, std::memory_order_release); + Atomic::Store(m_state, STATE_FAILED, std::memory_order_release); return false; } @@ -189,8 +190,8 @@ bool CD3D12CommandSlot::Acquire() if (FAILED(hr)) { DEBUG_ERROR_HR(hr, "Failed to reset the CommandList (%ls)", m_name); - m_queue->m_failed.store(true, std::memory_order_release); - m_state.store(STATE_FAILED, std::memory_order_release); + Atomic::Store(m_queue->m_failed, true, std::memory_order_release); + Atomic::Store(m_state, STATE_FAILED, std::memory_order_release); return false; } @@ -201,7 +202,7 @@ bool CD3D12CommandSlot::Acquire() void CD3D12CommandSlot::Cancel() { State expected = STATE_RECORDING; - if (!m_state.compare_exchange_strong(expected, STATE_CANCELLING, + if (!Atomic::CAS(m_state, expected, STATE_CANCELLING, std::memory_order_acq_rel)) { DEBUG_ERROR("Command slot cancelled while not recording (%ls)", m_name); @@ -223,23 +224,23 @@ void CD3D12CommandSlot::Cancel() { DEBUG_ERROR_HR(hr, "Failed to close the cancelled CommandList (%ls)", m_name); - m_queue->m_failed.store(true, std::memory_order_release); - m_state.store(STATE_FAILED, std::memory_order_release); + Atomic::Store(m_queue->m_failed, true, std::memory_order_release); + Atomic::Store(m_state, STATE_FAILED, std::memory_order_release); return; } m_completionCallback = nullptr; m_completionParams[0] = nullptr; m_completionParams[1] = nullptr; - m_submitted.store(false, std::memory_order_release); - m_state.store(STATE_FREE, std::memory_order_release); + Atomic::Store(m_submitted, false, std::memory_order_release); + Atomic::Store(m_state, STATE_FREE, std::memory_order_release); SetEvent(m_availableEvent.Get()); } bool CD3D12CommandSlot::Execute() { State expected = STATE_RECORDING; - if (!m_state.compare_exchange_strong(expected, STATE_SUBMITTED, + if (!Atomic::CAS(m_state, expected, STATE_SUBMITTED, std::memory_order_acq_rel)) { DEBUG_ERROR("Command slot executed while not recording (%ls)", m_name); @@ -251,17 +252,17 @@ bool CD3D12CommandSlot::Execute() if (FAILED(hr)) { DEBUG_ERROR_HR(hr, "Failed to close the CommandList (%ls)", m_name); - m_queue->m_failed.store(true, std::memory_order_release); - m_state.store(STATE_FAILED, std::memory_order_release); + Atomic::Store(m_queue->m_failed, true, std::memory_order_release); + Atomic::Store(m_state, STATE_FAILED, std::memory_order_release); return false; } if (m_queue->Submit(*this)) return true; - if (!m_submitted.load(std::memory_order_acquire)) + if (!Atomic::Load(m_submitted, std::memory_order_acquire)) { - m_state.store(STATE_FREE, std::memory_order_release); + Atomic::Store(m_state, STATE_FREE, std::memory_order_release); SetEvent(m_availableEvent.Get()); } return false; @@ -270,7 +271,7 @@ bool CD3D12CommandSlot::Execute() bool CD3D12CommandSlot::WaitFor(ID3D12Fence * fence, UINT64 value) { if (!fence || !value || - m_state.load(std::memory_order_acquire) != STATE_RECORDING) + Atomic::Load(m_state, std::memory_order_acquire) != STATE_RECORDING) return false; if (m_fenceWaitCount == MAX_FENCE_WAITS) @@ -331,7 +332,7 @@ bool CD3D12CommandSlot::GetGPUTimes( void CD3D12CommandSlot::OnCompletion(bool timeout) { - if (!m_queue || !m_submitted.load(std::memory_order_acquire)) + if (!m_queue || !Atomic::Load(m_submitted, std::memory_order_acquire)) return; const UINT64 completed = m_queue->m_fence->GetCompletedValue(); @@ -339,13 +340,13 @@ void CD3D12CommandSlot::OnCompletion(bool timeout) return; State expected = STATE_SUBMITTED; - if (!m_state.compare_exchange_strong(expected, STATE_COMPLETING, + if (!Atomic::CAS(m_state, expected, STATE_COMPLETING, std::memory_order_acq_rel)) return; m_completionResult = !timeout && completed != UINT64_MAX; if (!m_completionResult) - m_queue->m_failed.store(true, std::memory_order_release); + Atomic::Store(m_queue->m_failed, true, std::memory_order_release); if (m_completionCallback) m_completionCallback(this, m_completionResult, @@ -354,8 +355,8 @@ void CD3D12CommandSlot::OnCompletion(bool timeout) m_completionCallback = nullptr; m_completionParams[0] = nullptr; m_completionParams[1] = nullptr; - m_submitted.store(false, std::memory_order_release); - m_state.store(STATE_FREE, std::memory_order_release); + Atomic::Store(m_submitted, false, std::memory_order_release); + Atomic::Store(m_state, STATE_FREE, std::memory_order_release); SetEvent(m_availableEvent.Get()); } @@ -518,7 +519,7 @@ CD3D12CommandSlot * CD3D12CommandQueue::Acquire(UINT slotIndex) m_slots[slotIndex].OnCompletion(false); if (m_slots[slotIndex].Acquire()) return &m_slots[slotIndex]; - if (m_failed.load(std::memory_order_acquire)) + if (Atomic::Load(m_failed, std::memory_order_acquire)) break; const ULONGLONG now = GetTickCount64(); @@ -579,7 +580,7 @@ bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot) do { - if (m_failed.load(std::memory_order_relaxed)) + if (Atomic::Load(m_failed, std::memory_order_relaxed)) break; for (UINT i = 0; i < slot.m_fenceWaitCount; ++i) @@ -589,16 +590,16 @@ bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot) if (FAILED(hr)) { DEBUG_ERROR_HR(hr, "Failed to queue a fence wait (%ls)", m_name); - m_failed.store(true, std::memory_order_release); + Atomic::Store(m_failed, true, std::memory_order_release); break; } } - if (m_failed.load(std::memory_order_relaxed)) + if (Atomic::Load(m_failed, std::memory_order_relaxed)) break; const UINT64 fenceTarget = ++m_fenceValue; slot.m_fenceTarget = fenceTarget; - slot.m_submitted.store(true, std::memory_order_release); + Atomic::Store(slot.m_submitted, true, std::memory_order_release); ID3D12CommandList * lists[] = { slot.m_cmdList.Get() }; m_queue->ExecuteCommandLists(1, lists); @@ -607,7 +608,7 @@ bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot) if (FAILED(hr)) { DEBUG_ERROR_HR(hr, "Failed to signal the CommandQueue (%ls)", m_name); - m_failed.store(true, std::memory_order_release); + Atomic::Store(m_failed, true, std::memory_order_release); slot.OnCompletion(false); break; } @@ -621,10 +622,10 @@ bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot) // The work is already submitted and fenced. Poll only on this rare // error path so allocator, callback, and framebuffer ownership remain // valid until completion or confirmed device removal. - while (slot.m_submitted.load(std::memory_order_acquire)) + while (Atomic::Load(slot.m_submitted, std::memory_order_acquire)) { slot.OnCompletion(false); - if (slot.m_submitted.load(std::memory_order_acquire)) + if (Atomic::Load(slot.m_submitted, std::memory_order_acquire)) Sleep(1); } diff --git a/idd/LGIdd/d3d/CD3D12CommandQueue.h b/idd/LGIdd/d3d/CD3D12CommandQueue.h index 4a0a98ab..24f9bb0d 100644 --- a/idd/LGIdd/d3d/CD3D12CommandQueue.h +++ b/idd/LGIdd/d3d/CD3D12CommandQueue.h @@ -20,13 +20,13 @@ #pragma once +#include "Atomic.h" #include "CSRWLock.h" #include #include #include #include -#include #include using namespace Microsoft::WRL; @@ -123,15 +123,15 @@ class CD3D12CommandSlot bool IsIdle() const { - const State state = m_state.load(std::memory_order_acquire); + const State state = Atomic::Load(m_state, std::memory_order_acquire); return state == STATE_FREE || (state == STATE_FAILED && - !m_submitted.load(std::memory_order_acquire)); + !Atomic::Load(m_submitted, std::memory_order_acquire)); } bool HasSubmittedWork() const { - return m_submitted.load(std::memory_order_acquire); + return Atomic::Load(m_submitted, std::memory_order_acquire); } ComPtr GetGfxList() { return m_gfxList; } diff --git a/idd/LGIdd/display/CDeviceContext.cpp b/idd/LGIdd/display/CDeviceContext.cpp index 25fc5f85..aaf5c500 100644 --- a/idd/LGIdd/display/CDeviceContext.cpp +++ b/idd/LGIdd/display/CDeviceContext.cpp @@ -26,6 +26,7 @@ #include "transport/IFrameTransport.h" #include "transport/IInputTransport.h" #include "transport/TransportFactory.h" +#include "Atomic.h" #include "CDebug.h" #include @@ -168,7 +169,7 @@ void CDeviceContext::InitAdapter() } LONG initExpected = 0; - if (!m_initInProgress.compare_exchange_strong(initExpected, 1)) + if (!Atomic::CAS(m_initInProgress, initExpected, 1)) { DEBUG_TRACE("Adapter initialization skipped: initialization already in progress"); return; @@ -182,7 +183,7 @@ void CDeviceContext::InitAdapter() if (!m_transport) { DEBUG_ERROR("Failed to create the frame transport"); - m_initInProgress.store(0); + Atomic::Store(m_initInProgress, 0); return; } @@ -196,7 +197,7 @@ void CDeviceContext::InitAdapter() } else DEBUG_ERROR("Failed to open the frame transport"); - m_initInProgress.store(0); + Atomic::Store(m_initInProgress, 0); return; } m_transportOpened = true; @@ -268,13 +269,13 @@ void CDeviceContext::InitAdapter() DEBUG_TRACE("Initializing frame transport metadata"); if (!InitializeTransport()) { - m_initInProgress.store(0); + Atomic::Store(m_initInProgress, 0); return; } DEBUG_TRACE("Loading configured display modes"); if (!m_displayConfiguration.Load(*m_transport)) { - m_initInProgress.store(0); + Atomic::Store(m_initInProgress, 0); return; } DEBUG_TRACE("Initializing monitor EDID"); @@ -347,7 +348,7 @@ void CDeviceContext::InitAdapter() if (!NT_SUCCESS(status)) { DEBUG_ERROR_HR(status, "IddCxAdapterInitAsync Failed"); - m_initInProgress.store(0); + Atomic::Store(m_initInProgress, 0); return; } @@ -355,7 +356,7 @@ void CDeviceContext::InitAdapter() if (!m_adapter) { DEBUG_ERROR("IddCxAdapterInitAsync succeeded without returning an adapter object"); - m_initInProgress.store(0); + Atomic::Store(m_initInProgress, 0); return; } @@ -367,7 +368,7 @@ void CDeviceContext::InitAdapter() // Adapter is up; no need to keep retrying. StopInitRetry(); - m_initInProgress.store(0); + Atomic::Store(m_initInProgress, 0); DEBUG_INFO("Adapter initialization request complete; returning to IddCx"); } diff --git a/idd/LGIdd/display/CDeviceContext.h b/idd/LGIdd/display/CDeviceContext.h index 2ce8acac..6302517f 100644 --- a/idd/LGIdd/display/CDeviceContext.h +++ b/idd/LGIdd/display/CDeviceContext.h @@ -20,11 +20,12 @@ #pragma once +#include "Atomic.h" + #include #include #include -#include #include #include #include diff --git a/idd/LGIdd/display/CMonitorContext.cpp b/idd/LGIdd/display/CMonitorContext.cpp index 93d838fc..839325aa 100644 --- a/idd/LGIdd/display/CMonitorContext.cpp +++ b/idd/LGIdd/display/CMonitorContext.cpp @@ -48,8 +48,8 @@ NTSTATUS CMonitorContext::AssignSwapChain( // new generation is established. DetachSwapChain(); - const UINT64 assignmentGeneration = - m_assignmentGeneration.fetch_add(1, std::memory_order_acq_rel) + 1; + const UINT64 assignmentGeneration = Atomic::FetchAdd( + m_assignmentGeneration, 1, std::memory_order_acq_rel) + 1; // Build the D3D11 device into a local so the member is never observed // half-constructed. The worker binds it before performing the expensive @@ -96,7 +96,8 @@ void CMonitorContext::DetachSwapChain() // Invalidate setup in progress before waiting for m_lock. This also lets a // worker about to call SetDevice observe an unassign whose callback is // blocked waiting for the processor to be published. - m_assignmentGeneration.fetch_add(1, std::memory_order_acq_rel); + Atomic::FetchAdd( + m_assignmentGeneration, 1, std::memory_order_acq_rel); // Detach under the lock, then destroy outside it. Destroying the processor // joins its worker thread, whose teardown (WdfObjectDelete) re-enters this diff --git a/idd/LGIdd/display/CMonitorContext.h b/idd/LGIdd/display/CMonitorContext.h index 460429a0..d62a78b7 100644 --- a/idd/LGIdd/display/CMonitorContext.h +++ b/idd/LGIdd/display/CMonitorContext.h @@ -20,13 +20,13 @@ #pragma once +#include "Atomic.h" #include "CSRWLock.h" #include #include #include -#include #include #include @@ -71,7 +71,8 @@ public: void UnassignSwapChain(); bool IsAssignmentCurrent(UINT64 generation) const { - return m_assignmentGeneration.load(std::memory_order_acquire) == generation; + return Atomic::Load( + m_assignmentGeneration, std::memory_order_acquire) == generation; } CDeviceContext * GetDeviceContext() { return m_devContext; } diff --git a/idd/LGIdd/display/CMonitorManager.cpp b/idd/LGIdd/display/CMonitorManager.cpp index 3229b337..a47d2d9a 100644 --- a/idd/LGIdd/display/CMonitorManager.cpp +++ b/idd/LGIdd/display/CMonitorManager.cpp @@ -21,6 +21,7 @@ #include "display/CMonitorManager.h" #include "display/CMonitorContext.h" +#include "Atomic.h" #include "CDebug.h" bool CMonitorManager::Create(UINT connectorIndex, IDDCX_ADAPTER adapter, @@ -133,7 +134,7 @@ CMonitorManager::ReplugAction CMonitorManager::Replug() { // Either no monitor yet, or one is already pending; build it now and // cancel any queued rebuild so we do not create two. - m_createQueued.store(0); + Atomic::Store(m_createQueued, 0); return ReplugAction::CREATE; } @@ -163,7 +164,7 @@ CMonitorManager::ReplugAction CMonitorManager::Replug() // If there was no swap chain there will be no unassign callback to queue // the rebuild. Otherwise OnSwapChainReleased does so after teardown drains. if (rebuild) - m_createQueued.store(1); + Atomic::Store(m_createQueued, 1); return ReplugAction::NONE; } @@ -205,7 +206,7 @@ void CMonitorManager::OnSwapChainReleased() } if (rebuild) - m_createQueued.store(1); + Atomic::Store(m_createQueued, 1); } CMonitorManager::ReadyAction CMonitorManager::OnSwapChainReady() @@ -249,15 +250,15 @@ CMonitorManager::ReadyAction CMonitorManager::OnSwapChainReady() void CMonitorManager::QueueReplug() { - m_replugQueued.store(1); + Atomic::Store(m_replugQueued, 1); } CMonitorManager::DeferredAction CMonitorManager::TakeDeferredAction() { - if (m_createQueued.exchange(0)) + if (Atomic::Swap(m_createQueued, 0)) return DeferredAction::CREATE; - if (m_replugQueued.exchange(0)) + if (Atomic::Swap(m_replugQueued, 0)) return DeferredAction::REPLUG; return DeferredAction::NONE; diff --git a/idd/LGIdd/display/CMonitorManager.h b/idd/LGIdd/display/CMonitorManager.h index 9e50bb5a..d4c6b4de 100644 --- a/idd/LGIdd/display/CMonitorManager.h +++ b/idd/LGIdd/display/CMonitorManager.h @@ -20,11 +20,11 @@ #pragma once +#include "Atomic.h" #include "CSRWLock.h" #include #include #include -#include #include #include "config/CSettings.h" diff --git a/idd/LGIdd/ipc/CInputPipeServer.cpp b/idd/LGIdd/ipc/CInputPipeServer.cpp index 3ca48a36..bb16f61b 100644 --- a/idd/LGIdd/ipc/CInputPipeServer.cpp +++ b/idd/LGIdd/ipc/CInputPipeServer.cpp @@ -34,7 +34,7 @@ bool CInputPipeServer::Init() { DeInit(); - m_state.store(0, std::memory_order_release); + Atomic::Store(m_state, 0, std::memory_order_release); m_performanceFrequency.QuadPart = 0; if (!QueryPerformanceFrequency(&m_performanceFrequency)) m_performanceFrequency.QuadPart = 0; @@ -183,7 +183,7 @@ bool CInputPipeServer::QueueRawLocked( (m_queueHead + m_queueCount) % QUEUE_LENGTH; m_queue[index].type = type; m_queue[index].state = - m_state.load(std::memory_order_relaxed); + Atomic::Load(m_state, std::memory_order_relaxed); m_queue[index].payload = payload; m_queue[index].pureMotion = pureMotion; ++m_queueCount; @@ -236,11 +236,11 @@ void CInputPipeServer::ResyncLocked() m_statResyncDiscarded += m_queueCount; m_queueHead = 0; m_queueCount = 0; - uint64_t state = m_state.load(std::memory_order_relaxed); + uint64_t state = Atomic::Load(m_state, std::memory_order_relaxed); while (state & 1) { - if (m_state.compare_exchange_weak( - state, state + 2, std::memory_order_acq_rel)) + if (Atomic::CASWeak( + m_state, state, state + 2, std::memory_order_acq_rel)) { QueueResetLocked(); return; @@ -260,7 +260,7 @@ bool CInputPipeServer::SendMouseRelative( deltaY > LG_INPUT_MOUSE_DELTA_MAX || wheel < LG_INPUT_MOUSE_WHEEL_MIN_TOTAL || wheel > LG_INPUT_MOUSE_WHEEL_MAX || - !(m_state.load(std::memory_order_acquire) & 1)) + !(Atomic::Load(m_state, std::memory_order_acquire) & 1)) return false; KVMFRInputPayload payload = {}; @@ -272,7 +272,8 @@ bool CInputPipeServer::SendMouseRelative( CSRWExclusiveLock lock(m_queueLock); const bool pureMotion = wheel == 0 && buttons == m_relativeButtons; const bool switching = m_mouseMode == MouseMode::ABSOLUTE_INPUT; - bool queued = (m_state.load(std::memory_order_relaxed) & 1) != 0; + bool queued = + (Atomic::Load(m_state, std::memory_order_relaxed) & 1) != 0; if (queued && switching && m_absoluteButtons) { KVMFRInputPayload neutral = {}; @@ -307,7 +308,7 @@ bool CInputPipeServer::SendMouseAbsolute( y > LG_INPUT_MOUSE_ABSOLUTE_MAX || wheel < LG_INPUT_MOUSE_WHEEL_MIN_TOTAL || wheel > LG_INPUT_MOUSE_WHEEL_MAX || - !(m_state.load(std::memory_order_acquire) & 1)) + !(Atomic::Load(m_state, std::memory_order_acquire) & 1)) return false; KVMFRInputPayload payload = {}; @@ -319,7 +320,8 @@ bool CInputPipeServer::SendMouseAbsolute( CSRWExclusiveLock lock(m_queueLock); const bool pureMotion = wheel == 0 && buttons == m_absoluteButtons; const bool switching = m_mouseMode == MouseMode::RELATIVE_INPUT; - bool queued = (m_state.load(std::memory_order_relaxed) & 1) != 0; + bool queued = + (Atomic::Load(m_state, std::memory_order_relaxed) & 1) != 0; if (queued && switching && m_relativeButtons) { const KVMFRInputPayload neutral = {}; @@ -348,7 +350,7 @@ bool CInputPipeServer::SendKeyboard( uint8_t modifiers, const uint8_t * keys) { - if (!keys || !(m_state.load(std::memory_order_acquire) & 1)) + if (!keys || !(Atomic::Load(m_state, std::memory_order_acquire) & 1)) return false; KVMFRInputPayload payload = {}; @@ -361,7 +363,8 @@ bool CInputPipeServer::SendKeyboard( } CSRWExclusiveLock lock(m_queueLock); - const bool queued = (m_state.load(std::memory_order_relaxed) & 1) && + const bool queued = + (Atomic::Load(m_state, std::memory_order_relaxed) & 1) && QueueLocked(LG_INPUT_PIPE_MESSAGE_KEYBOARD, payload, false); if (!queued) ResyncLocked(); @@ -370,11 +373,12 @@ bool CInputPipeServer::SendKeyboard( bool CInputPipeServer::Reset() { - if (!(m_state.load(std::memory_order_acquire) & 1)) + if (!(Atomic::Load(m_state, std::memory_order_acquire) & 1)) return false; CSRWExclusiveLock lock(m_queueLock); - bool queued = (m_state.load(std::memory_order_relaxed) & 1) != 0; + bool queued = + (Atomic::Load(m_state, std::memory_order_relaxed) & 1) != 0; if (queued) queued = QueueResetLocked(); if (!queued) @@ -412,7 +416,8 @@ bool CInputPipeServer::Send(const QueueItem& item) LARGE_INTEGER end = {}; { CSRWSharedLock lock(m_connectionLock); - const uint64_t state = m_state.load(std::memory_order_acquire); + const uint64_t state = + Atomic::Load(m_state, std::memory_order_acquire); current = (state & 1) && item.state == state; if (current) { @@ -527,13 +532,13 @@ void CInputPipeServer::LogStatistics() void CInputPipeServer::Invalidate(uint64_t state, bool requireMatch) { CSRWExclusiveLock connectionLock(m_connectionLock); - uint64_t current = m_state.load(std::memory_order_relaxed); + uint64_t current = Atomic::Load(m_state, std::memory_order_relaxed); for (;;) { if (!(current & 1) || (requireMatch && state != current)) return; - if (m_state.compare_exchange_weak( - current, current + 1, std::memory_order_acq_rel)) + if (Atomic::CASWeak( + m_state, current, current + 1, std::memory_order_acq_rel)) break; } @@ -581,7 +586,7 @@ void CInputPipeServer::OnPipeConnected() CSRWExclusiveLock connectionLock(m_connectionLock); CSRWExclusiveLock queueLock(m_queueLock); - uint64_t state = m_state.load(std::memory_order_relaxed); + uint64_t state = Atomic::Load(m_state, std::memory_order_relaxed); if (state & 1) ++state; ++state; @@ -597,7 +602,7 @@ void CInputPipeServer::OnPipeConnected() m_queue[index].state = state; } if (reset) - m_state.store(state, std::memory_order_release); + Atomic::Store(m_state, state, std::memory_order_release); if (!reset) DEBUG_WARN("Failed to queue LGInput endpoint neutralization"); diff --git a/idd/LGIdd/ipc/CInputPipeServer.h b/idd/LGIdd/ipc/CInputPipeServer.h index 906a2d02..c949f764 100644 --- a/idd/LGIdd/ipc/CInputPipeServer.h +++ b/idd/LGIdd/ipc/CInputPipeServer.h @@ -20,12 +20,12 @@ #pragma once +#include "Atomic.h" #include "CPipeEndpoint.h" #include "CSRWLock.h" #include "InputPipeProtocol.h" #include "input/IInputSink.h" -#include #include #include @@ -115,7 +115,7 @@ public: uint64_t GetState() const override { - return m_state.load(std::memory_order_acquire); + return Atomic::Load(m_state, std::memory_order_acquire); } bool SendMouseRelative( diff --git a/idd/LGIdd/transport/CFrameHub.cpp b/idd/LGIdd/transport/CFrameHub.cpp index 34231bda..e0b881d2 100644 --- a/idd/LGIdd/transport/CFrameHub.cpp +++ b/idd/LGIdd/transport/CFrameHub.cpp @@ -11,6 +11,7 @@ #include "transport/CFrameHub.h" +#include "Atomic.h" #include "CDebug.h" static const uint64_t RETRY_NS = 1000000ULL; @@ -46,8 +47,9 @@ CFrameHub::~CFrameHub() for (Sink& sink : m_sinks) { const BackendId backend = - sink.backend.load(std::memory_order_acquire); - const uint32_t epoch = sink.epoch.load(std::memory_order_acquire); + Atomic::Load(sink.backend, std::memory_order_acquire); + const uint32_t epoch = + Atomic::Load(sink.epoch, std::memory_order_acquire); if (backend && epoch) Unbind(backend, epoch); } @@ -72,13 +74,15 @@ bool CFrameHub::Bind(BackendId backend, uint32_t epoch, bool primary, CSRWExclusiveLock lock(m_listLock); if (primary) { - if (!m_sinks[0].active.load(std::memory_order_acquire) && + if (!Atomic::Load( + m_sinks[0].active, std::memory_order_acquire) && !m_sinks[0].reserved) selected = &m_sinks[0]; } else for (unsigned i = 1; i < FRAME_MAX_SINKS; ++i) - if (!m_sinks[i].active.load(std::memory_order_acquire) && + if (!Atomic::Load( + m_sinks[i].active, std::memory_order_acquire) && !m_sinks[i].reserved) { selected = &m_sinks[i]; @@ -93,10 +97,10 @@ bool CFrameHub::Bind(BackendId backend, uint32_t epoch, bool primary, { CSRWExclusiveLock lock(selected->callLock); selected->target = ⌖ - selected->backend.store(backend, std::memory_order_release); - selected->epoch.store(epoch, std::memory_order_release); + Atomic::Store(selected->backend, backend, std::memory_order_release); + Atomic::Store(selected->epoch, epoch, std::memory_order_release); selected->primary = primary; - selected->outstanding.store(0, std::memory_order_release); + Atomic::Store(selected->outstanding, 0, std::memory_order_release); SetEvent(selected->drained); } { @@ -120,7 +124,7 @@ bool CFrameHub::Bind(BackendId backend, uint32_t epoch, bool primary, target.SetFrameScheduleEvent(m_wakeEvent); { CSRWExclusiveLock lock(m_listLock); - selected->active.store(true, std::memory_order_release); + Atomic::Store(selected->active, true, std::memory_order_release); selected->reserved = false; } target.ForceFrame(); @@ -134,11 +138,11 @@ void CFrameHub::Unbind(BackendId backend, uint32_t epoch) { CSRWExclusiveLock lock(m_listLock); for (Sink& sink : m_sinks) - if (sink.active.load(std::memory_order_acquire) && - sink.backend.load(std::memory_order_acquire) == backend && - sink.epoch.load(std::memory_order_acquire) == epoch) + if (Atomic::Load(sink.active, std::memory_order_acquire) && + Atomic::Load(sink.backend, std::memory_order_acquire) == backend && + Atomic::Load(sink.epoch, std::memory_order_acquire) == epoch) { - sink.active.store(false, std::memory_order_release); + Atomic::Store(sink.active, false, std::memory_order_release); sink.reserved = true; selected = &sink; break; @@ -184,8 +188,8 @@ void CFrameHub::Unbind(BackendId backend, uint32_t epoch) { CSRWExclusiveLock lock(selected->callLock); selected->target = nullptr; - selected->backend.store(0, std::memory_order_release); - selected->epoch.store(0, std::memory_order_release); + Atomic::Store(selected->backend, 0, std::memory_order_release); + Atomic::Store(selected->epoch, 0, std::memory_order_release); selected->primary = false; } { @@ -211,7 +215,7 @@ unsigned CFrameHub::Snapshot(SinkRef refs[FRAME_MAX_SINKS]) const unsigned count = 0; CSRWSharedLock lock(m_listLock); for (unsigned i = 0; i < FRAME_MAX_SINKS; ++i) - if (m_sinks[i].active.load(std::memory_order_acquire)) + if (Atomic::Load(m_sinks[i].active, std::memory_order_acquire)) refs[count++] = { const_cast(&m_sinks[i]), i, m_sinks[i].primary }; return count; @@ -233,7 +237,7 @@ void CFrameHub::ReleaseTarget(Batch& batch, BatchTarget& target) CSRWExclusiveLock lock(target.sink->laneLock); target.sink->lanes[target.resourceLane].busy = false; } - if (target.sink->outstanding.fetch_sub( + if (Atomic::FetchSub(target.sink->outstanding, 1, std::memory_order_acq_rel) == 1) SetEvent(target.sink->drained); @@ -341,7 +345,7 @@ void CFrameHub::FillLane(Sink& sink, unsigned laneIndex, { lane.phase = Sink::ResourceLane::PENDING; cancel = lane.cancelRequested || - !sink.active.load(std::memory_order_acquire); + !Atomic::Load(sink.active, std::memory_order_acquire); } } @@ -468,7 +472,8 @@ void CFrameHub::CompleteLane(Sink& sink, unsigned laneIndex, lane.busy = false; } } - if (sink.outstanding.fetch_sub(1, std::memory_order_acq_rel) == 1) + if (Atomic::FetchSub(sink.outstanding, + 1, std::memory_order_acq_rel) == 1) SetEvent(sink.drained); SetEvent(m_wakeEvent); } @@ -481,8 +486,9 @@ void CFrameHub::OnFrameDone(const FrameToken& token, FrameDone result, for (Sink& sink : m_sinks) { - if (sink.backend.load(std::memory_order_acquire) != token.backend || - sink.epoch.load(std::memory_order_acquire) != token.epoch) + if (Atomic::Load(sink.backend, std::memory_order_acquire) != + token.backend || + Atomic::Load(sink.epoch, std::memory_order_acquire) != token.epoch) continue; unsigned laneIndex = FRAME_SINK_BUFFERS; @@ -525,7 +531,8 @@ size_t CFrameHub::GetMaxFrameSize() const if (refs[i].primary) { CSRWSharedLock call(refs[i].sink->callLock); - if (refs[i].sink->active.load(std::memory_order_acquire) && + if (Atomic::Load( + refs[i].sink->active, std::memory_order_acquire) && refs[i].sink->target) return refs[i].sink->target->GetMaxFrameSize(); } @@ -534,21 +541,16 @@ size_t CFrameHub::GetMaxFrameSize() const uint64_t CFrameHub::NextContentSerial() { - uint64_t serial = m_nextContent.fetch_add( - 1, std::memory_order_acq_rel) + 1; - if (!serial) - serial = m_nextContent.fetch_add( - 1, std::memory_order_acq_rel) + 1; - return serial; + return Atomic::Next(m_nextContent, std::memory_order_acq_rel); } void CFrameHub::FrameProductReady(uint64_t contentSerial) { - uint64_t newest = m_newestContent.load(std::memory_order_acquire); + uint64_t newest = + Atomic::Load(m_newestContent, std::memory_order_acquire); while (ContentAfter(contentSerial, newest) && - !m_newestContent.compare_exchange_weak(newest, - contentSerial, std::memory_order_acq_rel, - std::memory_order_acquire)) + !Atomic::CASWeak(m_newestContent, newest, contentSerial, + std::memory_order_acq_rel, std::memory_order_acquire)) { } SetEvent(m_wakeEvent); @@ -557,13 +559,14 @@ void CFrameHub::FrameProductReady(uint64_t contentSerial) bool CFrameHub::NeedsFrame() const { const uint64_t newest = - m_newestContent.load(std::memory_order_acquire); + Atomic::Load(m_newestContent, std::memory_order_acquire); SinkRef refs[FRAME_MAX_SINKS]; const unsigned count = Snapshot(refs); for (unsigned i = 0; i < count; ++i) { CSRWSharedLock call(refs[i].sink->callLock); - if (!refs[i].sink->active.load(std::memory_order_acquire) || + if (!Atomic::Load( + refs[i].sink->active, std::memory_order_acquire) || !refs[i].sink->target) continue; const size_t maxFrameSize = refs[i].sink->target->GetMaxFrameSize(); @@ -590,7 +593,8 @@ bool CFrameHub::GetFramePlan( { Sink& sink = *refs[i].sink; CSRWSharedLock call(sink.callLock); - if (!sink.active.load(std::memory_order_acquire) || !sink.target) + if (!Atomic::Load(sink.active, std::memory_order_acquire) || + !sink.target) continue; sink.target->ProcessDeliveries(); @@ -641,8 +645,10 @@ bool CFrameHub::GetFramePlan( continue; FramePlanTarget& request = plan.targets[plan.count++]; request.sink = refs[i].index; - request.backend = sink.backend.load(std::memory_order_acquire); - request.epoch = sink.epoch.load(std::memory_order_acquire); + request.backend = Atomic::Load( + sink.backend, std::memory_order_acquire); + request.epoch = Atomic::Load( + sink.epoch, std::memory_order_acquire); request.schedule = schedule; request.commitSchedule = schedule; request.periodic = periodic; @@ -660,7 +666,8 @@ bool CFrameHub::GetImmediateFramePlan(uint64_t now, FramePlan& plan) { Sink& sink = *refs[i].sink; CSRWSharedLock call(sink.callLock); - if (!sink.active.load(std::memory_order_acquire) || !sink.target) + if (!Atomic::Load(sink.active, std::memory_order_acquire) || + !sink.target) continue; sink.target->ProcessDeliveries(); @@ -684,8 +691,10 @@ bool CFrameHub::GetImmediateFramePlan(uint64_t now, FramePlan& plan) continue; FramePlanTarget& request = plan.targets[plan.count++]; request.sink = refs[i].index; - request.backend = sink.backend.load(std::memory_order_acquire); - request.epoch = sink.epoch.load(std::memory_order_acquire); + request.backend = Atomic::Load( + sink.backend, std::memory_order_acquire); + request.epoch = Atomic::Load( + sink.epoch, std::memory_order_acquire); request.schedule = schedule; request.schedule.deliveryDeadlineSerial = 0; request.schedule.phaseEligible = false; @@ -708,9 +717,12 @@ void CFrameHub::MissFramePlan(const FramePlan& plan, uint64_t now) continue; Sink& sink = m_sinks[request.sink]; CSRWSharedLock call(sink.callLock); - if (sink.active.load(std::memory_order_acquire) && sink.target && - sink.backend.load(std::memory_order_acquire) == request.backend && - sink.epoch.load(std::memory_order_acquire) == request.epoch) + if (Atomic::Load(sink.active, std::memory_order_acquire) && + sink.target && + Atomic::Load(sink.backend, std::memory_order_acquire) == + request.backend && + Atomic::Load(sink.epoch, std::memory_order_acquire) == + request.epoch) sink.target->FrameMissed( request.commitSchedule, now, request.periodic); } @@ -736,11 +748,8 @@ bool CFrameHub::PrepareFrameBatch(const FramePlan& plan, continue; candidate.active = true; candidate.count = 0; - candidate.serial = m_nextSerial.fetch_add( - 1, std::memory_order_acq_rel) + 1; - if (!candidate.serial) - candidate.serial = m_nextSerial.fetch_add( - 1, std::memory_order_acq_rel) + 1; + candidate.serial = Atomic::Next( + m_nextSerial, std::memory_order_acq_rel); for (BatchTarget& target : candidate.targets) target = {}; prepared.token = { @@ -761,9 +770,12 @@ bool CFrameHub::PrepareFrameBatch(const FramePlan& plan, continue; Sink& sink = m_sinks[request.sink]; CSRWExclusiveLock call(sink.callLock); - if (!sink.active.load(std::memory_order_acquire) || !sink.target || - sink.backend.load(std::memory_order_acquire) != request.backend || - sink.epoch.load(std::memory_order_acquire) != request.epoch || + if (!Atomic::Load(sink.active, std::memory_order_acquire) || + !sink.target || + Atomic::Load(sink.backend, std::memory_order_acquire) != + request.backend || + Atomic::Load(sink.epoch, std::memory_order_acquire) != + request.epoch || !sink.target->FrameBufferAvailable( request.schedule, allowReadyReplacement)) continue; @@ -882,7 +894,7 @@ bool CFrameHub::PrepareFrameBatch(const FramePlan& plan, target.frameType = dstFormat.format; target.periodic = request.periodic; target.active = true; - if (sink.outstanding.fetch_add( + if (Atomic::FetchAdd(sink.outstanding, 1, std::memory_order_acq_rel) == 0) ResetEvent(sink.drained); @@ -928,9 +940,10 @@ uint32_t CFrameHub::PublishFrameBatch(const FrameBatchToken& token) CSRWExclusiveLock call(target.sink->callLock); bool delivered = false; const bool valid = target.sink->target && - target.sink->backend.load(std::memory_order_acquire) == + Atomic::Load(target.sink->backend, std::memory_order_acquire) == target.backend && - target.sink->epoch.load(std::memory_order_acquire) == target.epoch; + Atomic::Load(target.sink->epoch, std::memory_order_acquire) == + target.epoch; if (!valid || !target.sink->target->PublishFrameBuffer( target.localSlot, target.deliverySchedule, delivered)) { @@ -980,10 +993,10 @@ void CFrameHub::CommitFrameBatch(const FrameBatchToken& token) { CSRWExclusiveLock call(target.sink->callLock); if (target.sink->target && - target.sink->backend.load(std::memory_order_acquire) == - target.backend && - target.sink->epoch.load(std::memory_order_acquire) == - target.epoch) + Atomic::Load(target.sink->backend, + std::memory_order_acquire) == target.backend && + Atomic::Load(target.sink->epoch, + std::memory_order_acquire) == target.epoch) target.sink->target->CommitFrameBuffer(target.localSlot, target.schedule, target.periodic, target.delivered); } @@ -1022,9 +1035,10 @@ void CFrameHub::AbortFrameBatch(const FrameBatchToken& token) continue; CSRWExclusiveLock call(target.sink->callLock); if (target.sink->target && - target.sink->backend.load(std::memory_order_acquire) == - target.backend && - target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + Atomic::Load(target.sink->backend, + std::memory_order_acquire) == target.backend && + Atomic::Load(target.sink->epoch, + std::memory_order_acquire) == target.epoch) target.sink->target->AbortFrameBuffer(target.localSlot); { CSRWExclusiveLock state(target.sink->laneLock); @@ -1049,9 +1063,10 @@ void CFrameHub::FailFrameBatch(const FrameBatchToken& token) continue; CSRWExclusiveLock call(target.sink->callLock); if (target.sink->target && - target.sink->backend.load(std::memory_order_acquire) == - target.backend && - target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + Atomic::Load(target.sink->backend, + std::memory_order_acquire) == target.backend && + Atomic::Load(target.sink->epoch, + std::memory_order_acquire) == target.epoch) { if (target.published) target.sink->target->FailFrameBuffer(target.localSlot); @@ -1080,9 +1095,10 @@ void CFrameHub::WriteFrameTarget(const FrameBatchToken& token, BatchTarget& target = batch.targets[index]; CSRWSharedLock call(target.sink->callLock); if (target.sink->target && - target.sink->backend.load(std::memory_order_acquire) == + Atomic::Load(target.sink->backend, std::memory_order_acquire) == target.backend && - target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + Atomic::Load(target.sink->epoch, std::memory_order_acquire) == + target.epoch) target.sink->target->WriteFrameBuffer( target.localSlot, src, offset, len, setWritePos); } @@ -1101,9 +1117,10 @@ void CFrameHub::WriteFrameTargetRows(const FrameBatchToken& token, BatchTarget& target = batch.targets[index]; CSRWSharedLock call(target.sink->callLock); if (target.sink->target && - target.sink->backend.load(std::memory_order_acquire) == + Atomic::Load(target.sink->backend, std::memory_order_acquire) == target.backend && - target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + Atomic::Load(target.sink->epoch, std::memory_order_acquire) == + target.epoch) target.sink->target->WriteFrameBufferRows(target.localSlot, src, offset, rowBytes, pitch, rows); } @@ -1121,9 +1138,10 @@ void CFrameHub::FinalizeFrameTarget( BatchTarget& target = batch.targets[index]; CSRWSharedLock call(target.sink->callLock); if (target.sink->target && - target.sink->backend.load(std::memory_order_acquire) == + Atomic::Load(target.sink->backend, std::memory_order_acquire) == target.backend && - target.sink->epoch.load(std::memory_order_acquire) == target.epoch) + Atomic::Load(target.sink->epoch, std::memory_order_acquire) == + target.epoch) target.sink->target->FinalizeFrameBuffer(target.localSlot); } @@ -1201,7 +1219,8 @@ void CFrameHub::ObserveFrame(uint64_t now) for (unsigned i = 0; i < count; ++i) { CSRWSharedLock call(refs[i].sink->callLock); - if (refs[i].sink->active.load(std::memory_order_acquire) && + if (Atomic::Load( + refs[i].sink->active, std::memory_order_acquire) && refs[i].sink->target) refs[i].sink->target->ObserveFrame(now); } @@ -1214,7 +1233,8 @@ void CFrameHub::ForceFrame() for (unsigned i = 0; i < count; ++i) { CSRWSharedLock call(refs[i].sink->callLock); - if (refs[i].sink->active.load(std::memory_order_acquire) && + if (Atomic::Load( + refs[i].sink->active, std::memory_order_acquire) && refs[i].sink->target) refs[i].sink->target->ForceFrame(); } @@ -1227,7 +1247,8 @@ void CFrameHub::FrameSuperseded() for (unsigned i = 0; i < count; ++i) { CSRWSharedLock call(refs[i].sink->callLock); - if (refs[i].sink->active.load(std::memory_order_acquire) && + if (Atomic::Load( + refs[i].sink->active, std::memory_order_acquire) && refs[i].sink->target) refs[i].sink->target->FrameSuperseded(); } diff --git a/idd/LGIdd/transport/CFrameHub.h b/idd/LGIdd/transport/CFrameHub.h index 33209dbe..1c0d25ff 100644 --- a/idd/LGIdd/transport/CFrameHub.h +++ b/idd/LGIdd/transport/CFrameHub.h @@ -20,13 +20,12 @@ #pragma once +#include "Atomic.h" #include "CSRWLock.h" #include "transport/IFrameSink.h" #include "transport/IFrameTransport.h" #include "transport/ITransport.h" -#include - class CFrameHub final : public IFrameTransport, public IFrameEvents { private: diff --git a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp index 786e3b4a..9f8a0d06 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp +++ b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp @@ -23,6 +23,7 @@ #include "transport/lgmp/CIVSHMEM.h" #include "transport/lgmp/CLGMPFrameCaps.h" #include "transport/lgmp/CLGMPHost.h" +#include "Atomic.h" #include "CDebug.h" #include @@ -196,13 +197,13 @@ bool CLGMPFrameTransport::Setup(size_t alignSize) m_frame[i]->offset = (uint32_t)alignOffset; m_frameBuffer[i] = reinterpret_cast( reinterpret_cast(m_frame[i]) + alignOffset); - m_frameInFlight[i].store(false, std::memory_order_release); + Atomic::Store(m_frameInFlight[i], false, std::memory_order_release); m_frameCompleted[i] = false; } m_maxFrameSize = maxFrameSize; - m_submittedFrameIndex.store(-1, std::memory_order_release); - m_readyFrameIndex.store(-1, std::memory_order_release); + Atomic::Store(m_submittedFrameIndex, -1, std::memory_order_release); + Atomic::Store(m_readyFrameIndex, -1, std::memory_order_release); m_deferredOwnerFrameIndex = -1; m_framePublishSequence = 0; m_frameReadySequence = 0; @@ -222,8 +223,8 @@ void CLGMPFrameTransport::DeInit() { CSRWExclusiveLock lock(m_framePublishLock); - m_submittedFrameIndex.store(-1, std::memory_order_release); - m_readyFrameIndex.store(-1, std::memory_order_release); + Atomic::Store(m_submittedFrameIndex, -1, std::memory_order_release); + Atomic::Store(m_readyFrameIndex, -1, std::memory_order_release); m_deferredOwnerFrameIndex = -1; m_framePublishSequence = 0; m_frameReadySequence = 0; @@ -238,7 +239,7 @@ void CLGMPFrameTransport::DeInit() for (int i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) { - m_frameInFlight[i].store(false, std::memory_order_release); + Atomic::Store(m_frameInFlight[i], false, std::memory_order_release); lgmpHostMemFree(&m_frameMemory[i]); m_frame[i] = nullptr; m_frameBuffer[i] = nullptr; @@ -571,14 +572,15 @@ int CLGMPFrameTransport::FindAvailableFrameBuffer( bool allowReady) const { const LONG readyFrameIndex = - m_readyFrameIndex.load(std::memory_order_acquire); + Atomic::Load(m_readyFrameIndex, std::memory_order_acquire); int available = -1; uint64_t newestPublish = 0; for (unsigned frameIndex = 0; frameIndex < LGMP_Q_FRAME_BUFFER_LEN; ++frameIndex) { if (static_cast(frameIndex) == readyFrameIndex || - m_frameInFlight[frameIndex].load(std::memory_order_acquire) || + Atomic::Load( + m_frameInFlight[frameIndex], std::memory_order_acquire) || FrameBufferReferenced(frameIndex)) continue; @@ -591,7 +593,8 @@ int CLGMPFrameTransport::FindAvailableFrameBuffer( } if (available >= 0 || !allowReady || readyFrameIndex < 0 || - m_frameInFlight[readyFrameIndex].load(std::memory_order_acquire) || + Atomic::Load( + m_frameInFlight[readyFrameIndex], std::memory_order_acquire) || FrameBufferReferenced(static_cast(readyFrameIndex))) return available; @@ -610,7 +613,8 @@ int CLGMPFrameTransport::FindNewestCompletedFrame( frameIndex < LGMP_Q_FRAME_BUFFER_LEN; ++frameIndex) { if (frameIndex == excludeFrameIndex || !m_frameCompleted[frameIndex] || - m_frameInFlight[frameIndex].load(std::memory_order_acquire)) + Atomic::Load( + m_frameInFlight[frameIndex], std::memory_order_acquire)) continue; if (newestFrame < 0 || @@ -679,9 +683,10 @@ bool CLGMPFrameTransport::GetPendingDeliveryTarget(uint64_t now, CSRWSharedLock lock(m_framePublishLock); const LONG frameIndex = - m_readyFrameIndex.load(std::memory_order_acquire); + Atomic::Load(m_readyFrameIndex, std::memory_order_acquire); if (frameIndex < 0 || - m_frameInFlight[frameIndex].load(std::memory_order_acquire) || + Atomic::Load( + m_frameInFlight[frameIndex], std::memory_order_acquire) || lgmpHostQueuePending(m_frameQueue) != 0) return false; @@ -705,9 +710,10 @@ bool CLGMPFrameTransport::RetryPendingDelivery(uint64_t now, bool& retry) CSRWExclusiveLock lock(m_framePublishLock); const LONG frameIndex = - m_readyFrameIndex.load(std::memory_order_acquire); + Atomic::Load(m_readyFrameIndex, std::memory_order_acquire); if (frameIndex < 0 || - m_frameInFlight[frameIndex].load(std::memory_order_acquire) || + Atomic::Load( + m_frameInFlight[frameIndex], std::memory_order_acquire) || lgmpHostQueuePending(m_frameQueue) != 0) return false; @@ -747,14 +753,14 @@ SinkTarget CLGMPFrameTransport::PrepareFrameBuffer( FindAvailableFrameBuffer(allowReady); bool expected = false; const bool acquired = availableFrameIndex >= 0 && - m_frameInFlight[availableFrameIndex].compare_exchange_strong( - expected, true, std::memory_order_acq_rel); + Atomic::CAS(m_frameInFlight[availableFrameIndex], expected, true, + std::memory_order_acq_rel); if (acquired) { const LONG readyFrameIndex = - m_readyFrameIndex.load(std::memory_order_acquire); + Atomic::Load(m_readyFrameIndex, std::memory_order_acquire); if (availableFrameIndex == readyFrameIndex) - m_readyFrameIndex.store( + Atomic::Store(m_readyFrameIndex, FindNewestCompletedFrame( static_cast(availableFrameIndex)), std::memory_order_release); @@ -872,7 +878,7 @@ SinkTarget CLGMPFrameTransport::PrepareFrameBuffer( fi->scheduleGeneration = 0; fi->scheduleEpoch = 0; fi->scheduleDeadlineSerial = 0; - InterlockedExchange((volatile LONG *)&fi->timingValid, 0); + Atomic::Store(fi->timingValid, 0); fi->rotation = FRAME_ROT_0; fi->type = dstFormat.format; @@ -1015,7 +1021,7 @@ bool CLGMPFrameTransport::PublishFrameBuffer(unsigned frameIndex, m_frameDelivered[frameIndex] = deliveredToOwner; m_deferredOwnerFrameIndex = schedule.clientID && !deliveredToOwner ? static_cast(frameIndex) : -1; - m_submittedFrameIndex.store( + Atomic::Store(m_submittedFrameIndex, static_cast(frameIndex), std::memory_order_release); } lock.Unlock(); @@ -1046,15 +1052,18 @@ bool CLGMPFrameTransport::RepublishFrameBuffer( LONG frameIndex = m_deferredOwnerFrameIndex; if (frameIndex >= 0 && !m_frameCompleted[frameIndex] && - !m_frameInFlight[frameIndex].load(std::memory_order_acquire)) + !Atomic::Load( + m_frameInFlight[frameIndex], std::memory_order_acquire)) { m_deferredOwnerFrameIndex = -1; frameIndex = -1; } if (frameIndex < 0) - frameIndex = m_readyFrameIndex.load(std::memory_order_acquire); + frameIndex = Atomic::Load( + m_readyFrameIndex, std::memory_order_acquire); if (frameIndex < 0 || - m_frameInFlight[frameIndex].load(std::memory_order_acquire)) + Atomic::Load( + m_frameInFlight[frameIndex], std::memory_order_acquire)) return false; CFrameScheduler::Schedule deliverySchedule = schedule; @@ -1188,12 +1197,12 @@ void CLGMPFrameTransport::AbortFrameBuffer(unsigned frameIndex) CSRWExclusiveLock lock(m_framePublishLock); m_frameBuffer[frameIndex]->wp = 0; - InterlockedExchange( - (volatile LONG *)&m_frame[frameIndex]->timingValid, 0); + Atomic::Store(m_frame[frameIndex]->timingValid, 0); m_frameCompleted[frameIndex] = false; if (m_deferredOwnerFrameIndex == static_cast(frameIndex)) m_deferredOwnerFrameIndex = -1; - m_frameInFlight[frameIndex].store(false, std::memory_order_release); + Atomic::Store( + m_frameInFlight[frameIndex], false, std::memory_order_release); } void CLGMPFrameTransport::FailFrameBuffer(unsigned frameIndex) @@ -1201,8 +1210,7 @@ void CLGMPFrameTransport::FailFrameBuffer(unsigned frameIndex) if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return; - InterlockedExchange( - (volatile LONG *)&m_frame[frameIndex]->timingValid, 0); + Atomic::Store(m_frame[frameIndex]->timingValid, 0); FinalizeFrameBuffer(frameIndex); AbortFrameBuffer(frameIndex); } @@ -1228,14 +1236,15 @@ void CLGMPFrameTransport::CompleteFrameBuffer( // Completion callbacks may run out of order. Never replace a newer ready // frame with an older submission. const LONG readyFrameIndex = - m_readyFrameIndex.load(std::memory_order_acquire); + Atomic::Load(m_readyFrameIndex, std::memory_order_acquire); if (sequence && (readyFrameIndex < 0 || sequence > m_frameLastPublishSequence[readyFrameIndex])) - m_readyFrameIndex.store( + Atomic::Store(m_readyFrameIndex, static_cast(frameIndex), std::memory_order_release); } - m_frameInFlight[frameIndex].store(false, std::memory_order_release); + Atomic::Store( + m_frameInFlight[frameIndex], false, std::memory_order_release); const bool newerThanReady = sequence && sequence > m_frameReadySequence; if (result == FrameDone::READY && newerThanReady) @@ -1276,7 +1285,7 @@ void CLGMPFrameTransport::SetFrameTiming(unsigned frameIndex, frame->timingFlags = phaseValid ? KVMFR_FRAME_TIMING_PHASE_VALID : 0; frame->timingSerial = frame->frameSerial; - InterlockedExchange((volatile LONG *)&frame->timingValid, 1); + Atomic::Store(frame->timingValid, 1); } void CLGMPFrameTransport::WriteFrameBuffer(unsigned frameIndex, void * src, diff --git a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h index d5364b58..7d7b7518 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h +++ b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h @@ -20,10 +20,10 @@ #pragma once +#include "Atomic.h" #include "CSRWLock.h" #include -#include #include extern "C" { @@ -173,7 +173,7 @@ public: bool allowReadyReplacement = true) override; bool HasPublishedFrame() const override { - return m_readyFrameIndex.load(std::memory_order_acquire) >= 0; + return Atomic::Load(m_readyFrameIndex, std::memory_order_acquire) >= 0; } void ProcessDeliveries() override; bool GetPendingDeliveryTarget( diff --git a/idd/LGIdd/transport/lgmp/CLGMPInputTransport.cpp b/idd/LGIdd/transport/lgmp/CLGMPInputTransport.cpp index 481c4cad..94ca89bb 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPInputTransport.cpp +++ b/idd/LGIdd/transport/lgmp/CLGMPInputTransport.cpp @@ -21,6 +21,7 @@ #include "transport/lgmp/CLGMPInputTransport.h" #include "transport/lgmp/CLGMPHost.h" +#include "Atomic.h" #include "CDebug.h" #include "CSRWLock.h" #include "Seq.h" @@ -201,10 +202,11 @@ bool CLGMPInputTransport::PublishStatus() void CLGMPInputTransport::FlushStatus() { - if (m_statusFailed.load(std::memory_order_acquire) || PublishStatus()) + if (Atomic::Load(m_statusFailed, std::memory_order_acquire) || + PublishStatus()) return; - m_statusFailed.store(true, std::memory_order_release); + Atomic::Store(m_statusFailed, true, std::memory_order_release); CSRWSharedLock lock(m_lifecycleLock); if (m_stopEvent) SetEvent(m_stopEvent); @@ -264,7 +266,7 @@ bool CLGMPInputTransport::Start(IInputTarget& target) Seq::Inc(m_endpointGeneration); m_statusDirty = true; } - m_statusFailed.store(false, std::memory_order_release); + Atomic::Store(m_statusFailed, false, std::memory_order_release); m_thread = CreateThread(nullptr, 0, ThreadProc, this, 0, nullptr); if (!m_thread) { @@ -693,7 +695,7 @@ void CLGMPInputTransport::Thread() } if (!PublishStatus()) { - m_statusFailed.store(true, std::memory_order_release); + Atomic::Store(m_statusFailed, true, std::memory_order_release); failed = true; break; } @@ -716,7 +718,7 @@ void CLGMPInputTransport::Thread() _countof(waitHandles), waitHandles, FALSE, INFINITE); if (wait == WAIT_FIRST_OBJECT_VALUE) { - failed = m_statusFailed.load(std::memory_order_acquire); + failed = Atomic::Load(m_statusFailed, std::memory_order_acquire); break; } if (wait != WAIT_FIRST_OBJECT_VALUE + 1) diff --git a/idd/LGIdd/transport/lgmp/CLGMPInputTransport.h b/idd/LGIdd/transport/lgmp/CLGMPInputTransport.h index aeb96099..4ece3e13 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPInputTransport.h +++ b/idd/LGIdd/transport/lgmp/CLGMPInputTransport.h @@ -20,13 +20,13 @@ #pragma once +#include "Atomic.h" #include "CSRWLock.h" #include "transport/IInputSource.h" #include "common/LGMPConfig.h" #include -#include #include extern "C" { diff --git a/idd/LGIdd/transport/lgmp/CLGMPTransport.cpp b/idd/LGIdd/transport/lgmp/CLGMPTransport.cpp index 0df1f5d5..bb1e4d44 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPTransport.cpp +++ b/idd/LGIdd/transport/lgmp/CLGMPTransport.cpp @@ -20,6 +20,7 @@ #include "transport/lgmp/CLGMPTransport.h" +#include "Atomic.h" #include "CDebug.h" #include "common/KVMFR.h" #include "common/KVMFRRecovery.h" @@ -90,7 +91,7 @@ bool CLGMPTransport::Setup(size_t alignment) if (!m_frames.Setup(alignment)) return false; - m_ready.store(true, std::memory_order_release); + Atomic::Store(m_ready, true, std::memory_order_release); return true; } @@ -109,7 +110,7 @@ ITransport::ProcessResult CLGMPTransport::Process(ITransportEvents& events) // Before the swap chain establishes the frame-buffer alignment, service // only the protocol-independent recovery channel. This preserves the old // transport startup boundary while keeping recovery available immediately. - if (!m_ready.load(std::memory_order_acquire)) + if (!Atomic::Load(m_ready, std::memory_order_acquire)) return ProcessResult::OK; const LGMP_STATUS processStatus = m_host.Process(); @@ -224,7 +225,7 @@ ITransport::ProcessResult CLGMPTransport::Process(ITransportEvents& events) void CLGMPTransport::Stop() { - m_ready.store(false, std::memory_order_release); + Atomic::Store(m_ready, false, std::memory_order_release); m_input.Stop(); } diff --git a/idd/LGIdd/transport/lgmp/CLGMPTransport.h b/idd/LGIdd/transport/lgmp/CLGMPTransport.h index 265727ae..903e544b 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPTransport.h +++ b/idd/LGIdd/transport/lgmp/CLGMPTransport.h @@ -20,6 +20,7 @@ #pragma once +#include "Atomic.h" #include "transport/ITransport.h" #include "transport/lgmp/CIVSHMEM.h" #include "transport/lgmp/CLGMPControl.h" @@ -28,8 +29,6 @@ #include "transport/lgmp/CLGMPInputTransport.h" #include "transport/lgmp/CRecovery.h" -#include - class CLGMPTransport final : public ITransport { private: diff --git a/idd/LGIdd/transport/lgmp/CRecovery.cpp b/idd/LGIdd/transport/lgmp/CRecovery.cpp index d060fb80..e9f14fde 100644 --- a/idd/LGIdd/transport/lgmp/CRecovery.cpp +++ b/idd/LGIdd/transport/lgmp/CRecovery.cpp @@ -22,6 +22,7 @@ #include "transport/lgmp/CIVSHMEM.h" #include "platform/CPlatformInfo.h" +#include "Atomic.h" #include "CDebug.h" #include "VersionInfo.h" @@ -38,36 +39,6 @@ namespace static const uint64_t HELPER_TIMEOUT_MS = 30000; CSRWLock l_wireLock; - uint32_t AtomicRead(uint32_t& value) - { - return static_cast(InterlockedCompareExchange( - (volatile LONG *)&value, 0, 0)); - } - - void AtomicWrite(uint32_t& value, uint32_t data) - { - InterlockedExchange((volatile LONG *)&value, static_cast(data)); - } - - void AtomicIncrement(uint32_t& value) - { - InterlockedIncrement((volatile LONG *)&value); - } - - uint32_t AtomicAdd(uint32_t& value, uint32_t data) - { - return static_cast(InterlockedExchangeAdd( - (volatile LONG *)&value, static_cast(data))) + data; - } - - bool AtomicCompareExchange( - uint32_t& value, uint32_t expected, uint32_t data) - { - return static_cast(InterlockedCompareExchange( - (volatile LONG *)&value, static_cast(data), - static_cast(expected))) == expected; - } - uint64_t CreateSession(const void * memory, uint64_t previous) { LARGE_INTEGER counter; @@ -89,13 +60,13 @@ namespace bool CRecovery::OwnsSession() { - if (AtomicRead(m_data->header.ready) != KVMFR_R_READY) + if (Atomic::Load(m_data->header.ready) != KVMFR_R_READY) return false; const uint64_t session = m_data->header.session; - MemoryBarrier(); + Atomic::Fence(); return session == m_session && - AtomicRead(m_data->header.ready) == KVMFR_R_READY; + Atomic::Load(m_data->header.ready) == KVMFR_R_READY; } bool CRecovery::ReadRequest( @@ -103,14 +74,14 @@ bool CRecovery::ReadRequest( { for (unsigned i = 0; i < 4; ++i) { - const uint32_t serial = AtomicRead(source.serial); + const uint32_t serial = Atomic::Load(source.serial); if (!serial || (serial & 1U)) return false; const uint32_t type = source.request; const uint64_t session = source.session; - MemoryBarrier(); - if (AtomicRead(source.serial) == serial) + Atomic::Fence(); + if (Atomic::Load(source.serial) == serial) { result.serial = serial; result.request = type; @@ -126,7 +97,7 @@ bool CRecovery::ReadStatus(KVMFRRStatus& source, KVMFRRStatus& result) { for (unsigned i = 0; i < 4; ++i) { - const uint32_t serial = AtomicRead(source.serial); + const uint32_t serial = Atomic::Load(source.serial); if (!serial || (serial & 1U)) continue; @@ -135,8 +106,8 @@ bool CRecovery::ReadStatus(KVMFRRStatus& source, KVMFRRStatus& result) result.state = source.state; result.error = source.error; result.session = source.session; - MemoryBarrier(); - if (AtomicRead(source.serial) == serial) + Atomic::Fence(); + if (Atomic::Load(source.serial) == serial) { result.serial = serial; return true; @@ -154,10 +125,7 @@ bool CRecovery::SerialNewer(uint32_t serial, uint32_t reference) uint32_t CRecovery::NextTicket() { - uint32_t ticket = AtomicAdd(m_data->req.ticket, 2U); - if (!ticket) - ticket = AtomicAdd(m_data->req.ticket, 2U); - return ticket; + return Atomic::Next(m_data->req.ticket, 2U); } void CRecovery::Publish(uint32_t serial, uint32_t request, @@ -168,13 +136,13 @@ void CRecovery::Publish(uint32_t serial, uint32_t request, if (!published) published = KVMFR_R_REQ_FIRST; - AtomicWrite(m_data->status.serial, writing); + Atomic::Store(m_data->status.serial, writing); m_data->status.ackRequest = request; m_data->status.state = state; m_data->status.error = error; m_data->status.session = m_session; m_data->status.ackSerial = serial; - AtomicWrite(m_data->status.serial, published); + Atomic::Store(m_data->status.serial, published); m_statusSerial = published; } @@ -192,7 +160,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem) return false; } - const uint32_t oldReady = AtomicRead(m_data->header.ready); + const uint32_t oldReady = Atomic::Load(m_data->header.ready); const bool oldValid = oldReady == KVMFR_R_READY && memcmp(m_data->header.magic, KVMFR_R_MAGIC, sizeof(m_data->header.magic)) == 0 && @@ -220,7 +188,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem) } } - AtomicWrite(m_data->header.ready, 0); + Atomic::Store(m_data->header.ready, 0); if (oldValid) { ZeroMemory(&m_data->header, sizeof(m_data->header)); @@ -233,8 +201,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem) { KVMFRRRequest request = {}; if (ReadRequest(m_data->requests[i], request)) - AtomicCompareExchange( - m_data->requests[i].serial, request.serial, 0); + Atomic::CAS(m_data->requests[i].serial, request.serial, 0); } } else @@ -264,7 +231,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem) KVMFR_R_STATE_SWITCHING, KVMFR_R_ERR_NONE); m_nextHeartbeat = GetTickCount64() + KVMFR_R_HEARTBEAT_MS; - AtomicWrite(m_data->header.ready, KVMFR_R_READY); + Atomic::Store(m_data->header.ready, KVMFR_R_READY); DEBUG_INFO("Recovery channel initialized (session %llu%s)", (unsigned long long)m_session, @@ -289,7 +256,7 @@ CRecovery::Request CRecovery::Process() const uint64_t now = GetTickCount64(); if (now >= m_nextHeartbeat) { - AtomicIncrement(m_data->header.heartbeat); + Atomic::Inc(m_data->header.heartbeat); m_nextHeartbeat = now + KVMFR_R_HEARTBEAT_MS; } @@ -371,8 +338,7 @@ CRecovery::Request CRecovery::Process() // slot without a corresponding acknowledgement. for (unsigned i = 0; i < KVMFR_R_REQ_SLOTS; ++i) if (stable[i]) - AtomicCompareExchange( - m_data->requests[i].serial, requests[i].serial, 0); + Atomic::CAS(m_data->requests[i].serial, requests[i].serial, 0); if (m_waiting && now >= m_deadline) {