Mirrored_Repos/LookingGlass
1
0
Fork 0
mirror of https://github.com/gnif/LookingGlass.git synced 2026-08-22 07:01:30 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

[idd] common: centralize atomic operations

Browse Source
This commit is contained in:
Geoffrey McRae
2026-08-13 20:18:18 +10:00
parent 5d12ee4bce
commit af309de438
27 changed files with 477 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
idd/LGCommon/Atomic.h Normal file
View File

@@ -0,0 +1,188 @@
/**
* Looking Glass
* Copyright © 2017-2026 The Looking Glass Authors
* https://looking-glass.io
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc., 59
* Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#pragma once
#include "Seq.h"
#include <Windows.h>
#include <atomic>
#include <stdint.h>
namespace Atomic
{
namespace Detail
{
inline volatile LONG * Ptr(uint32_t& value)
{
static_assert(sizeof(value) == sizeof(LONG),
"atomic value must match the Windows interlocked width");
return reinterpret_cast<volatile LONG *>(&value);
}
}
template<typename T>
T Load(const std::atomic<T>& value,
std::memory_order order = std::memory_order_seq_cst)
{
return value.load(order);
}
template<typename T, typename U>
void Store(std::atomic<T>& value, U data,
std::memory_order order = std::memory_order_seq_cst)
{
value.store(static_cast<T>(data), order);
}
template<typename T, typename U>
T Swap(std::atomic<T>& value, U data,
std::memory_order order = std::memory_order_seq_cst)
{
return value.exchange(static_cast<T>(data), order);
}
template<typename T, typename U>
T FetchAdd(std::atomic<T>& value, U data,
std::memory_order order = std::memory_order_seq_cst)
{
return value.fetch_add(static_cast<T>(data), order);
}
template<typename T, typename U>
T FetchSub(std::atomic<T>& value, U data,
std::memory_order order = std::memory_order_seq_cst)
{
return value.fetch_sub(static_cast<T>(data), order);
}
template<typename T, typename U>
bool CAS(std::atomic<T>& value, T& expected, U data,
std::memory_order order = std::memory_order_seq_cst)
{
return value.compare_exchange_strong(
expected, static_cast<T>(data), order);
}
template<typename T, typename U>
bool CAS(std::atomic<T>& value, T& expected, U data,
std::memory_order success, std::memory_order failure)
{
return value.compare_exchange_strong(
expected, static_cast<T>(data), success, failure);
}
template<typename T, typename U>
bool CASWeak(std::atomic<T>& value, T& expected, U data,
std::memory_order order = std::memory_order_seq_cst)
{
return value.compare_exchange_weak(
expected, static_cast<T>(data), order);
}
template<typename T, typename U>
bool CASWeak(std::atomic<T>& value, T& expected, U data,
std::memory_order success, std::memory_order failure)
{
return value.compare_exchange_weak(
expected, static_cast<T>(data), success, failure);
}
template<typename T>
T Inc(std::atomic<T>& value,
std::memory_order order = std::memory_order_seq_cst)
{
return FetchAdd(value, static_cast<T>(1), order) + 1;
}
template<typename T>
T Next(std::atomic<T>& value,
std::memory_order order = std::memory_order_relaxed)
{
T current = Load(value, std::memory_order_relaxed);
for (;;)
{
const T next = Seq::Next(current);
if (CASWeak(value, current, next, order))
return next;
}
}
inline uint32_t Load(uint32_t& value)
{
return static_cast<uint32_t>(InterlockedCompareExchange(
Detail::Ptr(value), 0, 0));
}
inline void Store(uint32_t& value, uint32_t data)
{
InterlockedExchange(Detail::Ptr(value), static_cast<LONG>(data));
}
inline uint32_t Swap(uint32_t& value, uint32_t data)
{
return static_cast<uint32_t>(
InterlockedExchange(Detail::Ptr(value), static_cast<LONG>(data)));
}
inline uint32_t FetchAdd(uint32_t& value, uint32_t data)
{
return static_cast<uint32_t>(
InterlockedExchangeAdd(Detail::Ptr(value), static_cast<LONG>(data)));
}
inline uint32_t FetchSub(uint32_t& value, uint32_t data)
{
return static_cast<uint32_t>(InterlockedExchangeAdd(
Detail::Ptr(value), static_cast<LONG>(0U - data)));
}
inline bool CAS(uint32_t& value, uint32_t expected, uint32_t data)
{
const uint32_t actual = static_cast<uint32_t>(InterlockedCompareExchange(
Detail::Ptr(value), static_cast<LONG>(data),
static_cast<LONG>(expected)));
return actual == expected;
}
inline bool CASWeak(uint32_t& value, uint32_t expected, uint32_t data)
{
return CAS(value, expected, data);
}
inline uint32_t Inc(uint32_t& value)
{
return static_cast<uint32_t>(InterlockedIncrement(Detail::Ptr(value)));
}
inline uint32_t Next(uint32_t& value, uint32_t step = 1)
{
uint32_t result = FetchAdd(value, step) + step;
if (!result)
result = FetchAdd(value, step) + step;
return result;
}
inline void Fence()
{
MemoryBarrier();
}
}

24
idd/LGCommon/CPipeEndpoint.cpp
View File

@@ -217,12 +217,12 @@ bool CPipeEndpoint::Start(
PublishPipe(pipe);
}
m_running.store(true);
Atomic::Store(m_running, true);
m_thread = CreateThread(nullptr, 0, ThreadProc, this, 0, nullptr);
if (!m_thread)
{
DEBUG_ERROR_HR(GetLastError(), "Failed to create named pipe thread");
m_running.store(false);
Atomic::Store(m_running, false);
{
CSRWExclusiveLock lock(m_pipeLock);
@@ -245,8 +245,8 @@ bool CPipeEndpoint::Start(
void CPipeEndpoint::Stop()
{
m_running.store(false);
m_connected.store(false);
Atomic::Store(m_running, false);
Atomic::Store(m_connected, false);
if (m_stopEvent)
SetEvent(m_stopEvent);
@@ -284,7 +284,7 @@ void CPipeEndpoint::Stop()
m_writeEvent = nullptr;
}
m_connected.store(false);
Atomic::Store(m_connected, false);
}
bool CPipeEndpoint::Send(const void * message, size_t size)
@@ -303,7 +303,7 @@ bool CPipeEndpoint::Send(const void * message, size_t size)
success = result == PipeIoResult::Success;
if (!success)
{
m_connected.store(false);
Atomic::Store(m_connected, false);
CancelIoEx(m_pipe, nullptr);
}
}
@@ -323,8 +323,8 @@ void CPipeEndpoint::Thread()
else
RunClient();
m_running.store(false);
m_connected.store(false);
Atomic::Store(m_running, false);
Atomic::Store(m_connected, false);
}
HANDLE CPipeEndpoint::CreateServerPipe()
@@ -421,14 +421,14 @@ void CPipeEndpoint::RunServer()
WaitForSingleObject(m_stopEvent, 0) == WAIT_FIRST_OBJECT_VALUE)
break;
m_connected.store(true);
Atomic::Store(m_connected, true);
DEBUG_INFO("Named pipe client connected: %ls", m_pipeName.c_str());
if (m_handler)
m_handler->OnPipeConnected();
ReadMessages(pipe);
m_connected.store(false);
Atomic::Store(m_connected, false);
if (m_handler)
m_handler->OnPipeDisconnected();
DEBUG_INFO("Named pipe client disconnected: %ls", m_pipeName.c_str());
@@ -504,7 +504,7 @@ void CPipeEndpoint::RunClient()
}
PublishPipe(pipe);
m_connected.store(true);
Atomic::Store(m_connected, true);
retryDelay = CLIENT_RETRY_INITIAL_MS;
lastConnectError = ERROR_SUCCESS;
DEBUG_INFO("Named pipe connected: %ls", m_pipeName.c_str());
@@ -513,7 +513,7 @@ void CPipeEndpoint::RunClient()
ReadMessages(pipe);
m_connected.store(false);
Atomic::Store(m_connected, false);
if (m_handler)
m_handler->OnPipeDisconnected();
DEBUG_INFO("Named pipe disconnected: %ls", m_pipeName.c_str());

6
idd/LGCommon/CPipeEndpoint.h
View File

@@ -20,11 +20,11 @@
#pragma once
#include "Atomic.h"
#include "CSRWLock.h"
#include <Windows.h>
#include <atomic>
#include <stddef.h>
#include <string>
@@ -66,8 +66,8 @@ public:
_In_reads_bytes_(size) const void * message,
_In_ size_t size);
bool IsRunning() const { return m_running.load(); }
bool IsConnected() const { return m_connected.load(); }
bool IsRunning() const { return Atomic::Load(m_running); }
bool IsConnected() const { return Atomic::Load(m_connected); }
void SetHandler(_In_opt_ IPipeEndpointHandler * handler)
{

1
idd/LGCommon/LGCommon.vcxproj
View File

@@ -70,6 +70,7 @@
<ClCompile Include="RefreshRate.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="Atomic.h" />
<ClInclude Include="CDebug.h" />
<ClInclude Include="CPipeEndpoint.h" />
<ClInclude Include="CSRWLock.h" />

3
idd/LGCommon/LGCommon.vcxproj.filters
View File

@@ -23,6 +23,9 @@
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="Atomic.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="CDebug.h">
<Filter>Header Files</Filter>
</ClInclude>

11
idd/LGIdd/capture/CFrameBufferResource.h
View File

@@ -20,11 +20,12 @@
#pragma once
#include "Atomic.h"
#include <Windows.h>
#include <wdf.h>
#include <wrl.h>
#include <d3d12.h>
#include <atomic>
#include <stdint.h>
#include "capture/CFrameScheduler.h"
@@ -112,15 +113,17 @@ class CFrameBufferResource
}
void ResetCompletion()
{
m_completionHandled.store(false, std::memory_order_release);
Atomic::Store(
m_completionHandled, false, std::memory_order_release);
}
void MarkCompletion()
{
m_completionHandled.store(true, std::memory_order_release);
Atomic::Store(
m_completionHandled, true, std::memory_order_release);
}
bool CompletionHandled() const
{
return m_completionHandled.load(std::memory_order_acquire);
return Atomic::Load(m_completionHandled, std::memory_order_acquire);
}
void SetCandidateIndex(unsigned index) { m_candidateIndex = index; }
unsigned GetCandidateIndex() const { return m_candidateIndex; }

7
idd/LGIdd/capture/CSwapChainProcessor.cpp
View File

@@ -20,6 +20,7 @@
#include "capture/CSwapChainProcessor.h"
#include "capture/CFrameProcessorUtil.h"
#include "Atomic.h"
#include "CSRWLock.h"
#include "display/IddCxCompat.h"
#include "display/CDeviceContext.h"
@@ -355,7 +356,8 @@ void CSwapChainProcessor::SwapChainThreadCore()
surface = buffer.MetaData.pSurface;
colorSpace = buffer.MetaData.SurfaceColorSpace;
sdrWhiteLevel = buffer.MetaData.SdrWhiteLevel;
m_sdrWhiteLevel.store(sdrWhiteLevel, std::memory_order_relaxed);
Atomic::Store(
m_sdrWhiteLevel, sdrWhiteLevel, std::memory_order_relaxed);
UpdateHDRMetadata(buffer.MetaData);
}
}
@@ -865,7 +867,8 @@ bool CSwapChainProcessor::QueryHWCursor()
in.ShapeBufferSizeInBytes = 512 * 512 * 4;
IDARG_OUT_QUERY_HWCURSOR out = {};
UINT cursorWhiteLevel = m_sdrWhiteLevel.load(std::memory_order_relaxed);
UINT cursorWhiteLevel =
Atomic::Load(m_sdrWhiteLevel, std::memory_order_relaxed);
NTSTATUS status;
#ifdef HAS_IDDCX_110
if (m_devContext->HasIddCx110DDIs())

2
idd/LGIdd/capture/CSwapChainProcessor.h
View File

@@ -20,6 +20,7 @@
#pragma once
#include "Atomic.h"
#include "d3d/CD3D11Device.h"
#include "d3d/CD3D12Device.h"
#include "display/IddCxCompat.h"
@@ -30,7 +31,6 @@
#include <Windows.h>
#include <wrl.h>
#include <atomic>
#include <memory>
using namespace Microsoft::WRL;

63
idd/LGIdd/d3d/CD3D12CommandQueue.cpp
View File

@@ -147,11 +147,12 @@ void CD3D12CommandSlot::DeInit()
bool CD3D12CommandSlot::Acquire()
{
if (!m_queue || m_queue->m_failed.load(std::memory_order_acquire))
if (!m_queue || Atomic::Load(
m_queue->m_failed, std::memory_order_acquire))
return false;
State expected = STATE_FREE;
if (!m_state.compare_exchange_strong(expected, STATE_RECORDING,
if (!Atomic::CAS(m_state, expected, STATE_RECORDING,
std::memory_order_acq_rel))
return false;
@@ -165,7 +166,7 @@ bool CD3D12CommandSlot::Acquire()
m_timestampFrequency = 0;
m_calibrationGPU = 0;
m_calibrationCPU = 0;
m_submitted.store(false, std::memory_order_release);
Atomic::Store(m_submitted, false, std::memory_order_release);
for (UINT i = 0; i < MAX_FENCE_WAITS; ++i)
{
@@ -180,8 +181,8 @@ bool CD3D12CommandSlot::Acquire()
if (FAILED(hr))
{
DEBUG_ERROR_HR(hr, "Failed to reset the CommandAllocator (%ls)", m_name);
m_queue->m_failed.store(true, std::memory_order_release);
m_state.store(STATE_FAILED, std::memory_order_release);
Atomic::Store(m_queue->m_failed, true, std::memory_order_release);
Atomic::Store(m_state, STATE_FAILED, std::memory_order_release);
return false;
}
@@ -189,8 +190,8 @@ bool CD3D12CommandSlot::Acquire()
if (FAILED(hr))
{
DEBUG_ERROR_HR(hr, "Failed to reset the CommandList (%ls)", m_name);
m_queue->m_failed.store(true, std::memory_order_release);
m_state.store(STATE_FAILED, std::memory_order_release);
Atomic::Store(m_queue->m_failed, true, std::memory_order_release);
Atomic::Store(m_state, STATE_FAILED, std::memory_order_release);
return false;
}
@@ -201,7 +202,7 @@ bool CD3D12CommandSlot::Acquire()
void CD3D12CommandSlot::Cancel()
{
State expected = STATE_RECORDING;
if (!m_state.compare_exchange_strong(expected, STATE_CANCELLING,
if (!Atomic::CAS(m_state, expected, STATE_CANCELLING,
std::memory_order_acq_rel))
{
DEBUG_ERROR("Command slot cancelled while not recording (%ls)", m_name);
@@ -223,23 +224,23 @@ void CD3D12CommandSlot::Cancel()
{
DEBUG_ERROR_HR(hr, "Failed to close the cancelled CommandList (%ls)",
m_name);
m_queue->m_failed.store(true, std::memory_order_release);
m_state.store(STATE_FAILED, std::memory_order_release);
Atomic::Store(m_queue->m_failed, true, std::memory_order_release);
Atomic::Store(m_state, STATE_FAILED, std::memory_order_release);
return;
}
m_completionCallback = nullptr;
m_completionParams[0] = nullptr;
m_completionParams[1] = nullptr;
m_submitted.store(false, std::memory_order_release);
m_state.store(STATE_FREE, std::memory_order_release);
Atomic::Store(m_submitted, false, std::memory_order_release);
Atomic::Store(m_state, STATE_FREE, std::memory_order_release);
SetEvent(m_availableEvent.Get());
}
bool CD3D12CommandSlot::Execute()
{
State expected = STATE_RECORDING;
if (!m_state.compare_exchange_strong(expected, STATE_SUBMITTED,
if (!Atomic::CAS(m_state, expected, STATE_SUBMITTED,
std::memory_order_acq_rel))
{
DEBUG_ERROR("Command slot executed while not recording (%ls)", m_name);
@@ -251,17 +252,17 @@ bool CD3D12CommandSlot::Execute()
if (FAILED(hr))
{
DEBUG_ERROR_HR(hr, "Failed to close the CommandList (%ls)", m_name);
m_queue->m_failed.store(true, std::memory_order_release);
m_state.store(STATE_FAILED, std::memory_order_release);
Atomic::Store(m_queue->m_failed, true, std::memory_order_release);
Atomic::Store(m_state, STATE_FAILED, std::memory_order_release);
return false;
}
if (m_queue->Submit(*this))
return true;
if (!m_submitted.load(std::memory_order_acquire))
if (!Atomic::Load(m_submitted, std::memory_order_acquire))
{
m_state.store(STATE_FREE, std::memory_order_release);
Atomic::Store(m_state, STATE_FREE, std::memory_order_release);
SetEvent(m_availableEvent.Get());
}
return false;
@@ -270,7 +271,7 @@ bool CD3D12CommandSlot::Execute()
bool CD3D12CommandSlot::WaitFor(ID3D12Fence * fence, UINT64 value)
{
if (!fence || !value ||
m_state.load(std::memory_order_acquire) != STATE_RECORDING)
Atomic::Load(m_state, std::memory_order_acquire) != STATE_RECORDING)
return false;
if (m_fenceWaitCount == MAX_FENCE_WAITS)
@@ -331,7 +332,7 @@ bool CD3D12CommandSlot::GetGPUTimes(
void CD3D12CommandSlot::OnCompletion(bool timeout)
{
if (!m_queue || !m_submitted.load(std::memory_order_acquire))
if (!m_queue || !Atomic::Load(m_submitted, std::memory_order_acquire))
return;
const UINT64 completed = m_queue->m_fence->GetCompletedValue();
@@ -339,13 +340,13 @@ void CD3D12CommandSlot::OnCompletion(bool timeout)
return;
State expected = STATE_SUBMITTED;
if (!m_state.compare_exchange_strong(expected, STATE_COMPLETING,
if (!Atomic::CAS(m_state, expected, STATE_COMPLETING,
std::memory_order_acq_rel))
return;
m_completionResult = !timeout && completed != UINT64_MAX;
if (!m_completionResult)
m_queue->m_failed.store(true, std::memory_order_release);
Atomic::Store(m_queue->m_failed, true, std::memory_order_release);
if (m_completionCallback)
m_completionCallback(this, m_completionResult,
@@ -354,8 +355,8 @@ void CD3D12CommandSlot::OnCompletion(bool timeout)
m_completionCallback = nullptr;
m_completionParams[0] = nullptr;
m_completionParams[1] = nullptr;
m_submitted.store(false, std::memory_order_release);
m_state.store(STATE_FREE, std::memory_order_release);
Atomic::Store(m_submitted, false, std::memory_order_release);
Atomic::Store(m_state, STATE_FREE, std::memory_order_release);
SetEvent(m_availableEvent.Get());
}
@@ -518,7 +519,7 @@ CD3D12CommandSlot * CD3D12CommandQueue::Acquire(UINT slotIndex)
m_slots[slotIndex].OnCompletion(false);
if (m_slots[slotIndex].Acquire())
return &m_slots[slotIndex];
if (m_failed.load(std::memory_order_acquire))
if (Atomic::Load(m_failed, std::memory_order_acquire))
break;
const ULONGLONG now = GetTickCount64();
@@ -579,7 +580,7 @@ bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot)
do
{
if (m_failed.load(std::memory_order_relaxed))
if (Atomic::Load(m_failed, std::memory_order_relaxed))
break;
for (UINT i = 0; i < slot.m_fenceWaitCount; ++i)
@@ -589,16 +590,16 @@ bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot)
if (FAILED(hr))
{
DEBUG_ERROR_HR(hr, "Failed to queue a fence wait (%ls)", m_name);
m_failed.store(true, std::memory_order_release);
Atomic::Store(m_failed, true, std::memory_order_release);
break;
}
}
if (m_failed.load(std::memory_order_relaxed))
if (Atomic::Load(m_failed, std::memory_order_relaxed))
break;
const UINT64 fenceTarget = ++m_fenceValue;
slot.m_fenceTarget = fenceTarget;
slot.m_submitted.store(true, std::memory_order_release);
Atomic::Store(slot.m_submitted, true, std::memory_order_release);
ID3D12CommandList * lists[] = { slot.m_cmdList.Get() };
m_queue->ExecuteCommandLists(1, lists);
@@ -607,7 +608,7 @@ bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot)
if (FAILED(hr))
{
DEBUG_ERROR_HR(hr, "Failed to signal the CommandQueue (%ls)", m_name);
m_failed.store(true, std::memory_order_release);
Atomic::Store(m_failed, true, std::memory_order_release);
slot.OnCompletion(false);
break;
}
@@ -621,10 +622,10 @@ bool CD3D12CommandQueue::Submit(CD3D12CommandSlot& slot)
// The work is already submitted and fenced. Poll only on this rare
// error path so allocator, callback, and framebuffer ownership remain
// valid until completion or confirmed device removal.
while (slot.m_submitted.load(std::memory_order_acquire))
while (Atomic::Load(slot.m_submitted, std::memory_order_acquire))
{
slot.OnCompletion(false);
if (slot.m_submitted.load(std::memory_order_acquire))
if (Atomic::Load(slot.m_submitted, std::memory_order_acquire))
Sleep(1);
}

8
idd/LGIdd/d3d/CD3D12CommandQueue.h
View File

@@ -20,13 +20,13 @@
#pragma once
#include "Atomic.h"
#include "CSRWLock.h"
#include <Windows.h>
#include <wdf.h>
#include <wrl.h>
#include <d3d12.h>
#include <atomic>
#include <stdint.h>
using namespace Microsoft::WRL;
@@ -123,15 +123,15 @@ class CD3D12CommandSlot
bool IsIdle() const
{
const State state = m_state.load(std::memory_order_acquire);
const State state = Atomic::Load(m_state, std::memory_order_acquire);
return state == STATE_FREE ||
(state == STATE_FAILED &&
!m_submitted.load(std::memory_order_acquire));
!Atomic::Load(m_submitted, std::memory_order_acquire));
}
bool HasSubmittedWork() const
{
return m_submitted.load(std::memory_order_acquire);
return Atomic::Load(m_submitted, std::memory_order_acquire);
}
ComPtr<ID3D12GraphicsCommandList> GetGfxList() { return m_gfxList; }

17
idd/LGIdd/display/CDeviceContext.cpp
View File

@@ -26,6 +26,7 @@
#include "transport/IFrameTransport.h"
#include "transport/IInputTransport.h"
#include "transport/TransportFactory.h"
#include "Atomic.h"
#include "CDebug.h"
#include <dxgi1_2.h>
@@ -168,7 +169,7 @@ void CDeviceContext::InitAdapter()
}
LONG initExpected = 0;
if (!m_initInProgress.compare_exchange_strong(initExpected, 1))
if (!Atomic::CAS(m_initInProgress, initExpected, 1))
{
DEBUG_TRACE("Adapter initialization skipped: initialization already in progress");
return;
@@ -182,7 +183,7 @@ void CDeviceContext::InitAdapter()
if (!m_transport)
{
DEBUG_ERROR("Failed to create the frame transport");
m_initInProgress.store(0);
Atomic::Store(m_initInProgress, 0);
return;
}
@@ -196,7 +197,7 @@ void CDeviceContext::InitAdapter()
}
else
DEBUG_ERROR("Failed to open the frame transport");
m_initInProgress.store(0);
Atomic::Store(m_initInProgress, 0);
return;
}
m_transportOpened = true;
@@ -268,13 +269,13 @@ void CDeviceContext::InitAdapter()
DEBUG_TRACE("Initializing frame transport metadata");
if (!InitializeTransport())
{
m_initInProgress.store(0);
Atomic::Store(m_initInProgress, 0);
return;
}
DEBUG_TRACE("Loading configured display modes");
if (!m_displayConfiguration.Load(*m_transport))
{
m_initInProgress.store(0);
Atomic::Store(m_initInProgress, 0);
return;
}
DEBUG_TRACE("Initializing monitor EDID");
@@ -347,7 +348,7 @@ void CDeviceContext::InitAdapter()
if (!NT_SUCCESS(status))
{
DEBUG_ERROR_HR(status, "IddCxAdapterInitAsync Failed");
m_initInProgress.store(0);
Atomic::Store(m_initInProgress, 0);
return;
}
@@ -355,7 +356,7 @@ void CDeviceContext::InitAdapter()
if (!m_adapter)
{
DEBUG_ERROR("IddCxAdapterInitAsync succeeded without returning an adapter object");
m_initInProgress.store(0);
Atomic::Store(m_initInProgress, 0);
return;
}
@@ -367,7 +368,7 @@ void CDeviceContext::InitAdapter()
// Adapter is up; no need to keep retrying.
StopInitRetry();
m_initInProgress.store(0);
Atomic::Store(m_initInProgress, 0);
DEBUG_INFO("Adapter initialization request complete; returning to IddCx");
}

3
idd/LGIdd/display/CDeviceContext.h
View File

@@ -20,11 +20,12 @@
#pragma once
#include "Atomic.h"
#include <Windows.h>
#include <wdf.h>
#include <IddCx.h>
#include <atomic>
#include <memory>
#include <stddef.h>
#include <stdint.h>

7
idd/LGIdd/display/CMonitorContext.cpp
View File

@@ -48,8 +48,8 @@ NTSTATUS CMonitorContext::AssignSwapChain(
// new generation is established.
DetachSwapChain();
const UINT64 assignmentGeneration =
m_assignmentGeneration.fetch_add(1, std::memory_order_acq_rel) + 1;
const UINT64 assignmentGeneration = Atomic::FetchAdd(
m_assignmentGeneration, 1, std::memory_order_acq_rel) + 1;
// Build the D3D11 device into a local so the member is never observed
// half-constructed. The worker binds it before performing the expensive
@@ -96,7 +96,8 @@ void CMonitorContext::DetachSwapChain()
// Invalidate setup in progress before waiting for m_lock. This also lets a
// worker about to call SetDevice observe an unassign whose callback is
// blocked waiting for the processor to be published.
m_assignmentGeneration.fetch_add(1, std::memory_order_acq_rel);
Atomic::FetchAdd(
m_assignmentGeneration, 1, std::memory_order_acq_rel);
// Detach under the lock, then destroy outside it. Destroying the processor
// joins its worker thread, whose teardown (WdfObjectDelete) re-enters this

5
idd/LGIdd/display/CMonitorContext.h
View File

@@ -20,13 +20,13 @@
#pragma once
#include "Atomic.h"
#include "CSRWLock.h"
#include <Windows.h>
#include <wdf.h>
#include <IddCx.h>
#include <atomic>
#include <memory>
#include <mutex>
@@ -71,7 +71,8 @@ public:
void UnassignSwapChain();
bool IsAssignmentCurrent(UINT64 generation) const
{
return m_assignmentGeneration.load(std::memory_order_acquire) == generation;
return Atomic::Load(
m_assignmentGeneration, std::memory_order_acquire) == generation;
}
CDeviceContext * GetDeviceContext() { return m_devContext; }

13
idd/LGIdd/display/CMonitorManager.cpp
View File

@@ -21,6 +21,7 @@
#include "display/CMonitorManager.h"
#include "display/CMonitorContext.h"
#include "Atomic.h"
#include "CDebug.h"
bool CMonitorManager::Create(UINT connectorIndex, IDDCX_ADAPTER adapter,
@@ -133,7 +134,7 @@ CMonitorManager::ReplugAction CMonitorManager::Replug()
{
// Either no monitor yet, or one is already pending; build it now and
// cancel any queued rebuild so we do not create two.
m_createQueued.store(0);
Atomic::Store(m_createQueued, 0);
return ReplugAction::CREATE;
}
@@ -163,7 +164,7 @@ CMonitorManager::ReplugAction CMonitorManager::Replug()
// If there was no swap chain there will be no unassign callback to queue
// the rebuild. Otherwise OnSwapChainReleased does so after teardown drains.
if (rebuild)
m_createQueued.store(1);
Atomic::Store(m_createQueued, 1);
return ReplugAction::NONE;
}
@@ -205,7 +206,7 @@ void CMonitorManager::OnSwapChainReleased()
}
if (rebuild)
m_createQueued.store(1);
Atomic::Store(m_createQueued, 1);
}
CMonitorManager::ReadyAction CMonitorManager::OnSwapChainReady()
@@ -249,15 +250,15 @@ CMonitorManager::ReadyAction CMonitorManager::OnSwapChainReady()
void CMonitorManager::QueueReplug()
{
m_replugQueued.store(1);
Atomic::Store(m_replugQueued, 1);
}
CMonitorManager::DeferredAction CMonitorManager::TakeDeferredAction()
{
if (m_createQueued.exchange(0))
if (Atomic::Swap(m_createQueued, 0))
return DeferredAction::CREATE;
if (m_replugQueued.exchange(0))
if (Atomic::Swap(m_replugQueued, 0))
return DeferredAction::REPLUG;
return DeferredAction::NONE;

2
idd/LGIdd/display/CMonitorManager.h
View File

@@ -20,11 +20,11 @@
#pragma once
#include "Atomic.h"
#include "CSRWLock.h"
#include <Windows.h>
#include <wdf.h>
#include <IddCx.h>
#include <atomic>
#include <vector>
#include "config/CSettings.h"

43
idd/LGIdd/ipc/CInputPipeServer.cpp
View File

@@ -34,7 +34,7 @@ bool CInputPipeServer::Init()
{
DeInit();
m_state.store(0, std::memory_order_release);
Atomic::Store(m_state, 0, std::memory_order_release);
m_performanceFrequency.QuadPart = 0;
if (!QueryPerformanceFrequency(&m_performanceFrequency))
m_performanceFrequency.QuadPart = 0;
@@ -183,7 +183,7 @@ bool CInputPipeServer::QueueRawLocked(
(m_queueHead + m_queueCount) % QUEUE_LENGTH;
m_queue[index].type = type;
m_queue[index].state =
m_state.load(std::memory_order_relaxed);
Atomic::Load(m_state, std::memory_order_relaxed);
m_queue[index].payload = payload;
m_queue[index].pureMotion = pureMotion;
++m_queueCount;
@@ -236,11 +236,11 @@ void CInputPipeServer::ResyncLocked()
m_statResyncDiscarded += m_queueCount;
m_queueHead = 0;
m_queueCount = 0;
uint64_t state = m_state.load(std::memory_order_relaxed);
uint64_t state = Atomic::Load(m_state, std::memory_order_relaxed);
while (state & 1)
{
if (m_state.compare_exchange_weak(
state, state + 2, std::memory_order_acq_rel))
if (Atomic::CASWeak(
m_state, state, state + 2, std::memory_order_acq_rel))
{
QueueResetLocked();
return;
@@ -260,7 +260,7 @@ bool CInputPipeServer::SendMouseRelative(
deltaY > LG_INPUT_MOUSE_DELTA_MAX ||
wheel < LG_INPUT_MOUSE_WHEEL_MIN_TOTAL ||
wheel > LG_INPUT_MOUSE_WHEEL_MAX ||
!(m_state.load(std::memory_order_acquire) & 1))
!(Atomic::Load(m_state, std::memory_order_acquire) & 1))
return false;
KVMFRInputPayload payload = {};
@@ -272,7 +272,8 @@ bool CInputPipeServer::SendMouseRelative(
CSRWExclusiveLock lock(m_queueLock);
const bool pureMotion = wheel == 0 && buttons == m_relativeButtons;
const bool switching = m_mouseMode == MouseMode::ABSOLUTE_INPUT;
bool queued = (m_state.load(std::memory_order_relaxed) & 1) != 0;
bool queued =
(Atomic::Load(m_state, std::memory_order_relaxed) & 1) != 0;
if (queued && switching && m_absoluteButtons)
{
KVMFRInputPayload neutral = {};
@@ -307,7 +308,7 @@ bool CInputPipeServer::SendMouseAbsolute(
y > LG_INPUT_MOUSE_ABSOLUTE_MAX ||
wheel < LG_INPUT_MOUSE_WHEEL_MIN_TOTAL ||
wheel > LG_INPUT_MOUSE_WHEEL_MAX ||
!(m_state.load(std::memory_order_acquire) & 1))
!(Atomic::Load(m_state, std::memory_order_acquire) & 1))
return false;
KVMFRInputPayload payload = {};
@@ -319,7 +320,8 @@ bool CInputPipeServer::SendMouseAbsolute(
CSRWExclusiveLock lock(m_queueLock);
const bool pureMotion = wheel == 0 && buttons == m_absoluteButtons;
const bool switching = m_mouseMode == MouseMode::RELATIVE_INPUT;
bool queued = (m_state.load(std::memory_order_relaxed) & 1) != 0;
bool queued =
(Atomic::Load(m_state, std::memory_order_relaxed) & 1) != 0;
if (queued && switching && m_relativeButtons)
{
const KVMFRInputPayload neutral = {};
@@ -348,7 +350,7 @@ bool CInputPipeServer::SendKeyboard(
uint8_t modifiers,
const uint8_t * keys)
{
if (!keys || !(m_state.load(std::memory_order_acquire) & 1))
if (!keys || !(Atomic::Load(m_state, std::memory_order_acquire) & 1))
return false;
KVMFRInputPayload payload = {};
@@ -361,7 +363,8 @@ bool CInputPipeServer::SendKeyboard(
}
CSRWExclusiveLock lock(m_queueLock);
const bool queued = (m_state.load(std::memory_order_relaxed) & 1) &&
const bool queued =
(Atomic::Load(m_state, std::memory_order_relaxed) & 1) &&
QueueLocked(LG_INPUT_PIPE_MESSAGE_KEYBOARD, payload, false);
if (!queued)
ResyncLocked();
@@ -370,11 +373,12 @@ bool CInputPipeServer::SendKeyboard(
bool CInputPipeServer::Reset()
{
if (!(m_state.load(std::memory_order_acquire) & 1))
if (!(Atomic::Load(m_state, std::memory_order_acquire) & 1))
return false;
CSRWExclusiveLock lock(m_queueLock);
bool queued = (m_state.load(std::memory_order_relaxed) & 1) != 0;
bool queued =
(Atomic::Load(m_state, std::memory_order_relaxed) & 1) != 0;
if (queued)
queued = QueueResetLocked();
if (!queued)
@@ -412,7 +416,8 @@ bool CInputPipeServer::Send(const QueueItem& item)
LARGE_INTEGER end = {};
{
CSRWSharedLock lock(m_connectionLock);
const uint64_t state = m_state.load(std::memory_order_acquire);
const uint64_t state =
Atomic::Load(m_state, std::memory_order_acquire);
current = (state & 1) && item.state == state;
if (current)
{
@@ -527,13 +532,13 @@ void CInputPipeServer::LogStatistics()
void CInputPipeServer::Invalidate(uint64_t state, bool requireMatch)
{
CSRWExclusiveLock connectionLock(m_connectionLock);
uint64_t current = m_state.load(std::memory_order_relaxed);
uint64_t current = Atomic::Load(m_state, std::memory_order_relaxed);
for (;;)
{
if (!(current & 1) || (requireMatch && state != current))
return;
if (m_state.compare_exchange_weak(
current, current + 1, std::memory_order_acq_rel))
if (Atomic::CASWeak(
m_state, current, current + 1, std::memory_order_acq_rel))
break;
}
@@ -581,7 +586,7 @@ void CInputPipeServer::OnPipeConnected()
CSRWExclusiveLock connectionLock(m_connectionLock);
CSRWExclusiveLock queueLock(m_queueLock);
uint64_t state = m_state.load(std::memory_order_relaxed);
uint64_t state = Atomic::Load(m_state, std::memory_order_relaxed);
if (state & 1)
++state;
++state;
@@ -597,7 +602,7 @@ void CInputPipeServer::OnPipeConnected()
m_queue[index].state = state;
}
if (reset)
m_state.store(state, std::memory_order_release);
Atomic::Store(m_state, state, std::memory_order_release);
if (!reset)
DEBUG_WARN("Failed to queue LGInput endpoint neutralization");

4
idd/LGIdd/ipc/CInputPipeServer.h
View File

@@ -20,12 +20,12 @@
#pragma once
#include "Atomic.h"
#include "CPipeEndpoint.h"
#include "CSRWLock.h"
#include "InputPipeProtocol.h"
#include "input/IInputSink.h"
#include <atomic>
#include <stddef.h>
#include <stdint.h>
@@ -115,7 +115,7 @@ public:
uint64_t GetState() const override
{
return m_state.load(std::memory_order_acquire);
return Atomic::Load(m_state, std::memory_order_acquire);
}
bool SendMouseRelative(

165
idd/LGIdd/transport/CFrameHub.cpp
View File

@@ -11,6 +11,7 @@
#include "transport/CFrameHub.h"
#include "Atomic.h"
#include "CDebug.h"
static const uint64_t RETRY_NS = 1000000ULL;
@@ -46,8 +47,9 @@ CFrameHub::~CFrameHub()
for (Sink& sink : m_sinks)
{
const BackendId backend =
sink.backend.load(std::memory_order_acquire);
const uint32_t epoch = sink.epoch.load(std::memory_order_acquire);
Atomic::Load(sink.backend, std::memory_order_acquire);
const uint32_t epoch =
Atomic::Load(sink.epoch, std::memory_order_acquire);
if (backend && epoch)
Unbind(backend, epoch);
}
@@ -72,13 +74,15 @@ bool CFrameHub::Bind(BackendId backend, uint32_t epoch, bool primary,
CSRWExclusiveLock lock(m_listLock);
if (primary)
{
if (!m_sinks[0].active.load(std::memory_order_acquire) &&
if (!Atomic::Load(
m_sinks[0].active, std::memory_order_acquire) &&
!m_sinks[0].reserved)
selected = &m_sinks[0];
}
else
for (unsigned i = 1; i < FRAME_MAX_SINKS; ++i)
if (!m_sinks[i].active.load(std::memory_order_acquire) &&
if (!Atomic::Load(
m_sinks[i].active, std::memory_order_acquire) &&
!m_sinks[i].reserved)
{
selected = &m_sinks[i];
@@ -93,10 +97,10 @@ bool CFrameHub::Bind(BackendId backend, uint32_t epoch, bool primary,
{
CSRWExclusiveLock lock(selected->callLock);
selected->target = &target;
selected->backend.store(backend, std::memory_order_release);
selected->epoch.store(epoch, std::memory_order_release);
Atomic::Store(selected->backend, backend, std::memory_order_release);
Atomic::Store(selected->epoch, epoch, std::memory_order_release);
selected->primary = primary;
selected->outstanding.store(0, std::memory_order_release);
Atomic::Store(selected->outstanding, 0, std::memory_order_release);
SetEvent(selected->drained);
}
{
@@ -120,7 +124,7 @@ bool CFrameHub::Bind(BackendId backend, uint32_t epoch, bool primary,
target.SetFrameScheduleEvent(m_wakeEvent);
{
CSRWExclusiveLock lock(m_listLock);
selected->active.store(true, std::memory_order_release);
Atomic::Store(selected->active, true, std::memory_order_release);
selected->reserved = false;
}
target.ForceFrame();
@@ -134,11 +138,11 @@ void CFrameHub::Unbind(BackendId backend, uint32_t epoch)
{
CSRWExclusiveLock lock(m_listLock);
for (Sink& sink : m_sinks)
if (sink.active.load(std::memory_order_acquire) &&
sink.backend.load(std::memory_order_acquire) == backend &&
sink.epoch.load(std::memory_order_acquire) == epoch)
if (Atomic::Load(sink.active, std::memory_order_acquire) &&
Atomic::Load(sink.backend, std::memory_order_acquire) == backend &&
Atomic::Load(sink.epoch, std::memory_order_acquire) == epoch)
{
sink.active.store(false, std::memory_order_release);
Atomic::Store(sink.active, false, std::memory_order_release);
sink.reserved = true;
selected = &sink;
break;
@@ -184,8 +188,8 @@ void CFrameHub::Unbind(BackendId backend, uint32_t epoch)
{
CSRWExclusiveLock lock(selected->callLock);
selected->target = nullptr;
selected->backend.store(0, std::memory_order_release);
selected->epoch.store(0, std::memory_order_release);
Atomic::Store(selected->backend, 0, std::memory_order_release);
Atomic::Store(selected->epoch, 0, std::memory_order_release);
selected->primary = false;
}
{
@@ -211,7 +215,7 @@ unsigned CFrameHub::Snapshot(SinkRef refs[FRAME_MAX_SINKS]) const
unsigned count = 0;
CSRWSharedLock lock(m_listLock);
for (unsigned i = 0; i < FRAME_MAX_SINKS; ++i)
if (m_sinks[i].active.load(std::memory_order_acquire))
if (Atomic::Load(m_sinks[i].active, std::memory_order_acquire))
refs[count++] = {
const_cast<Sink *>(&m_sinks[i]), i, m_sinks[i].primary };
return count;
@@ -233,7 +237,7 @@ void CFrameHub::ReleaseTarget(Batch& batch, BatchTarget& target)
CSRWExclusiveLock lock(target.sink->laneLock);
target.sink->lanes[target.resourceLane].busy = false;
}
if (target.sink->outstanding.fetch_sub(
if (Atomic::FetchSub(target.sink->outstanding,
1, std::memory_order_acq_rel) == 1)
SetEvent(target.sink->drained);
@@ -341,7 +345,7 @@ void CFrameHub::FillLane(Sink& sink, unsigned laneIndex,
{
lane.phase = Sink::ResourceLane::PENDING;
cancel = lane.cancelRequested ||
!sink.active.load(std::memory_order_acquire);
!Atomic::Load(sink.active, std::memory_order_acquire);
}
}
@@ -468,7 +472,8 @@ void CFrameHub::CompleteLane(Sink& sink, unsigned laneIndex,
lane.busy = false;
}
}
if (sink.outstanding.fetch_sub(1, std::memory_order_acq_rel) == 1)
if (Atomic::FetchSub(sink.outstanding,
1, std::memory_order_acq_rel) == 1)
SetEvent(sink.drained);
SetEvent(m_wakeEvent);
}
@@ -481,8 +486,9 @@ void CFrameHub::OnFrameDone(const FrameToken& token, FrameDone result,
for (Sink& sink : m_sinks)
{
if (sink.backend.load(std::memory_order_acquire) != token.backend ||
sink.epoch.load(std::memory_order_acquire) != token.epoch)
if (Atomic::Load(sink.backend, std::memory_order_acquire) !=
token.backend ||
Atomic::Load(sink.epoch, std::memory_order_acquire) != token.epoch)
continue;
unsigned laneIndex = FRAME_SINK_BUFFERS;
@@ -525,7 +531,8 @@ size_t CFrameHub::GetMaxFrameSize() const
if (refs[i].primary)
{
CSRWSharedLock call(refs[i].sink->callLock);
if (refs[i].sink->active.load(std::memory_order_acquire) &&
if (Atomic::Load(
refs[i].sink->active, std::memory_order_acquire) &&
refs[i].sink->target)
return refs[i].sink->target->GetMaxFrameSize();
}
@@ -534,21 +541,16 @@ size_t CFrameHub::GetMaxFrameSize() const
uint64_t CFrameHub::NextContentSerial()
{
uint64_t serial = m_nextContent.fetch_add(
1, std::memory_order_acq_rel) + 1;
if (!serial)
serial = m_nextContent.fetch_add(
1, std::memory_order_acq_rel) + 1;
return serial;
return Atomic::Next(m_nextContent, std::memory_order_acq_rel);
}
void CFrameHub::FrameProductReady(uint64_t contentSerial)
{
uint64_t newest = m_newestContent.load(std::memory_order_acquire);
uint64_t newest =
Atomic::Load(m_newestContent, std::memory_order_acquire);
while (ContentAfter(contentSerial, newest) &&
!m_newestContent.compare_exchange_weak(newest,
contentSerial, std::memory_order_acq_rel,
std::memory_order_acquire))
!Atomic::CASWeak(m_newestContent, newest, contentSerial,
std::memory_order_acq_rel, std::memory_order_acquire))
{
}
SetEvent(m_wakeEvent);
@@ -557,13 +559,14 @@ void CFrameHub::FrameProductReady(uint64_t contentSerial)
bool CFrameHub::NeedsFrame() const
{
const uint64_t newest =
m_newestContent.load(std::memory_order_acquire);
Atomic::Load(m_newestContent, std::memory_order_acquire);
SinkRef refs[FRAME_MAX_SINKS];
const unsigned count = Snapshot(refs);
for (unsigned i = 0; i < count; ++i)
{
CSRWSharedLock call(refs[i].sink->callLock);
if (!refs[i].sink->active.load(std::memory_order_acquire) ||
if (!Atomic::Load(
refs[i].sink->active, std::memory_order_acquire) ||
!refs[i].sink->target)
continue;
const size_t maxFrameSize = refs[i].sink->target->GetMaxFrameSize();
@@ -590,7 +593,8 @@ bool CFrameHub::GetFramePlan(
{
Sink& sink = *refs[i].sink;
CSRWSharedLock call(sink.callLock);
if (!sink.active.load(std::memory_order_acquire) || !sink.target)
if (!Atomic::Load(sink.active, std::memory_order_acquire) ||
!sink.target)
continue;
sink.target->ProcessDeliveries();
@@ -641,8 +645,10 @@ bool CFrameHub::GetFramePlan(
continue;
FramePlanTarget& request = plan.targets[plan.count++];
request.sink = refs[i].index;
request.backend = sink.backend.load(std::memory_order_acquire);
request.epoch = sink.epoch.load(std::memory_order_acquire);
request.backend = Atomic::Load(
sink.backend, std::memory_order_acquire);
request.epoch = Atomic::Load(
sink.epoch, std::memory_order_acquire);
request.schedule = schedule;
request.commitSchedule = schedule;
request.periodic = periodic;
@@ -660,7 +666,8 @@ bool CFrameHub::GetImmediateFramePlan(uint64_t now, FramePlan& plan)
{
Sink& sink = *refs[i].sink;
CSRWSharedLock call(sink.callLock);
if (!sink.active.load(std::memory_order_acquire) || !sink.target)
if (!Atomic::Load(sink.active, std::memory_order_acquire) ||
!sink.target)
continue;
sink.target->ProcessDeliveries();
@@ -684,8 +691,10 @@ bool CFrameHub::GetImmediateFramePlan(uint64_t now, FramePlan& plan)
continue;
FramePlanTarget& request = plan.targets[plan.count++];
request.sink = refs[i].index;
request.backend = sink.backend.load(std::memory_order_acquire);
request.epoch = sink.epoch.load(std::memory_order_acquire);
request.backend = Atomic::Load(
sink.backend, std::memory_order_acquire);
request.epoch = Atomic::Load(
sink.epoch, std::memory_order_acquire);
request.schedule = schedule;
request.schedule.deliveryDeadlineSerial = 0;
request.schedule.phaseEligible = false;
@@ -708,9 +717,12 @@ void CFrameHub::MissFramePlan(const FramePlan& plan, uint64_t now)
continue;
Sink& sink = m_sinks[request.sink];
CSRWSharedLock call(sink.callLock);
if (sink.active.load(std::memory_order_acquire) && sink.target &&
sink.backend.load(std::memory_order_acquire) == request.backend &&
sink.epoch.load(std::memory_order_acquire) == request.epoch)
if (Atomic::Load(sink.active, std::memory_order_acquire) &&
sink.target &&
Atomic::Load(sink.backend, std::memory_order_acquire) ==
request.backend &&
Atomic::Load(sink.epoch, std::memory_order_acquire) ==
request.epoch)
sink.target->FrameMissed(
request.commitSchedule, now, request.periodic);
}
@@ -736,11 +748,8 @@ bool CFrameHub::PrepareFrameBatch(const FramePlan& plan,
continue;
candidate.active = true;
candidate.count = 0;
candidate.serial = m_nextSerial.fetch_add(
1, std::memory_order_acq_rel) + 1;
if (!candidate.serial)
candidate.serial = m_nextSerial.fetch_add(
1, std::memory_order_acq_rel) + 1;
candidate.serial = Atomic::Next(
m_nextSerial, std::memory_order_acq_rel);
for (BatchTarget& target : candidate.targets)
target = {};
prepared.token = {
@@ -761,9 +770,12 @@ bool CFrameHub::PrepareFrameBatch(const FramePlan& plan,
continue;
Sink& sink = m_sinks[request.sink];
CSRWExclusiveLock call(sink.callLock);
if (!sink.active.load(std::memory_order_acquire) || !sink.target ||
sink.backend.load(std::memory_order_acquire) != request.backend ||
sink.epoch.load(std::memory_order_acquire) != request.epoch ||
if (!Atomic::Load(sink.active, std::memory_order_acquire) ||
!sink.target ||
Atomic::Load(sink.backend, std::memory_order_acquire) !=
request.backend ||
Atomic::Load(sink.epoch, std::memory_order_acquire) !=
request.epoch ||
!sink.target->FrameBufferAvailable(
request.schedule, allowReadyReplacement))
continue;
@@ -882,7 +894,7 @@ bool CFrameHub::PrepareFrameBatch(const FramePlan& plan,
target.frameType = dstFormat.format;
target.periodic = request.periodic;
target.active = true;
if (sink.outstanding.fetch_add(
if (Atomic::FetchAdd(sink.outstanding,
1, std::memory_order_acq_rel) == 0)
ResetEvent(sink.drained);
@@ -928,9 +940,10 @@ uint32_t CFrameHub::PublishFrameBatch(const FrameBatchToken& token)
CSRWExclusiveLock call(target.sink->callLock);
bool delivered = false;
const bool valid = target.sink->target &&
target.sink->backend.load(std::memory_order_acquire) ==
Atomic::Load(target.sink->backend, std::memory_order_acquire) ==
target.backend &&
target.sink->epoch.load(std::memory_order_acquire) == target.epoch;
Atomic::Load(target.sink->epoch, std::memory_order_acquire) ==
target.epoch;
if (!valid || !target.sink->target->PublishFrameBuffer(
target.localSlot, target.deliverySchedule, delivered))
{
@@ -980,10 +993,10 @@ void CFrameHub::CommitFrameBatch(const FrameBatchToken& token)
{
CSRWExclusiveLock call(target.sink->callLock);
if (target.sink->target &&
target.sink->backend.load(std::memory_order_acquire) ==
target.backend &&
target.sink->epoch.load(std::memory_order_acquire) ==
target.epoch)
Atomic::Load(target.sink->backend,
std::memory_order_acquire) == target.backend &&
Atomic::Load(target.sink->epoch,
std::memory_order_acquire) == target.epoch)
target.sink->target->CommitFrameBuffer(target.localSlot,
target.schedule, target.periodic, target.delivered);
}
@@ -1022,9 +1035,10 @@ void CFrameHub::AbortFrameBatch(const FrameBatchToken& token)
continue;
CSRWExclusiveLock call(target.sink->callLock);
if (target.sink->target &&
target.sink->backend.load(std::memory_order_acquire) ==
target.backend &&
target.sink->epoch.load(std::memory_order_acquire) == target.epoch)
Atomic::Load(target.sink->backend,
std::memory_order_acquire) == target.backend &&
Atomic::Load(target.sink->epoch,
std::memory_order_acquire) == target.epoch)
target.sink->target->AbortFrameBuffer(target.localSlot);
{
CSRWExclusiveLock state(target.sink->laneLock);
@@ -1049,9 +1063,10 @@ void CFrameHub::FailFrameBatch(const FrameBatchToken& token)
continue;
CSRWExclusiveLock call(target.sink->callLock);
if (target.sink->target &&
target.sink->backend.load(std::memory_order_acquire) ==
target.backend &&
target.sink->epoch.load(std::memory_order_acquire) == target.epoch)
Atomic::Load(target.sink->backend,
std::memory_order_acquire) == target.backend &&
Atomic::Load(target.sink->epoch,
std::memory_order_acquire) == target.epoch)
{
if (target.published)
target.sink->target->FailFrameBuffer(target.localSlot);
@@ -1080,9 +1095,10 @@ void CFrameHub::WriteFrameTarget(const FrameBatchToken& token,
BatchTarget& target = batch.targets[index];
CSRWSharedLock call(target.sink->callLock);
if (target.sink->target &&
target.sink->backend.load(std::memory_order_acquire) ==
Atomic::Load(target.sink->backend, std::memory_order_acquire) ==
target.backend &&
target.sink->epoch.load(std::memory_order_acquire) == target.epoch)
Atomic::Load(target.sink->epoch, std::memory_order_acquire) ==
target.epoch)
target.sink->target->WriteFrameBuffer(
target.localSlot, src, offset, len, setWritePos);
}
@@ -1101,9 +1117,10 @@ void CFrameHub::WriteFrameTargetRows(const FrameBatchToken& token,
BatchTarget& target = batch.targets[index];
CSRWSharedLock call(target.sink->callLock);
if (target.sink->target &&
target.sink->backend.load(std::memory_order_acquire) ==
Atomic::Load(target.sink->backend, std::memory_order_acquire) ==
target.backend &&
target.sink->epoch.load(std::memory_order_acquire) == target.epoch)
Atomic::Load(target.sink->epoch, std::memory_order_acquire) ==
target.epoch)
target.sink->target->WriteFrameBufferRows(target.localSlot, src,
offset, rowBytes, pitch, rows);
}
@@ -1121,9 +1138,10 @@ void CFrameHub::FinalizeFrameTarget(
BatchTarget& target = batch.targets[index];
CSRWSharedLock call(target.sink->callLock);
if (target.sink->target &&
target.sink->backend.load(std::memory_order_acquire) ==
Atomic::Load(target.sink->backend, std::memory_order_acquire) ==
target.backend &&
target.sink->epoch.load(std::memory_order_acquire) == target.epoch)
Atomic::Load(target.sink->epoch, std::memory_order_acquire) ==
target.epoch)
target.sink->target->FinalizeFrameBuffer(target.localSlot);
}
@@ -1201,7 +1219,8 @@ void CFrameHub::ObserveFrame(uint64_t now)
for (unsigned i = 0; i < count; ++i)
{
CSRWSharedLock call(refs[i].sink->callLock);
if (refs[i].sink->active.load(std::memory_order_acquire) &&
if (Atomic::Load(
refs[i].sink->active, std::memory_order_acquire) &&
refs[i].sink->target)
refs[i].sink->target->ObserveFrame(now);
}
@@ -1214,7 +1233,8 @@ void CFrameHub::ForceFrame()
for (unsigned i = 0; i < count; ++i)
{
CSRWSharedLock call(refs[i].sink->callLock);
if (refs[i].sink->active.load(std::memory_order_acquire) &&
if (Atomic::Load(
refs[i].sink->active, std::memory_order_acquire) &&
refs[i].sink->target)
refs[i].sink->target->ForceFrame();
}
@@ -1227,7 +1247,8 @@ void CFrameHub::FrameSuperseded()
for (unsigned i = 0; i < count; ++i)
{
CSRWSharedLock call(refs[i].sink->callLock);
if (refs[i].sink->active.load(std::memory_order_acquire) &&
if (Atomic::Load(
refs[i].sink->active, std::memory_order_acquire) &&
refs[i].sink->target)
refs[i].sink->target->FrameSuperseded();
}

3
idd/LGIdd/transport/CFrameHub.h
View File

@@ -20,13 +20,12 @@
#pragma once
#include "Atomic.h"
#include "CSRWLock.h"
#include "transport/IFrameSink.h"
#include "transport/IFrameTransport.h"
#include "transport/ITransport.h"
#include <atomic>
class CFrameHub final : public IFrameTransport, public IFrameEvents
{
private:

73
idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp
View File

@@ -23,6 +23,7 @@
#include "transport/lgmp/CIVSHMEM.h"
#include "transport/lgmp/CLGMPFrameCaps.h"
#include "transport/lgmp/CLGMPHost.h"
#include "Atomic.h"
#include "CDebug.h"
#include <cstring>
@@ -196,13 +197,13 @@ bool CLGMPFrameTransport::Setup(size_t alignSize)
m_frame[i]->offset = (uint32_t)alignOffset;
m_frameBuffer[i] = reinterpret_cast<LGMPBuffer *>(
reinterpret_cast<uint8_t *>(m_frame[i]) + alignOffset);
m_frameInFlight[i].store(false, std::memory_order_release);
Atomic::Store(m_frameInFlight[i], false, std::memory_order_release);
m_frameCompleted[i] = false;
}
m_maxFrameSize = maxFrameSize;
m_submittedFrameIndex.store(-1, std::memory_order_release);
m_readyFrameIndex.store(-1, std::memory_order_release);
Atomic::Store(m_submittedFrameIndex, -1, std::memory_order_release);
Atomic::Store(m_readyFrameIndex, -1, std::memory_order_release);
m_deferredOwnerFrameIndex = -1;
m_framePublishSequence = 0;
m_frameReadySequence = 0;
@@ -222,8 +223,8 @@ void CLGMPFrameTransport::DeInit()
{
CSRWExclusiveLock lock(m_framePublishLock);
m_submittedFrameIndex.store(-1, std::memory_order_release);
m_readyFrameIndex.store(-1, std::memory_order_release);
Atomic::Store(m_submittedFrameIndex, -1, std::memory_order_release);
Atomic::Store(m_readyFrameIndex, -1, std::memory_order_release);
m_deferredOwnerFrameIndex = -1;
m_framePublishSequence = 0;
m_frameReadySequence = 0;
@@ -238,7 +239,7 @@ void CLGMPFrameTransport::DeInit()
for (int i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i)
{
m_frameInFlight[i].store(false, std::memory_order_release);
Atomic::Store(m_frameInFlight[i], false, std::memory_order_release);
lgmpHostMemFree(&m_frameMemory[i]);
m_frame[i] = nullptr;
m_frameBuffer[i] = nullptr;
@@ -571,14 +572,15 @@ int CLGMPFrameTransport::FindAvailableFrameBuffer(
bool allowReady) const
{
const LONG readyFrameIndex =
m_readyFrameIndex.load(std::memory_order_acquire);
Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
int available = -1;
uint64_t newestPublish = 0;
for (unsigned frameIndex = 0;
frameIndex < LGMP_Q_FRAME_BUFFER_LEN; ++frameIndex)
{
if (static_cast<LONG>(frameIndex) == readyFrameIndex ||
m_frameInFlight[frameIndex].load(std::memory_order_acquire) ||
Atomic::Load(
m_frameInFlight[frameIndex], std::memory_order_acquire) ||
FrameBufferReferenced(frameIndex))
continue;
@@ -591,7 +593,8 @@ int CLGMPFrameTransport::FindAvailableFrameBuffer(
}
if (available >= 0 || !allowReady || readyFrameIndex < 0 ||
m_frameInFlight[readyFrameIndex].load(std::memory_order_acquire) ||
Atomic::Load(
m_frameInFlight[readyFrameIndex], std::memory_order_acquire) ||
FrameBufferReferenced(static_cast<unsigned>(readyFrameIndex)))
return available;
@@ -610,7 +613,8 @@ int CLGMPFrameTransport::FindNewestCompletedFrame(
frameIndex < LGMP_Q_FRAME_BUFFER_LEN; ++frameIndex)
{
if (frameIndex == excludeFrameIndex || !m_frameCompleted[frameIndex] ||
m_frameInFlight[frameIndex].load(std::memory_order_acquire))
Atomic::Load(
m_frameInFlight[frameIndex], std::memory_order_acquire))
continue;
if (newestFrame < 0 ||
@@ -679,9 +683,10 @@ bool CLGMPFrameTransport::GetPendingDeliveryTarget(uint64_t now,
CSRWSharedLock lock(m_framePublishLock);
const LONG frameIndex =
m_readyFrameIndex.load(std::memory_order_acquire);
Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
if (frameIndex < 0 ||
m_frameInFlight[frameIndex].load(std::memory_order_acquire) ||
Atomic::Load(
m_frameInFlight[frameIndex], std::memory_order_acquire) ||
lgmpHostQueuePending(m_frameQueue) != 0)
return false;
@@ -705,9 +710,10 @@ bool CLGMPFrameTransport::RetryPendingDelivery(uint64_t now, bool& retry)
CSRWExclusiveLock lock(m_framePublishLock);
const LONG frameIndex =
m_readyFrameIndex.load(std::memory_order_acquire);
Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
if (frameIndex < 0 ||
m_frameInFlight[frameIndex].load(std::memory_order_acquire) ||
Atomic::Load(
m_frameInFlight[frameIndex], std::memory_order_acquire) ||
lgmpHostQueuePending(m_frameQueue) != 0)
return false;
@@ -747,14 +753,14 @@ SinkTarget CLGMPFrameTransport::PrepareFrameBuffer(
FindAvailableFrameBuffer(allowReady);
bool expected = false;
const bool acquired = availableFrameIndex >= 0 &&
m_frameInFlight[availableFrameIndex].compare_exchange_strong(
expected, true, std::memory_order_acq_rel);
Atomic::CAS(m_frameInFlight[availableFrameIndex], expected, true,
std::memory_order_acq_rel);
if (acquired)
{
const LONG readyFrameIndex =
m_readyFrameIndex.load(std::memory_order_acquire);
Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
if (availableFrameIndex == readyFrameIndex)
m_readyFrameIndex.store(
Atomic::Store(m_readyFrameIndex,
FindNewestCompletedFrame(
static_cast<unsigned>(availableFrameIndex)),
std::memory_order_release);
@@ -872,7 +878,7 @@ SinkTarget CLGMPFrameTransport::PrepareFrameBuffer(
fi->scheduleGeneration = 0;
fi->scheduleEpoch = 0;
fi->scheduleDeadlineSerial = 0;
InterlockedExchange((volatile LONG *)&fi->timingValid, 0);
Atomic::Store(fi->timingValid, 0);
fi->rotation = FRAME_ROT_0;
fi->type = dstFormat.format;
@@ -1015,7 +1021,7 @@ bool CLGMPFrameTransport::PublishFrameBuffer(unsigned frameIndex,
m_frameDelivered[frameIndex] = deliveredToOwner;
m_deferredOwnerFrameIndex = schedule.clientID && !deliveredToOwner ?
static_cast<LONG>(frameIndex) : -1;
m_submittedFrameIndex.store(
Atomic::Store(m_submittedFrameIndex,
static_cast<LONG>(frameIndex), std::memory_order_release);
}
lock.Unlock();
@@ -1046,15 +1052,18 @@ bool CLGMPFrameTransport::RepublishFrameBuffer(
LONG frameIndex = m_deferredOwnerFrameIndex;
if (frameIndex >= 0 &&
!m_frameCompleted[frameIndex] &&
!m_frameInFlight[frameIndex].load(std::memory_order_acquire))
!Atomic::Load(
m_frameInFlight[frameIndex], std::memory_order_acquire))
{
m_deferredOwnerFrameIndex = -1;
frameIndex = -1;
}
if (frameIndex < 0)
frameIndex = m_readyFrameIndex.load(std::memory_order_acquire);
frameIndex = Atomic::Load(
m_readyFrameIndex, std::memory_order_acquire);
if (frameIndex < 0 ||
m_frameInFlight[frameIndex].load(std::memory_order_acquire))
Atomic::Load(
m_frameInFlight[frameIndex], std::memory_order_acquire))
return false;
CFrameScheduler::Schedule deliverySchedule = schedule;
@@ -1188,12 +1197,12 @@ void CLGMPFrameTransport::AbortFrameBuffer(unsigned frameIndex)
CSRWExclusiveLock lock(m_framePublishLock);
m_frameBuffer[frameIndex]->wp = 0;
InterlockedExchange(
(volatile LONG *)&m_frame[frameIndex]->timingValid, 0);
Atomic::Store(m_frame[frameIndex]->timingValid, 0);
m_frameCompleted[frameIndex] = false;
if (m_deferredOwnerFrameIndex == static_cast<LONG>(frameIndex))
m_deferredOwnerFrameIndex = -1;
m_frameInFlight[frameIndex].store(false, std::memory_order_release);
Atomic::Store(
m_frameInFlight[frameIndex], false, std::memory_order_release);
}
void CLGMPFrameTransport::FailFrameBuffer(unsigned frameIndex)
@@ -1201,8 +1210,7 @@ void CLGMPFrameTransport::FailFrameBuffer(unsigned frameIndex)
if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN)
return;
InterlockedExchange(
(volatile LONG *)&m_frame[frameIndex]->timingValid, 0);
Atomic::Store(m_frame[frameIndex]->timingValid, 0);
FinalizeFrameBuffer(frameIndex);
AbortFrameBuffer(frameIndex);
}
@@ -1228,14 +1236,15 @@ void CLGMPFrameTransport::CompleteFrameBuffer(
// Completion callbacks may run out of order. Never replace a newer ready
// frame with an older submission.
const LONG readyFrameIndex =
m_readyFrameIndex.load(std::memory_order_acquire);
Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
if (sequence &&
(readyFrameIndex < 0 ||
sequence > m_frameLastPublishSequence[readyFrameIndex]))
m_readyFrameIndex.store(
Atomic::Store(m_readyFrameIndex,
static_cast<LONG>(frameIndex), std::memory_order_release);
}
m_frameInFlight[frameIndex].store(false, std::memory_order_release);
Atomic::Store(
m_frameInFlight[frameIndex], false, std::memory_order_release);
const bool newerThanReady =
sequence && sequence > m_frameReadySequence;
if (result == FrameDone::READY && newerThanReady)
@@ -1276,7 +1285,7 @@ void CLGMPFrameTransport::SetFrameTiming(unsigned frameIndex,
frame->timingFlags = phaseValid ?
KVMFR_FRAME_TIMING_PHASE_VALID : 0;
frame->timingSerial = frame->frameSerial;
InterlockedExchange((volatile LONG *)&frame->timingValid, 1);
Atomic::Store(frame->timingValid, 1);
}
void CLGMPFrameTransport::WriteFrameBuffer(unsigned frameIndex, void * src,

4
idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h
View File

@@ -20,10 +20,10 @@
#pragma once
#include "Atomic.h"
#include "CSRWLock.h"
#include <Windows.h>
#include <atomic>
#include <stdint.h>
extern "C" {
@@ -173,7 +173,7 @@ public:
bool allowReadyReplacement = true) override;
bool HasPublishedFrame() const override
{
return m_readyFrameIndex.load(std::memory_order_acquire) >= 0;
return Atomic::Load(m_readyFrameIndex, std::memory_order_acquire) >= 0;
}
void ProcessDeliveries() override;
bool GetPendingDeliveryTarget(

12
idd/LGIdd/transport/lgmp/CLGMPInputTransport.cpp
View File

@@ -21,6 +21,7 @@
#include "transport/lgmp/CLGMPInputTransport.h"
#include "transport/lgmp/CLGMPHost.h"
#include "Atomic.h"
#include "CDebug.h"
#include "CSRWLock.h"
#include "Seq.h"
@@ -201,10 +202,11 @@ bool CLGMPInputTransport::PublishStatus()
void CLGMPInputTransport::FlushStatus()
{
if (m_statusFailed.load(std::memory_order_acquire) || PublishStatus())
if (Atomic::Load(m_statusFailed, std::memory_order_acquire) ||
PublishStatus())
return;
m_statusFailed.store(true, std::memory_order_release);
Atomic::Store(m_statusFailed, true, std::memory_order_release);
CSRWSharedLock lock(m_lifecycleLock);
if (m_stopEvent)
SetEvent(m_stopEvent);
@@ -264,7 +266,7 @@ bool CLGMPInputTransport::Start(IInputTarget& target)
Seq::Inc(m_endpointGeneration);
m_statusDirty = true;
}
m_statusFailed.store(false, std::memory_order_release);
Atomic::Store(m_statusFailed, false, std::memory_order_release);
m_thread = CreateThread(nullptr, 0, ThreadProc, this, 0, nullptr);
if (!m_thread)
{
@@ -693,7 +695,7 @@ void CLGMPInputTransport::Thread()
}
if (!PublishStatus())
{
m_statusFailed.store(true, std::memory_order_release);
Atomic::Store(m_statusFailed, true, std::memory_order_release);
failed = true;
break;
}
@@ -716,7 +718,7 @@ void CLGMPInputTransport::Thread()
_countof(waitHandles), waitHandles, FALSE, INFINITE);
if (wait == WAIT_FIRST_OBJECT_VALUE)
{
failed = m_statusFailed.load(std::memory_order_acquire);
failed = Atomic::Load(m_statusFailed, std::memory_order_acquire);
break;
}
if (wait != WAIT_FIRST_OBJECT_VALUE + 1)

2
idd/LGIdd/transport/lgmp/CLGMPInputTransport.h
View File

@@ -20,13 +20,13 @@
#pragma once
#include "Atomic.h"
#include "CSRWLock.h"
#include "transport/IInputSource.h"
#include "common/LGMPConfig.h"
#include <Windows.h>
#include <atomic>
#include <stdint.h>
extern "C" {

7
idd/LGIdd/transport/lgmp/CLGMPTransport.cpp
View File

@@ -20,6 +20,7 @@
#include "transport/lgmp/CLGMPTransport.h"
#include "Atomic.h"
#include "CDebug.h"
#include "common/KVMFR.h"
#include "common/KVMFRRecovery.h"
@@ -90,7 +91,7 @@ bool CLGMPTransport::Setup(size_t alignment)
if (!m_frames.Setup(alignment))
return false;
m_ready.store(true, std::memory_order_release);
Atomic::Store(m_ready, true, std::memory_order_release);
return true;
}
@@ -109,7 +110,7 @@ ITransport::ProcessResult CLGMPTransport::Process(ITransportEvents& events)
// Before the swap chain establishes the frame-buffer alignment, service
// only the protocol-independent recovery channel. This preserves the old
// transport startup boundary while keeping recovery available immediately.
if (!m_ready.load(std::memory_order_acquire))
if (!Atomic::Load(m_ready, std::memory_order_acquire))
return ProcessResult::OK;
const LGMP_STATUS processStatus = m_host.Process();
@@ -224,7 +225,7 @@ ITransport::ProcessResult CLGMPTransport::Process(ITransportEvents& events)
void CLGMPTransport::Stop()
{
m_ready.store(false, std::memory_order_release);
Atomic::Store(m_ready, false, std::memory_order_release);
m_input.Stop();
}

3
idd/LGIdd/transport/lgmp/CLGMPTransport.h
View File

@@ -20,6 +20,7 @@
#pragma once
#include "Atomic.h"
#include "transport/ITransport.h"
#include "transport/lgmp/CIVSHMEM.h"
#include "transport/lgmp/CLGMPControl.h"
@@ -28,8 +29,6 @@
#include "transport/lgmp/CLGMPInputTransport.h"
#include "transport/lgmp/CRecovery.h"
#include <atomic>
class CLGMPTransport final : public ITransport
{
private:

72
idd/LGIdd/transport/lgmp/CRecovery.cpp
View File

@@ -22,6 +22,7 @@
#include "transport/lgmp/CIVSHMEM.h"
#include "platform/CPlatformInfo.h"
#include "Atomic.h"
#include "CDebug.h"
#include "VersionInfo.h"
@@ -38,36 +39,6 @@ namespace
static const uint64_t HELPER_TIMEOUT_MS = 30000;
CSRWLock l_wireLock;
uint32_t AtomicRead(uint32_t& value)
{
return static_cast<uint32_t>(InterlockedCompareExchange(
(volatile LONG *)&value, 0, 0));
}
void AtomicWrite(uint32_t& value, uint32_t data)
{
InterlockedExchange((volatile LONG *)&value, static_cast<LONG>(data));
}
void AtomicIncrement(uint32_t& value)
{
InterlockedIncrement((volatile LONG *)&value);
}
uint32_t AtomicAdd(uint32_t& value, uint32_t data)
{
return static_cast<uint32_t>(InterlockedExchangeAdd(
(volatile LONG *)&value, static_cast<LONG>(data))) + data;
}
bool AtomicCompareExchange(
uint32_t& value, uint32_t expected, uint32_t data)
{
return static_cast<uint32_t>(InterlockedCompareExchange(
(volatile LONG *)&value, static_cast<LONG>(data),
static_cast<LONG>(expected))) == expected;
}
uint64_t CreateSession(const void * memory, uint64_t previous)
{
LARGE_INTEGER counter;
@@ -89,13 +60,13 @@ namespace
bool CRecovery::OwnsSession()
{
if (AtomicRead(m_data->header.ready) != KVMFR_R_READY)
if (Atomic::Load(m_data->header.ready) != KVMFR_R_READY)
return false;
const uint64_t session = m_data->header.session;
MemoryBarrier();
Atomic::Fence();
return session == m_session &&
AtomicRead(m_data->header.ready) == KVMFR_R_READY;
Atomic::Load(m_data->header.ready) == KVMFR_R_READY;
}
bool CRecovery::ReadRequest(
@@ -103,14 +74,14 @@ bool CRecovery::ReadRequest(
{
for (unsigned i = 0; i < 4; ++i)
{
const uint32_t serial = AtomicRead(source.serial);
const uint32_t serial = Atomic::Load(source.serial);
if (!serial || (serial & 1U))
return false;
const uint32_t type = source.request;
const uint64_t session = source.session;
MemoryBarrier();
if (AtomicRead(source.serial) == serial)
Atomic::Fence();
if (Atomic::Load(source.serial) == serial)
{
result.serial = serial;
result.request = type;
@@ -126,7 +97,7 @@ bool CRecovery::ReadStatus(KVMFRRStatus& source, KVMFRRStatus& result)
{
for (unsigned i = 0; i < 4; ++i)
{
const uint32_t serial = AtomicRead(source.serial);
const uint32_t serial = Atomic::Load(source.serial);
if (!serial || (serial & 1U))
continue;
@@ -135,8 +106,8 @@ bool CRecovery::ReadStatus(KVMFRRStatus& source, KVMFRRStatus& result)
result.state = source.state;
result.error = source.error;
result.session = source.session;
MemoryBarrier();
if (AtomicRead(source.serial) == serial)
Atomic::Fence();
if (Atomic::Load(source.serial) == serial)
{
result.serial = serial;
return true;
@@ -154,10 +125,7 @@ bool CRecovery::SerialNewer(uint32_t serial, uint32_t reference)
uint32_t CRecovery::NextTicket()
{
uint32_t ticket = AtomicAdd(m_data->req.ticket, 2U);
if (!ticket)
ticket = AtomicAdd(m_data->req.ticket, 2U);
return ticket;
return Atomic::Next(m_data->req.ticket, 2U);
}
void CRecovery::Publish(uint32_t serial, uint32_t request,
@@ -168,13 +136,13 @@ void CRecovery::Publish(uint32_t serial, uint32_t request,
if (!published)
published = KVMFR_R_REQ_FIRST;
AtomicWrite(m_data->status.serial, writing);
Atomic::Store(m_data->status.serial, writing);
m_data->status.ackRequest = request;
m_data->status.state = state;
m_data->status.error = error;
m_data->status.session = m_session;
m_data->status.ackSerial = serial;
AtomicWrite(m_data->status.serial, published);
Atomic::Store(m_data->status.serial, published);
m_statusSerial = published;
}
@@ -192,7 +160,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem)
return false;
}
const uint32_t oldReady = AtomicRead(m_data->header.ready);
const uint32_t oldReady = Atomic::Load(m_data->header.ready);
const bool oldValid = oldReady == KVMFR_R_READY &&
memcmp(m_data->header.magic, KVMFR_R_MAGIC,
sizeof(m_data->header.magic)) == 0 &&
@@ -220,7 +188,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem)
}
}
AtomicWrite(m_data->header.ready, 0);
Atomic::Store(m_data->header.ready, 0);
if (oldValid)
{
ZeroMemory(&m_data->header, sizeof(m_data->header));
@@ -233,8 +201,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem)
{
KVMFRRRequest request = {};
if (ReadRequest(m_data->requests[i], request))
AtomicCompareExchange(
m_data->requests[i].serial, request.serial, 0);
Atomic::CAS(m_data->requests[i].serial, request.serial, 0);
}
}
else
@@ -264,7 +231,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem)
KVMFR_R_STATE_SWITCHING, KVMFR_R_ERR_NONE);
m_nextHeartbeat = GetTickCount64() + KVMFR_R_HEARTBEAT_MS;
AtomicWrite(m_data->header.ready, KVMFR_R_READY);
Atomic::Store(m_data->header.ready, KVMFR_R_READY);
DEBUG_INFO("Recovery channel initialized (session %llu%s)",
(unsigned long long)m_session,
@@ -289,7 +256,7 @@ CRecovery::Request CRecovery::Process()
const uint64_t now = GetTickCount64();
if (now >= m_nextHeartbeat)
{
AtomicIncrement(m_data->header.heartbeat);
Atomic::Inc(m_data->header.heartbeat);
m_nextHeartbeat = now + KVMFR_R_HEARTBEAT_MS;
}
@@ -371,8 +338,7 @@ CRecovery::Request CRecovery::Process()
// slot without a corresponding acknowledgement.
for (unsigned i = 0; i < KVMFR_R_REQ_SLOTS; ++i)
if (stable[i])
AtomicCompareExchange(
m_data->requests[i].serial, requests[i].serial, 0);
Atomic::CAS(m_data->requests[i].serial, requests[i].serial, 0);
if (m_waiting && now >= m_deadline)
{