[idd] common: centralize atomic operations

idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp

@@ -23,6 +23,7 @@
 #include "transport/lgmp/CIVSHMEM.h"
 #include "transport/lgmp/CLGMPFrameCaps.h"
 #include "transport/lgmp/CLGMPHost.h"
+#include "Atomic.h"
 #include "CDebug.h"
 
 #include <cstring>
@@ -196,13 +197,13 @@ bool CLGMPFrameTransport::Setup(size_t alignSize)
     m_frame[i]->offset = (uint32_t)alignOffset;
     m_frameBuffer[i] = reinterpret_cast<LGMPBuffer *>(
       reinterpret_cast<uint8_t *>(m_frame[i]) + alignOffset);
-    m_frameInFlight[i].store(false, std::memory_order_release);
+    Atomic::Store(m_frameInFlight[i], false, std::memory_order_release);
     m_frameCompleted[i] = false;
   }
 
   m_maxFrameSize = maxFrameSize;
-  m_submittedFrameIndex.store(-1, std::memory_order_release);
-  m_readyFrameIndex.store(-1, std::memory_order_release);
+  Atomic::Store(m_submittedFrameIndex, -1, std::memory_order_release);
+  Atomic::Store(m_readyFrameIndex, -1, std::memory_order_release);
   m_deferredOwnerFrameIndex = -1;
   m_framePublishSequence = 0;
   m_frameReadySequence   = 0;
@@ -222,8 +223,8 @@ void CLGMPFrameTransport::DeInit()
 
   {
     CSRWExclusiveLock lock(m_framePublishLock);
-    m_submittedFrameIndex.store(-1, std::memory_order_release);
-    m_readyFrameIndex.store(-1, std::memory_order_release);
+    Atomic::Store(m_submittedFrameIndex, -1, std::memory_order_release);
+    Atomic::Store(m_readyFrameIndex, -1, std::memory_order_release);
     m_deferredOwnerFrameIndex = -1;
     m_framePublishSequence = 0;
     m_frameReadySequence   = 0;
@@ -238,7 +239,7 @@ void CLGMPFrameTransport::DeInit()
 
   for (int i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i)
   {
-    m_frameInFlight[i].store(false, std::memory_order_release);
+    Atomic::Store(m_frameInFlight[i], false, std::memory_order_release);
     lgmpHostMemFree(&m_frameMemory[i]);
     m_frame[i]       = nullptr;
     m_frameBuffer[i] = nullptr;
@@ -571,14 +572,15 @@ int CLGMPFrameTransport::FindAvailableFrameBuffer(
   bool allowReady) const
 {
   const LONG readyFrameIndex =
-    m_readyFrameIndex.load(std::memory_order_acquire);
+    Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
   int      available     = -1;
   uint64_t newestPublish = 0;
   for (unsigned frameIndex = 0;
        frameIndex < LGMP_Q_FRAME_BUFFER_LEN; ++frameIndex)
   {
     if (static_cast<LONG>(frameIndex) == readyFrameIndex ||
-        m_frameInFlight[frameIndex].load(std::memory_order_acquire) ||
+        Atomic::Load(
+          m_frameInFlight[frameIndex], std::memory_order_acquire) ||
         FrameBufferReferenced(frameIndex))
       continue;
 
@@ -591,7 +593,8 @@ int CLGMPFrameTransport::FindAvailableFrameBuffer(
   }
 
   if (available >= 0 || !allowReady || readyFrameIndex < 0 ||
-      m_frameInFlight[readyFrameIndex].load(std::memory_order_acquire) ||
+      Atomic::Load(
+        m_frameInFlight[readyFrameIndex], std::memory_order_acquire) ||
       FrameBufferReferenced(static_cast<unsigned>(readyFrameIndex)))
     return available;
 
@@ -610,7 +613,8 @@ int CLGMPFrameTransport::FindNewestCompletedFrame(
        frameIndex < LGMP_Q_FRAME_BUFFER_LEN; ++frameIndex)
   {
     if (frameIndex == excludeFrameIndex || !m_frameCompleted[frameIndex] ||
-        m_frameInFlight[frameIndex].load(std::memory_order_acquire))
+        Atomic::Load(
+          m_frameInFlight[frameIndex], std::memory_order_acquire))
       continue;
 
     if (newestFrame < 0 ||
@@ -679,9 +683,10 @@ bool CLGMPFrameTransport::GetPendingDeliveryTarget(uint64_t now,
 
   CSRWSharedLock lock(m_framePublishLock);
   const LONG frameIndex =
-    m_readyFrameIndex.load(std::memory_order_acquire);
+    Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
   if (frameIndex < 0 ||
-      m_frameInFlight[frameIndex].load(std::memory_order_acquire) ||
+      Atomic::Load(
+        m_frameInFlight[frameIndex], std::memory_order_acquire) ||
       lgmpHostQueuePending(m_frameQueue) != 0)
     return false;
 
@@ -705,9 +710,10 @@ bool CLGMPFrameTransport::RetryPendingDelivery(uint64_t now, bool& retry)
 
   CSRWExclusiveLock lock(m_framePublishLock);
   const LONG frameIndex =
-    m_readyFrameIndex.load(std::memory_order_acquire);
+    Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
   if (frameIndex < 0 ||
-      m_frameInFlight[frameIndex].load(std::memory_order_acquire) ||
+      Atomic::Load(
+        m_frameInFlight[frameIndex], std::memory_order_acquire) ||
       lgmpHostQueuePending(m_frameQueue) != 0)
     return false;
 
@@ -747,14 +753,14 @@ SinkTarget CLGMPFrameTransport::PrepareFrameBuffer(
     FindAvailableFrameBuffer(allowReady);
   bool expected = false;
   const bool acquired = availableFrameIndex >= 0 &&
-    m_frameInFlight[availableFrameIndex].compare_exchange_strong(
-      expected, true, std::memory_order_acq_rel);
+    Atomic::CAS(m_frameInFlight[availableFrameIndex], expected, true,
+      std::memory_order_acq_rel);
   if (acquired)
   {
     const LONG readyFrameIndex =
-      m_readyFrameIndex.load(std::memory_order_acquire);
+      Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
     if (availableFrameIndex == readyFrameIndex)
-      m_readyFrameIndex.store(
+      Atomic::Store(m_readyFrameIndex,
         FindNewestCompletedFrame(
           static_cast<unsigned>(availableFrameIndex)),
         std::memory_order_release);
@@ -872,7 +878,7 @@ SinkTarget CLGMPFrameTransport::PrepareFrameBuffer(
   fi->scheduleGeneration     = 0;
   fi->scheduleEpoch          = 0;
   fi->scheduleDeadlineSerial = 0;
-  InterlockedExchange((volatile LONG *)&fi->timingValid, 0);
+  Atomic::Store(fi->timingValid, 0);
   fi->rotation = FRAME_ROT_0;
   fi->type     = dstFormat.format;
 
@@ -1015,7 +1021,7 @@ bool CLGMPFrameTransport::PublishFrameBuffer(unsigned frameIndex,
     m_frameDelivered[frameIndex]  = deliveredToOwner;
     m_deferredOwnerFrameIndex = schedule.clientID && !deliveredToOwner ?
       static_cast<LONG>(frameIndex) : -1;
-    m_submittedFrameIndex.store(
+    Atomic::Store(m_submittedFrameIndex,
       static_cast<LONG>(frameIndex), std::memory_order_release);
   }
   lock.Unlock();
@@ -1046,15 +1052,18 @@ bool CLGMPFrameTransport::RepublishFrameBuffer(
   LONG frameIndex = m_deferredOwnerFrameIndex;
   if (frameIndex >= 0 &&
       !m_frameCompleted[frameIndex] &&
-      !m_frameInFlight[frameIndex].load(std::memory_order_acquire))
+      !Atomic::Load(
+        m_frameInFlight[frameIndex], std::memory_order_acquire))
   {
     m_deferredOwnerFrameIndex = -1;
     frameIndex = -1;
   }
   if (frameIndex < 0)
-    frameIndex = m_readyFrameIndex.load(std::memory_order_acquire);
+    frameIndex = Atomic::Load(
+      m_readyFrameIndex, std::memory_order_acquire);
   if (frameIndex < 0 ||
-      m_frameInFlight[frameIndex].load(std::memory_order_acquire))
+      Atomic::Load(
+        m_frameInFlight[frameIndex], std::memory_order_acquire))
     return false;
 
   CFrameScheduler::Schedule deliverySchedule = schedule;
@@ -1188,12 +1197,12 @@ void CLGMPFrameTransport::AbortFrameBuffer(unsigned frameIndex)
 
   CSRWExclusiveLock lock(m_framePublishLock);
   m_frameBuffer[frameIndex]->wp = 0;
-  InterlockedExchange(
-    (volatile LONG *)&m_frame[frameIndex]->timingValid, 0);
+  Atomic::Store(m_frame[frameIndex]->timingValid, 0);
   m_frameCompleted[frameIndex] = false;
   if (m_deferredOwnerFrameIndex == static_cast<LONG>(frameIndex))
     m_deferredOwnerFrameIndex = -1;
-  m_frameInFlight[frameIndex].store(false, std::memory_order_release);
+  Atomic::Store(
+    m_frameInFlight[frameIndex], false, std::memory_order_release);
 }
 
 void CLGMPFrameTransport::FailFrameBuffer(unsigned frameIndex)
@@ -1201,8 +1210,7 @@ void CLGMPFrameTransport::FailFrameBuffer(unsigned frameIndex)
   if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN)
     return;
 
-  InterlockedExchange(
-    (volatile LONG *)&m_frame[frameIndex]->timingValid, 0);
+  Atomic::Store(m_frame[frameIndex]->timingValid, 0);
   FinalizeFrameBuffer(frameIndex);
   AbortFrameBuffer(frameIndex);
 }
@@ -1228,14 +1236,15 @@ void CLGMPFrameTransport::CompleteFrameBuffer(
     // Completion callbacks may run out of order. Never replace a newer ready
     // frame with an older submission.
     const LONG readyFrameIndex =
-      m_readyFrameIndex.load(std::memory_order_acquire);
+      Atomic::Load(m_readyFrameIndex, std::memory_order_acquire);
     if (sequence &&
         (readyFrameIndex < 0 ||
           sequence > m_frameLastPublishSequence[readyFrameIndex]))
-      m_readyFrameIndex.store(
+      Atomic::Store(m_readyFrameIndex,
         static_cast<LONG>(frameIndex), std::memory_order_release);
   }
-  m_frameInFlight[frameIndex].store(false, std::memory_order_release);
+  Atomic::Store(
+    m_frameInFlight[frameIndex], false, std::memory_order_release);
   const bool newerThanReady =
     sequence && sequence > m_frameReadySequence;
   if (result == FrameDone::READY && newerThanReady)
@@ -1276,7 +1285,7 @@ void CLGMPFrameTransport::SetFrameTiming(unsigned frameIndex,
   frame->timingFlags  = phaseValid ?
     KVMFR_FRAME_TIMING_PHASE_VALID : 0;
   frame->timingSerial = frame->frameSerial;
-  InterlockedExchange((volatile LONG *)&frame->timingValid, 1);
+  Atomic::Store(frame->timingValid, 1);
 }
 
 void CLGMPFrameTransport::WriteFrameBuffer(unsigned frameIndex, void * src,

idd/LGIdd/transport/lgmp/CLGMPFrameTransport.h

@@ -20,10 +20,10 @@
 
 #pragma once
 
+#include "Atomic.h"
 #include "CSRWLock.h"
 
 #include <Windows.h>
-#include <atomic>
 #include <stdint.h>
 
 extern "C" {
@@ -173,7 +173,7 @@ public:
     bool allowReadyReplacement = true) override;
   bool HasPublishedFrame() const override
   {
-    return m_readyFrameIndex.load(std::memory_order_acquire) >= 0;
+    return Atomic::Load(m_readyFrameIndex, std::memory_order_acquire) >= 0;
   }
   void ProcessDeliveries() override;
   bool GetPendingDeliveryTarget(

idd/LGIdd/transport/lgmp/CLGMPInputTransport.cpp

@@ -21,6 +21,7 @@
 #include "transport/lgmp/CLGMPInputTransport.h"
 
 #include "transport/lgmp/CLGMPHost.h"
+#include "Atomic.h"
 #include "CDebug.h"
 #include "CSRWLock.h"
 #include "Seq.h"
@@ -201,10 +202,11 @@ bool CLGMPInputTransport::PublishStatus()
 
 void CLGMPInputTransport::FlushStatus()
 {
-  if (m_statusFailed.load(std::memory_order_acquire) || PublishStatus())
+  if (Atomic::Load(m_statusFailed, std::memory_order_acquire) ||
+      PublishStatus())
     return;
 
-  m_statusFailed.store(true, std::memory_order_release);
+  Atomic::Store(m_statusFailed, true, std::memory_order_release);
   CSRWSharedLock lock(m_lifecycleLock);
   if (m_stopEvent)
     SetEvent(m_stopEvent);
@@ -264,7 +266,7 @@ bool CLGMPInputTransport::Start(IInputTarget& target)
     Seq::Inc(m_endpointGeneration);
     m_statusDirty = true;
   }
-  m_statusFailed.store(false, std::memory_order_release);
+  Atomic::Store(m_statusFailed, false, std::memory_order_release);
   m_thread = CreateThread(nullptr, 0, ThreadProc, this, 0, nullptr);
   if (!m_thread)
   {
@@ -693,7 +695,7 @@ void CLGMPInputTransport::Thread()
     }
     if (!PublishStatus())
     {
-      m_statusFailed.store(true, std::memory_order_release);
+      Atomic::Store(m_statusFailed, true, std::memory_order_release);
       failed = true;
       break;
     }
@@ -716,7 +718,7 @@ void CLGMPInputTransport::Thread()
       _countof(waitHandles), waitHandles, FALSE, INFINITE);
     if (wait == WAIT_FIRST_OBJECT_VALUE)
     {
-      failed = m_statusFailed.load(std::memory_order_acquire);
+      failed = Atomic::Load(m_statusFailed, std::memory_order_acquire);
       break;
     }
     if (wait != WAIT_FIRST_OBJECT_VALUE + 1)

idd/LGIdd/transport/lgmp/CLGMPInputTransport.h

@@ -20,13 +20,13 @@
 
 #pragma once
 
+#include "Atomic.h"
 #include "CSRWLock.h"
 #include "transport/IInputSource.h"
 #include "common/LGMPConfig.h"
 
 #include <Windows.h>
 
-#include <atomic>
 #include <stdint.h>
 
 extern "C" {

idd/LGIdd/transport/lgmp/CLGMPTransport.cpp

@@ -20,6 +20,7 @@
 
 #include "transport/lgmp/CLGMPTransport.h"
 
+#include "Atomic.h"
 #include "CDebug.h"
 #include "common/KVMFR.h"
 #include "common/KVMFRRecovery.h"
@@ -90,7 +91,7 @@ bool CLGMPTransport::Setup(size_t alignment)
   if (!m_frames.Setup(alignment))
     return false;
 
-  m_ready.store(true, std::memory_order_release);
+  Atomic::Store(m_ready, true, std::memory_order_release);
   return true;
 }
 
@@ -109,7 +110,7 @@ ITransport::ProcessResult CLGMPTransport::Process(ITransportEvents& events)
   // Before the swap chain establishes the frame-buffer alignment, service
   // only the protocol-independent recovery channel. This preserves the old
   // transport startup boundary while keeping recovery available immediately.
-  if (!m_ready.load(std::memory_order_acquire))
+  if (!Atomic::Load(m_ready, std::memory_order_acquire))
     return ProcessResult::OK;
 
   const LGMP_STATUS processStatus = m_host.Process();
@@ -224,7 +225,7 @@ ITransport::ProcessResult CLGMPTransport::Process(ITransportEvents& events)
 
 void CLGMPTransport::Stop()
 {
-  m_ready.store(false, std::memory_order_release);
+  Atomic::Store(m_ready, false, std::memory_order_release);
   m_input.Stop();
 }
 

idd/LGIdd/transport/lgmp/CLGMPTransport.h

@@ -20,6 +20,7 @@
 
 #pragma once
 
+#include "Atomic.h"
 #include "transport/ITransport.h"
 #include "transport/lgmp/CIVSHMEM.h"
 #include "transport/lgmp/CLGMPControl.h"
@@ -28,8 +29,6 @@
 #include "transport/lgmp/CLGMPInputTransport.h"
 #include "transport/lgmp/CRecovery.h"
 
-#include <atomic>
-
 class CLGMPTransport final : public ITransport
 {
 private:

idd/LGIdd/transport/lgmp/CRecovery.cpp

@@ -22,6 +22,7 @@
 
 #include "transport/lgmp/CIVSHMEM.h"
 #include "platform/CPlatformInfo.h"
+#include "Atomic.h"
 #include "CDebug.h"
 #include "VersionInfo.h"
 
@@ -38,36 +39,6 @@ namespace
   static const uint64_t HELPER_TIMEOUT_MS = 30000;
   CSRWLock l_wireLock;
 
-  uint32_t AtomicRead(uint32_t& value)
-  {
-    return static_cast<uint32_t>(InterlockedCompareExchange(
-      (volatile LONG *)&value, 0, 0));
-  }
-
-  void AtomicWrite(uint32_t& value, uint32_t data)
-  {
-    InterlockedExchange((volatile LONG *)&value, static_cast<LONG>(data));
-  }
-
-  void AtomicIncrement(uint32_t& value)
-  {
-    InterlockedIncrement((volatile LONG *)&value);
-  }
-
-  uint32_t AtomicAdd(uint32_t& value, uint32_t data)
-  {
-    return static_cast<uint32_t>(InterlockedExchangeAdd(
-      (volatile LONG *)&value, static_cast<LONG>(data))) + data;
-  }
-
-  bool AtomicCompareExchange(
-    uint32_t& value, uint32_t expected, uint32_t data)
-  {
-    return static_cast<uint32_t>(InterlockedCompareExchange(
-      (volatile LONG *)&value, static_cast<LONG>(data),
-      static_cast<LONG>(expected))) == expected;
-  }
-
   uint64_t CreateSession(const void * memory, uint64_t previous)
   {
     LARGE_INTEGER counter;
@@ -89,13 +60,13 @@ namespace
 
 bool CRecovery::OwnsSession()
 {
-  if (AtomicRead(m_data->header.ready) != KVMFR_R_READY)
+  if (Atomic::Load(m_data->header.ready) != KVMFR_R_READY)
     return false;
 
   const uint64_t session = m_data->header.session;
-  MemoryBarrier();
+  Atomic::Fence();
   return session == m_session &&
-    AtomicRead(m_data->header.ready) == KVMFR_R_READY;
+    Atomic::Load(m_data->header.ready) == KVMFR_R_READY;
 }
 
 bool CRecovery::ReadRequest(
@@ -103,14 +74,14 @@ bool CRecovery::ReadRequest(
 {
   for (unsigned i = 0; i < 4; ++i)
   {
-    const uint32_t serial = AtomicRead(source.serial);
+    const uint32_t serial = Atomic::Load(source.serial);
     if (!serial || (serial & 1U))
       return false;
 
     const uint32_t type    = source.request;
     const uint64_t session = source.session;
-    MemoryBarrier();
-    if (AtomicRead(source.serial) == serial)
+    Atomic::Fence();
+    if (Atomic::Load(source.serial) == serial)
     {
       result.serial  = serial;
       result.request = type;
@@ -126,7 +97,7 @@ bool CRecovery::ReadStatus(KVMFRRStatus& source, KVMFRRStatus& result)
 {
   for (unsigned i = 0; i < 4; ++i)
   {
-    const uint32_t serial = AtomicRead(source.serial);
+    const uint32_t serial = Atomic::Load(source.serial);
     if (!serial || (serial & 1U))
       continue;
 
@@ -135,8 +106,8 @@ bool CRecovery::ReadStatus(KVMFRRStatus& source, KVMFRRStatus& result)
     result.state      = source.state;
     result.error      = source.error;
     result.session    = source.session;
-    MemoryBarrier();
-    if (AtomicRead(source.serial) == serial)
+    Atomic::Fence();
+    if (Atomic::Load(source.serial) == serial)
     {
       result.serial = serial;
       return true;
@@ -154,10 +125,7 @@ bool CRecovery::SerialNewer(uint32_t serial, uint32_t reference)
 
 uint32_t CRecovery::NextTicket()
 {
-  uint32_t ticket = AtomicAdd(m_data->req.ticket, 2U);
-  if (!ticket)
-    ticket = AtomicAdd(m_data->req.ticket, 2U);
-  return ticket;
+  return Atomic::Next(m_data->req.ticket, 2U);
 }
 
 void CRecovery::Publish(uint32_t serial, uint32_t request,
@@ -168,13 +136,13 @@ void CRecovery::Publish(uint32_t serial, uint32_t request,
   if (!published)
     published = KVMFR_R_REQ_FIRST;
 
-  AtomicWrite(m_data->status.serial, writing);
+  Atomic::Store(m_data->status.serial, writing);
   m_data->status.ackRequest = request;
   m_data->status.state      = state;
   m_data->status.error      = error;
   m_data->status.session    = m_session;
   m_data->status.ackSerial  = serial;
-  AtomicWrite(m_data->status.serial, published);
+  Atomic::Store(m_data->status.serial, published);
   m_statusSerial = published;
 }
 
@@ -192,7 +160,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem)
     return false;
   }
 
-  const uint32_t oldReady = AtomicRead(m_data->header.ready);
+  const uint32_t oldReady = Atomic::Load(m_data->header.ready);
   const bool oldValid = oldReady == KVMFR_R_READY &&
     memcmp(m_data->header.magic, KVMFR_R_MAGIC,
       sizeof(m_data->header.magic)) == 0 &&
@@ -220,7 +188,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem)
     }
   }
 
-  AtomicWrite(m_data->header.ready, 0);
+  Atomic::Store(m_data->header.ready, 0);
   if (oldValid)
   {
     ZeroMemory(&m_data->header, sizeof(m_data->header));
@@ -233,8 +201,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem)
     {
       KVMFRRRequest request = {};
       if (ReadRequest(m_data->requests[i], request))
-        AtomicCompareExchange(
-          m_data->requests[i].serial, request.serial, 0);
+        Atomic::CAS(m_data->requests[i].serial, request.serial, 0);
     }
   }
   else
@@ -264,7 +231,7 @@ bool CRecovery::Initialize(CIVSHMEM& ivshmem)
     KVMFR_R_STATE_SWITCHING, KVMFR_R_ERR_NONE);
 
   m_nextHeartbeat = GetTickCount64() + KVMFR_R_HEARTBEAT_MS;
-  AtomicWrite(m_data->header.ready, KVMFR_R_READY);
+  Atomic::Store(m_data->header.ready, KVMFR_R_READY);
 
   DEBUG_INFO("Recovery channel initialized (session %llu%s)",
     (unsigned long long)m_session,
@@ -289,7 +256,7 @@ CRecovery::Request CRecovery::Process()
   const uint64_t now = GetTickCount64();
   if (now >= m_nextHeartbeat)
   {
-    AtomicIncrement(m_data->header.heartbeat);
+    Atomic::Inc(m_data->header.heartbeat);
     m_nextHeartbeat = now + KVMFR_R_HEARTBEAT_MS;
   }
 
@@ -371,8 +338,7 @@ CRecovery::Request CRecovery::Process()
   // slot without a corresponding acknowledgement.
   for (unsigned i = 0; i < KVMFR_R_REQ_SLOTS; ++i)
     if (stable[i])
-      AtomicCompareExchange(
-        m_data->requests[i].serial, requests[i].serial, 0);
+      Atomic::CAS(m_data->requests[i].serial, requests[i].serial, 0);
 
   if (m_waiting && now >= m_deadline)
   {