[client/idd] align frame delivery to presentation

client/src/frame_scheduler.c

@@ -31,6 +31,7 @@
 
 #define FRAME_SCHEDULER_LEASE_MS        1000U
 #define FRAME_SCHEDULER_RENEW_NS        250000000ULL
+#define FRAME_SCHEDULER_FEEDBACK_NS     50000000ULL
 #define FRAME_SCHEDULER_TARGET_SLACK_NS 500000ULL
 #define FRAME_SCHEDULER_MIN_PERIOD_NS   2000000ULL
 #define FRAME_SCHEDULER_MAX_PERIOD_NS   1000000000ULL
@@ -40,14 +41,19 @@ static struct
 {
   LG_Lock lock;
 
-  bool     supported;
+  _Atomic(bool)     supported;
-  bool     active;
+  _Atomic(bool)     active;
   bool              controlPending;
-  uint32_t generation;
+  _Atomic(uint32_t) generation;
-  uint64_t period;
+  _Atomic(uint64_t) period;
   uint64_t          lastSend;
   uint64_t          lastCadence;
 
+  int64_t  phaseError;
+  uint32_t feedbackFrameSerial;
+  unsigned feedbackSamples;
+  bool     feedbackDirty;
+
   LG_TransportControlToken controlToken;
 
   uint64_t renderSamples[FRAME_SCHEDULER_SAMPLE_COUNT];
@@ -79,7 +85,11 @@ static uint64_t fallbackPeriod(void)
     return 0;
 
   qsort(samples, count, sizeof(*samples), compareU64);
-  return samples[count / 2];
+  const uint64_t period = samples[count / 2];
+  if (samples[count * 3 / 4] - samples[count / 4] > period / 20)
+    return 0;
+
+  return period;
 }
 
 static uint64_t presentationPeriod(void)
@@ -89,7 +99,7 @@ static uint64_t presentationPeriod(void)
       g_state.ds->getFramePeriod(&period))
     return period;
 
-  return g_state.jitRender ? fallbackPeriod() : 0;
+  return fallbackPeriod();
 }
 
 static bool controlReady(void)
@@ -112,6 +122,13 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags,
   if (!controlReady())
     return false;
 
+  int64_t  phaseError;
+  uint32_t feedbackFrameSerial;
+  LG_LOCK(l_frameScheduler.lock);
+  phaseError          = l_frameScheduler.phaseError;
+  feedbackFrameSerial = l_frameScheduler.feedbackFrameSerial;
+  LG_UNLOCK(l_frameScheduler.lock);
+
   const LG_TransportControl control = {
     .type = LG_TRANSPORT_CONTROL_FRAME_SCHEDULE,
     .frameSchedule = {
@@ -119,6 +136,8 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags,
       .flags               = flags,
       .period              = period,
       .targetSlack         = FRAME_SCHEDULER_TARGET_SLACK_NS,
+      .phaseError          = phaseError,
+      .feedbackFrameSerial = feedbackFrameSerial,
       .lease               = FRAME_SCHEDULER_LEASE_MS,
     },
   };
@@ -134,6 +153,10 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags,
   }
 
   l_frameScheduler.controlPending = true;
+  LG_LOCK(l_frameScheduler.lock);
+  if (l_frameScheduler.feedbackFrameSerial == feedbackFrameSerial)
+    l_frameScheduler.feedbackDirty = false;
+  LG_UNLOCK(l_frameScheduler.lock);
   return true;
 }
 
@@ -163,6 +186,10 @@ void frameScheduler_start(LG_TransportFeatureFlags features)
   l_frameScheduler.renderSampleIndex   = 0;
   l_frameScheduler.renderSampleCount   = 0;
   l_frameScheduler.lastRender          = 0;
+  l_frameScheduler.phaseError          = 0;
+  l_frameScheduler.feedbackFrameSerial = 0;
+  l_frameScheduler.feedbackSamples     = 0;
+  l_frameScheduler.feedbackDirty       = false;
   LG_UNLOCK(l_frameScheduler.lock);
 }
 
@@ -213,9 +240,16 @@ void frameScheduler_update(void)
     l_frameScheduler.period =
       (l_frameScheduler.period * 7 + period) / 8;
 
-  if (l_frameScheduler.active && !reset &&
+  if (l_frameScheduler.active && !reset)
-      now - l_frameScheduler.lastSend < FRAME_SCHEDULER_RENEW_NS)
+  {
+    LG_LOCK(l_frameScheduler.lock);
+    const bool feedbackDirty = l_frameScheduler.feedbackDirty;
+    LG_UNLOCK(l_frameScheduler.lock);
+    const uint64_t interval = feedbackDirty ?
+      FRAME_SCHEDULER_FEEDBACK_NS : FRAME_SCHEDULER_RENEW_NS;
+    if (now - l_frameScheduler.lastSend < interval)
       return;
+  }
 
   LG_TransportFrameScheduleFlags flags =
     LG_TRANSPORT_FRAME_SCHEDULE_ACTIVE;
@@ -231,9 +265,6 @@ void frameScheduler_update(void)
 
 void frameScheduler_observeRender(uint64_t timestamp)
 {
-  if (!g_state.jitRender)
-    return;
-
   LG_LOCK(l_frameScheduler.lock);
   if (l_frameScheduler.lastRender &&
       timestamp > l_frameScheduler.lastRender)
@@ -249,3 +280,40 @@ void frameScheduler_observeRender(uint64_t timestamp)
   l_frameScheduler.lastRender = timestamp;
   LG_UNLOCK(l_frameScheduler.lock);
 }
+
+void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation,
+    uint64_t queueStart, uint64_t prepareStart)
+{
+  if (!generation || !queueStart || prepareStart < queueStart)
+    return;
+
+  const uint64_t slack = prepareStart - queueStart;
+  LG_LOCK(l_frameScheduler.lock);
+  if (!l_frameScheduler.supported || !l_frameScheduler.active ||
+      generation != l_frameScheduler.generation)
+  {
+    LG_UNLOCK(l_frameScheduler.lock);
+    return;
+  }
+
+  int64_t error = slack > FRAME_SCHEDULER_TARGET_SLACK_NS ?
+    (int64_t)(slack - FRAME_SCHEDULER_TARGET_SLACK_NS) :
+    -(int64_t)(FRAME_SCHEDULER_TARGET_SLACK_NS - slack);
+  const int64_t limit = (int64_t)(l_frameScheduler.period / 2);
+  if (error > limit)
+    error = limit;
+  else if (error < -limit)
+    error = -limit;
+
+  if (!l_frameScheduler.feedbackSamples)
+    l_frameScheduler.phaseError = error;
+  else
+    l_frameScheduler.phaseError =
+      (l_frameScheduler.phaseError * 7 + error) / 8;
+  if (l_frameScheduler.feedbackSamples < 32)
+    ++l_frameScheduler.feedbackSamples;
+
+  l_frameScheduler.feedbackFrameSerial = (uint32_t)frameSerial;
+  l_frameScheduler.feedbackDirty       = true;
+  LG_UNLOCK(l_frameScheduler.lock);
+}

client/src/frame_scheduler.h

@@ -31,5 +31,7 @@ void frameScheduler_start(LG_TransportFeatureFlags features);
 void frameScheduler_stop(void);
 void frameScheduler_update(void);
 void frameScheduler_observeRender(uint64_t timestamp);
+void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation,
+    uint64_t queueStart, uint64_t prepareStart);
 
 #endif

client/src/main.c

@@ -212,6 +212,8 @@ enum FrameTimingReady
 struct FrameTimingRecord
 {
   LG_RendererFrameToken token;
+  uint64_t              frameSerial;
+  uint32_t              scheduleGeneration;
   unsigned              readyMask;
   bool                  producerValid;
 
@@ -323,7 +325,8 @@ static void frameTimingCancel(LG_RendererFrameToken token)
   });
 }
 
-static void frameTimingQueue(LG_RendererFrameToken token, uint64_t importTime,
+static void frameTimingQueue(LG_RendererFrameToken token, uint64_t frameSerial,
+    uint32_t scheduleGeneration, uint64_t importTime,
     uint64_t importWaitTime, uint64_t dispatchStart, uint64_t queueStart)
 {
   INTERLOCKED_SECTION(l_frameTiming.lock, {
@@ -337,6 +340,8 @@ static void frameTimingQueue(LG_RendererFrameToken token, uint64_t importTime,
       record->importWaitTime     = importWaitTime;
       record->dispatchTime       = elapsed > accounted ? elapsed - accounted : 0;
       record->queueStart         = queueStart;
+      record->frameSerial        = frameSerial;
+      record->scheduleGeneration = scheduleGeneration;
       if (record->timestamp < queueStart)
         record->timestamp = queueStart;
     }
@@ -382,7 +387,11 @@ static void frameTimingFinishFrame(LG_RendererFrameToken token,
 static void frameTimingFinishRender(const LG_RendererFrameTiming * timing,
     uint64_t prepareStart, uint64_t prepareTime, uint64_t timestamp)
 {
-  INTERLOCKED_SECTION(l_frameTiming.lock, {
+  uint64_t feedbackFrameSerial = 0;
+  uint64_t feedbackQueueStart  = 0;
+  uint32_t feedbackGeneration  = 0;
+
+  LG_LOCK(l_frameTiming.lock);
   if (l_frameTiming.retireToken <= timing->frameToken)
     l_frameTiming.retireToken = timing->frameToken + 1;
 
@@ -400,6 +409,10 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing,
     record->swapTime    = timing->swapTime;
     record->readyMask  |= FRAME_TIMING_RENDER_READY;
 
+    feedbackFrameSerial = record->frameSerial;
+    feedbackGeneration  = record->scheduleGeneration;
+    feedbackQueueStart  = record->queueStart;
+
     if (unlikely(
         l_frameTiming.publishCount == FRAME_TIMING_RECORD_COUNT))
     {
@@ -419,7 +432,10 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing,
       ++l_frameTiming.publishCount;
     }
   }
-  });
+  LG_UNLOCK(l_frameTiming.lock);
+
+  frameScheduler_feedback(feedbackFrameSerial, feedbackGeneration,
+      feedbackQueueStart, prepareStart);
 }
 
 static void frameTimingPublishReady(void)
@@ -1083,8 +1099,9 @@ int main_frameThread(void * unused)
     atomic_store_explicit(&l_testFrameSerial, frame.serial,
         memory_order_release);
 #endif
-    frameTimingQueue(frameToken, g_state.frameImportTime,
+    frameTimingQueue(frameToken, frame.serial, frame.scheduleGeneration,
-        g_state.frameImportWaitTime, dispatchStart, queueStart);
+        g_state.frameImportTime, g_state.frameImportWaitTime,
+        dispatchStart, queueStart);
 
     if (g_state.jitRender)
     {

common/include/common/KVMFR.h

@@ -291,11 +291,12 @@ typedef struct KVMFRFrameSchedule
   uint32_t                clientID;
   uint32_t                generation;
   KVMFRFrameScheduleFlags flags;
-  uint64_t                period;
+  uint64_t                period;       // requested presentation period (ns)
-  uint64_t                targetSlack;
+  uint64_t                targetSlack;  // desired ready-to-render lead (ns)
-  int64_t                 phaseError;
+  // Positive when the frame arrived early, negative when it arrived late.
+  int64_t                 phaseError;   // ready-to-render phase error (ns)
   uint32_t                feedbackFrameSerial;
-  uint32_t                lease;
+  uint32_t                lease;        // lease duration (ms)
   uint8_t                 reserved[16];
 }
 KVMFRFrameSchedule;

idd/LGIdd/CFrameScheduler.cpp

@@ -111,6 +111,8 @@ void CFrameScheduler::ElectOwner(uint64_t now)
     m_nextDeadline = m_scheduling ? now + m_schedule.period : 0;
     m_forceNext    = m_scheduling;
 
+    m_lastPublishedFrameSerial = 0;
+
     if (m_scheduling)
       DEBUG_INFO("Frame timing owner %u generation %u at %.3f Hz",
         m_schedule.clientID, m_schedule.generation,
@@ -136,6 +138,8 @@ void CFrameScheduler::Reset()
   m_nextDeadline   = 0;
   m_arrivalSamples = 0;
   m_timingSamples  = 0;
+
+  m_lastPublishedFrameSerial = 0;
   ReleaseSRWLockExclusive(&m_lock);
 }
 
@@ -204,22 +208,61 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule,
       schedule.period < MIN_PERIOD_NS ||
       schedule.period > MAX_PERIOD_NS ||
       schedule.targetSlack >= schedule.period ||
+      schedule.phaseError > static_cast<int64_t>(schedule.period) ||
+      schedule.phaseError < -static_cast<int64_t>(schedule.period) ||
       schedule.lease < MIN_LEASE_MS || schedule.lease > MAX_LEASE_MS)
   {
     ReleaseSRWLockExclusive(&m_lock);
     return false;
   }
 
+  if (client->generation != schedule.generation)
+    client->lastFeedbackFrameSerial = 0;
   client->generation  = schedule.generation;
   client->period      = schedule.period;
   client->targetSlack = schedule.targetSlack;
   client->expiry      = now + static_cast<uint64_t>(schedule.lease) * 1000000;
   client->active      = true;
+  if (schedule.flags & KVMFR_FRAME_SCHEDULE_IMMEDIATE)
+    m_forceNext = true;
   ElectOwner(now);
+  ApplyFeedback(*client, schedule);
   ReleaseSRWLockExclusive(&m_lock);
   return true;
 }
 
+void CFrameScheduler::ApplyFeedback(Client& client,
+  const KVMFRFrameSchedule& schedule)
+{
+  if (!m_scheduling || client.clientID != m_schedule.clientID ||
+      schedule.generation != m_schedule.generation ||
+      !schedule.feedbackFrameSerial ||
+      (client.lastFeedbackFrameSerial &&
+        static_cast<int32_t>(schedule.feedbackFrameSerial -
+          client.lastFeedbackFrameSerial) <= 0) ||
+      !m_lastPublishedFrameSerial ||
+      static_cast<int32_t>(schedule.feedbackFrameSerial -
+        m_lastPublishedFrameSerial) > 0)
+    return;
+
+  int64_t correction = schedule.phaseError / 4;
+  const int64_t limit = static_cast<int64_t>(m_schedule.period / 4);
+  if (correction > limit)
+    correction = limit;
+  else if (correction < -limit)
+    correction = -limit;
+
+  if (correction >= 0)
+    m_nextDeadline += static_cast<uint64_t>(correction);
+  else
+  {
+    const uint64_t advance = static_cast<uint64_t>(-correction);
+    m_nextDeadline = m_nextDeadline > advance ?
+      m_nextDeadline - advance : 0;
+  }
+  client.lastFeedbackFrameSerial = schedule.feedbackFrameSerial;
+}
+
 bool CFrameScheduler::GetSchedule(Schedule& schedule) const
 {
   AcquireSRWLockShared(&m_lock);
@@ -290,12 +333,14 @@ bool CFrameScheduler::SelectFrame(uint64_t now, bool force,
   return process;
 }
 
-void CFrameScheduler::FramePublished(uint32_t generation, uint64_t now)
+void CFrameScheduler::FramePublished(uint32_t generation,
+  uint32_t frameSerial, uint64_t now)
 {
   AcquireSRWLockExclusive(&m_lock);
   if (m_scheduling && generation == m_schedule.generation)
   {
     m_forceNext = false;
+    m_lastPublishedFrameSerial = frameSerial;
     do
       m_nextDeadline += m_schedule.period;
     while (m_nextDeadline <= now);

idd/LGIdd/CFrameScheduler.h

@@ -48,6 +48,7 @@ private:
     uint64_t period;
     uint64_t targetSlack;
     uint64_t expiry;
+    uint32_t lastFeedbackFrameSerial;
     bool     subscribed;
     bool     active;
   };
@@ -66,8 +67,11 @@ private:
   unsigned m_arrivalSamples = 0;
   unsigned m_timingSamples  = 0;
 
+  uint32_t m_lastPublishedFrameSerial = 0;
+
   Client * FindClient(uint32_t clientID);
   void ElectOwner(uint64_t now);
+  void ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule);
 
 public:
   static uint64_t Nanotime();
@@ -79,6 +83,7 @@ public:
   bool GetSchedule(Schedule& schedule) const;
   void ObserveFrame(uint64_t now);
   bool SelectFrame(uint64_t now, bool force, uint32_t& generation);
-  void FramePublished(uint32_t generation, uint64_t now);
+  void FramePublished(uint32_t generation, uint32_t frameSerial,
+    uint64_t now);
   void RecordFrameTiming(uint64_t duration);
 };

idd/LGIdd/CIndirectDeviceContext.cpp

@@ -945,7 +945,8 @@ bool CIndirectDeviceContext::InitializeLGMP()
     kvmfr.version  = KVMFR_VERSION;
     kvmfr.features =
       KVMFR_FEATURE_SETCURSORPOS |
-      KVMFR_FEATURE_WINDOWSIZE;
+      KVMFR_FEATURE_WINDOWSIZE   |
+      KVMFR_FEATURE_FRAME_SCHEDULE;
     strncpy_s(kvmfr.hostver, LG_VERSION_STR, sizeof(kvmfr.hostver) - 1);
     ss.write(reinterpret_cast<const char *>(&kvmfr), sizeof(kvmfr));
   }
@@ -1523,7 +1524,8 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex,
 
   if (status == LGMP_OK)
     m_frameScheduler.FramePublished(
-      scheduleGeneration, CFrameScheduler::Nanotime());
+      scheduleGeneration, m_frame[frameIndex]->frameSerial,
+      CFrameScheduler::Nanotime());
 
   if (status != LGMP_OK)
   {