[client/idd] align frame delivery to presentation

2026-08-04 06:12:04 +00:00 · 2026-08-04 13:15:37 +10:00
parent 32ced2f24a
commit 97e7f439c2
7 changed files with 211 additions and 71 deletions
--- a/client/src/frame_scheduler.c
+++ b/client/src/frame_scheduler.c
@@ -31,6 +31,7 @@

 #define FRAME_SCHEDULER_LEASE_MS        1000U
 #define FRAME_SCHEDULER_RENEW_NS        250000000ULL
+#define FRAME_SCHEDULER_FEEDBACK_NS     50000000ULL
 #define FRAME_SCHEDULER_TARGET_SLACK_NS 500000ULL
 #define FRAME_SCHEDULER_MIN_PERIOD_NS   2000000ULL
 #define FRAME_SCHEDULER_MAX_PERIOD_NS   1000000000ULL
@@ -40,14 +41,19 @@ static struct
 {
  LG_Lock lock;

-  bool     supported;
-  bool     active;
+  _Atomic(bool)     supported;
+  _Atomic(bool)     active;
  bool              controlPending;
-  uint32_t generation;
-  uint64_t period;
+  _Atomic(uint32_t) generation;
+  _Atomic(uint64_t) period;
  uint64_t          lastSend;
  uint64_t          lastCadence;

+  int64_t  phaseError;
+  uint32_t feedbackFrameSerial;
+  unsigned feedbackSamples;
+  bool     feedbackDirty;
+
  LG_TransportControlToken controlToken;

  uint64_t renderSamples[FRAME_SCHEDULER_SAMPLE_COUNT];
@@ -79,7 +85,11 @@ static uint64_t fallbackPeriod(void)
    return 0;

  qsort(samples, count, sizeof(*samples), compareU64);
-  return samples[count / 2];
+  const uint64_t period = samples[count / 2];
+  if (samples[count * 3 / 4] - samples[count / 4] > period / 20)
+    return 0;
+
+  return period;
 }

 static uint64_t presentationPeriod(void)
@@ -89,7 +99,7 @@ static uint64_t presentationPeriod(void)
      g_state.ds->getFramePeriod(&period))
    return period;

-  return g_state.jitRender ? fallbackPeriod() : 0;
+  return fallbackPeriod();
 }

 static bool controlReady(void)
@@ -112,6 +122,13 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags,
  if (!controlReady())
    return false;

+  int64_t  phaseError;
+  uint32_t feedbackFrameSerial;
+  LG_LOCK(l_frameScheduler.lock);
+  phaseError          = l_frameScheduler.phaseError;
+  feedbackFrameSerial = l_frameScheduler.feedbackFrameSerial;
+  LG_UNLOCK(l_frameScheduler.lock);
+
  const LG_TransportControl control = {
    .type = LG_TRANSPORT_CONTROL_FRAME_SCHEDULE,
    .frameSchedule = {
@@ -119,6 +136,8 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags,
      .flags               = flags,
      .period              = period,
      .targetSlack         = FRAME_SCHEDULER_TARGET_SLACK_NS,
+      .phaseError          = phaseError,
+      .feedbackFrameSerial = feedbackFrameSerial,
      .lease               = FRAME_SCHEDULER_LEASE_MS,
    },
  };
@@ -134,6 +153,10 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags,
  }

  l_frameScheduler.controlPending = true;
+  LG_LOCK(l_frameScheduler.lock);
+  if (l_frameScheduler.feedbackFrameSerial == feedbackFrameSerial)
+    l_frameScheduler.feedbackDirty = false;
+  LG_UNLOCK(l_frameScheduler.lock);
  return true;
 }

@@ -163,6 +186,10 @@ void frameScheduler_start(LG_TransportFeatureFlags features)
  l_frameScheduler.renderSampleIndex   = 0;
  l_frameScheduler.renderSampleCount   = 0;
  l_frameScheduler.lastRender          = 0;
+  l_frameScheduler.phaseError          = 0;
+  l_frameScheduler.feedbackFrameSerial = 0;
+  l_frameScheduler.feedbackSamples     = 0;
+  l_frameScheduler.feedbackDirty       = false;
  LG_UNLOCK(l_frameScheduler.lock);
 }

@@ -213,9 +240,16 @@ void frameScheduler_update(void)
    l_frameScheduler.period =
      (l_frameScheduler.period * 7 + period) / 8;

-  if (l_frameScheduler.active && !reset &&
-      now - l_frameScheduler.lastSend < FRAME_SCHEDULER_RENEW_NS)
+  if (l_frameScheduler.active && !reset)
+  {
+    LG_LOCK(l_frameScheduler.lock);
+    const bool feedbackDirty = l_frameScheduler.feedbackDirty;
+    LG_UNLOCK(l_frameScheduler.lock);
+    const uint64_t interval = feedbackDirty ?
+      FRAME_SCHEDULER_FEEDBACK_NS : FRAME_SCHEDULER_RENEW_NS;
+    if (now - l_frameScheduler.lastSend < interval)
      return;
+  }

  LG_TransportFrameScheduleFlags flags =
    LG_TRANSPORT_FRAME_SCHEDULE_ACTIVE;
@@ -231,9 +265,6 @@ void frameScheduler_update(void)

 void frameScheduler_observeRender(uint64_t timestamp)
 {
-  if (!g_state.jitRender)
-    return;
-
  LG_LOCK(l_frameScheduler.lock);
  if (l_frameScheduler.lastRender &&
      timestamp > l_frameScheduler.lastRender)
@@ -249,3 +280,40 @@ void frameScheduler_observeRender(uint64_t timestamp)
  l_frameScheduler.lastRender = timestamp;
  LG_UNLOCK(l_frameScheduler.lock);
 }
+
+void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation,
+    uint64_t queueStart, uint64_t prepareStart)
+{
+  if (!generation || !queueStart || prepareStart < queueStart)
+    return;
+
+  const uint64_t slack = prepareStart - queueStart;
+  LG_LOCK(l_frameScheduler.lock);
+  if (!l_frameScheduler.supported || !l_frameScheduler.active ||
+      generation != l_frameScheduler.generation)
+  {
+    LG_UNLOCK(l_frameScheduler.lock);
+    return;
+  }
+
+  int64_t error = slack > FRAME_SCHEDULER_TARGET_SLACK_NS ?
+    (int64_t)(slack - FRAME_SCHEDULER_TARGET_SLACK_NS) :
+    -(int64_t)(FRAME_SCHEDULER_TARGET_SLACK_NS - slack);
+  const int64_t limit = (int64_t)(l_frameScheduler.period / 2);
+  if (error > limit)
+    error = limit;
+  else if (error < -limit)
+    error = -limit;
+
+  if (!l_frameScheduler.feedbackSamples)
+    l_frameScheduler.phaseError = error;
+  else
+    l_frameScheduler.phaseError =
+      (l_frameScheduler.phaseError * 7 + error) / 8;
+  if (l_frameScheduler.feedbackSamples < 32)
+    ++l_frameScheduler.feedbackSamples;
+
+  l_frameScheduler.feedbackFrameSerial = (uint32_t)frameSerial;
+  l_frameScheduler.feedbackDirty       = true;
+  LG_UNLOCK(l_frameScheduler.lock);
+}
--- a/client/src/frame_scheduler.h
+++ b/client/src/frame_scheduler.h
@@ -31,5 +31,7 @@ void frameScheduler_start(LG_TransportFeatureFlags features);
 void frameScheduler_stop(void);
 void frameScheduler_update(void);
 void frameScheduler_observeRender(uint64_t timestamp);
+void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation,
+    uint64_t queueStart, uint64_t prepareStart);

 #endif
--- a/client/src/main.c
+++ b/client/src/main.c
@@ -212,6 +212,8 @@ enum FrameTimingReady
 struct FrameTimingRecord
 {
  LG_RendererFrameToken token;
+  uint64_t              frameSerial;
+  uint32_t              scheduleGeneration;
  unsigned              readyMask;
  bool                  producerValid;

@@ -323,7 +325,8 @@ static void frameTimingCancel(LG_RendererFrameToken token)
  });
 }

-static void frameTimingQueue(LG_RendererFrameToken token, uint64_t importTime,
+static void frameTimingQueue(LG_RendererFrameToken token, uint64_t frameSerial,
+    uint32_t scheduleGeneration, uint64_t importTime,
    uint64_t importWaitTime, uint64_t dispatchStart, uint64_t queueStart)
 {
  INTERLOCKED_SECTION(l_frameTiming.lock, {
@@ -337,6 +340,8 @@ static void frameTimingQueue(LG_RendererFrameToken token, uint64_t importTime,
      record->importWaitTime     = importWaitTime;
      record->dispatchTime       = elapsed > accounted ? elapsed - accounted : 0;
      record->queueStart         = queueStart;
+      record->frameSerial        = frameSerial;
+      record->scheduleGeneration = scheduleGeneration;
      if (record->timestamp < queueStart)
        record->timestamp = queueStart;
    }
@@ -382,7 +387,11 @@ static void frameTimingFinishFrame(LG_RendererFrameToken token,
 static void frameTimingFinishRender(const LG_RendererFrameTiming * timing,
    uint64_t prepareStart, uint64_t prepareTime, uint64_t timestamp)
 {
-  INTERLOCKED_SECTION(l_frameTiming.lock, {
+  uint64_t feedbackFrameSerial = 0;
+  uint64_t feedbackQueueStart  = 0;
+  uint32_t feedbackGeneration  = 0;
+
+  LG_LOCK(l_frameTiming.lock);
  if (l_frameTiming.retireToken <= timing->frameToken)
    l_frameTiming.retireToken = timing->frameToken + 1;

@@ -400,6 +409,10 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing,
    record->swapTime    = timing->swapTime;
    record->readyMask  |= FRAME_TIMING_RENDER_READY;

+    feedbackFrameSerial = record->frameSerial;
+    feedbackGeneration  = record->scheduleGeneration;
+    feedbackQueueStart  = record->queueStart;
+
    if (unlikely(
        l_frameTiming.publishCount == FRAME_TIMING_RECORD_COUNT))
    {
@@ -419,7 +432,10 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing,
      ++l_frameTiming.publishCount;
    }
  }
-  });
+  LG_UNLOCK(l_frameTiming.lock);
+
+  frameScheduler_feedback(feedbackFrameSerial, feedbackGeneration,
+      feedbackQueueStart, prepareStart);
 }

 static void frameTimingPublishReady(void)
@@ -1083,8 +1099,9 @@ int main_frameThread(void * unused)
    atomic_store_explicit(&l_testFrameSerial, frame.serial,
        memory_order_release);
 #endif
-    frameTimingQueue(frameToken, g_state.frameImportTime,
-        g_state.frameImportWaitTime, dispatchStart, queueStart);
+    frameTimingQueue(frameToken, frame.serial, frame.scheduleGeneration,
+        g_state.frameImportTime, g_state.frameImportWaitTime,
+        dispatchStart, queueStart);

    if (g_state.jitRender)
    {
--- a/common/include/common/KVMFR.h
+++ b/common/include/common/KVMFR.h
@@ -291,11 +291,12 @@ typedef struct KVMFRFrameSchedule
  uint32_t                clientID;
  uint32_t                generation;
  KVMFRFrameScheduleFlags flags;
-  uint64_t                period;
-  uint64_t                targetSlack;
-  int64_t                 phaseError;
+  uint64_t                period;       // requested presentation period (ns)
+  uint64_t                targetSlack;  // desired ready-to-render lead (ns)
+  // Positive when the frame arrived early, negative when it arrived late.
+  int64_t                 phaseError;   // ready-to-render phase error (ns)
  uint32_t                feedbackFrameSerial;
-  uint32_t                lease;
+  uint32_t                lease;        // lease duration (ms)
  uint8_t                 reserved[16];
 }
 KVMFRFrameSchedule;
--- a/idd/LGIdd/CFrameScheduler.cpp
+++ b/idd/LGIdd/CFrameScheduler.cpp
@@ -111,6 +111,8 @@ void CFrameScheduler::ElectOwner(uint64_t now)
    m_nextDeadline = m_scheduling ? now + m_schedule.period : 0;
    m_forceNext    = m_scheduling;

+    m_lastPublishedFrameSerial = 0;
+
    if (m_scheduling)
      DEBUG_INFO("Frame timing owner %u generation %u at %.3f Hz",
        m_schedule.clientID, m_schedule.generation,
@@ -136,6 +138,8 @@ void CFrameScheduler::Reset()
  m_nextDeadline   = 0;
  m_arrivalSamples = 0;
  m_timingSamples  = 0;
+
+  m_lastPublishedFrameSerial = 0;
  ReleaseSRWLockExclusive(&m_lock);
 }

@@ -204,22 +208,61 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule,
      schedule.period < MIN_PERIOD_NS ||
      schedule.period > MAX_PERIOD_NS ||
      schedule.targetSlack >= schedule.period ||
+      schedule.phaseError > static_cast<int64_t>(schedule.period) ||
+      schedule.phaseError < -static_cast<int64_t>(schedule.period) ||
      schedule.lease < MIN_LEASE_MS || schedule.lease > MAX_LEASE_MS)
  {
    ReleaseSRWLockExclusive(&m_lock);
    return false;
  }

+  if (client->generation != schedule.generation)
+    client->lastFeedbackFrameSerial = 0;
  client->generation  = schedule.generation;
  client->period      = schedule.period;
  client->targetSlack = schedule.targetSlack;
  client->expiry      = now + static_cast<uint64_t>(schedule.lease) * 1000000;
  client->active      = true;
+  if (schedule.flags & KVMFR_FRAME_SCHEDULE_IMMEDIATE)
+    m_forceNext = true;
  ElectOwner(now);
+  ApplyFeedback(*client, schedule);
  ReleaseSRWLockExclusive(&m_lock);
  return true;
 }

+void CFrameScheduler::ApplyFeedback(Client& client,
+  const KVMFRFrameSchedule& schedule)
+{
+  if (!m_scheduling || client.clientID != m_schedule.clientID ||
+      schedule.generation != m_schedule.generation ||
+      !schedule.feedbackFrameSerial ||
+      (client.lastFeedbackFrameSerial &&
+        static_cast<int32_t>(schedule.feedbackFrameSerial -
+          client.lastFeedbackFrameSerial) <= 0) ||
+      !m_lastPublishedFrameSerial ||
+      static_cast<int32_t>(schedule.feedbackFrameSerial -
+        m_lastPublishedFrameSerial) > 0)
+    return;
+
+  int64_t correction = schedule.phaseError / 4;
+  const int64_t limit = static_cast<int64_t>(m_schedule.period / 4);
+  if (correction > limit)
+    correction = limit;
+  else if (correction < -limit)
+    correction = -limit;
+
+  if (correction >= 0)
+    m_nextDeadline += static_cast<uint64_t>(correction);
+  else
+  {
+    const uint64_t advance = static_cast<uint64_t>(-correction);
+    m_nextDeadline = m_nextDeadline > advance ?
+      m_nextDeadline - advance : 0;
+  }
+  client.lastFeedbackFrameSerial = schedule.feedbackFrameSerial;
+}
+
 bool CFrameScheduler::GetSchedule(Schedule& schedule) const
 {
  AcquireSRWLockShared(&m_lock);
@@ -290,12 +333,14 @@ bool CFrameScheduler::SelectFrame(uint64_t now, bool force,
  return process;
 }

-void CFrameScheduler::FramePublished(uint32_t generation, uint64_t now)
+void CFrameScheduler::FramePublished(uint32_t generation,
+  uint32_t frameSerial, uint64_t now)
 {
  AcquireSRWLockExclusive(&m_lock);
  if (m_scheduling && generation == m_schedule.generation)
  {
    m_forceNext = false;
+    m_lastPublishedFrameSerial = frameSerial;
    do
      m_nextDeadline += m_schedule.period;
    while (m_nextDeadline <= now);
--- a/idd/LGIdd/CFrameScheduler.h
+++ b/idd/LGIdd/CFrameScheduler.h
@@ -48,6 +48,7 @@ private:
    uint64_t period;
    uint64_t targetSlack;
    uint64_t expiry;
+    uint32_t lastFeedbackFrameSerial;
    bool     subscribed;
    bool     active;
  };
@@ -66,8 +67,11 @@ private:
  unsigned m_arrivalSamples = 0;
  unsigned m_timingSamples  = 0;

+  uint32_t m_lastPublishedFrameSerial = 0;
+
  Client * FindClient(uint32_t clientID);
  void ElectOwner(uint64_t now);
+  void ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule);

 public:
  static uint64_t Nanotime();
@@ -79,6 +83,7 @@ public:
  bool GetSchedule(Schedule& schedule) const;
  void ObserveFrame(uint64_t now);
  bool SelectFrame(uint64_t now, bool force, uint32_t& generation);
-  void FramePublished(uint32_t generation, uint64_t now);
+  void FramePublished(uint32_t generation, uint32_t frameSerial,
+    uint64_t now);
  void RecordFrameTiming(uint64_t duration);
 };
--- a/idd/LGIdd/CIndirectDeviceContext.cpp
+++ b/idd/LGIdd/CIndirectDeviceContext.cpp
@@ -945,7 +945,8 @@ bool CIndirectDeviceContext::InitializeLGMP()
    kvmfr.version  = KVMFR_VERSION;
    kvmfr.features =
      KVMFR_FEATURE_SETCURSORPOS |
-      KVMFR_FEATURE_WINDOWSIZE;
+      KVMFR_FEATURE_WINDOWSIZE   |
+      KVMFR_FEATURE_FRAME_SCHEDULE;
    strncpy_s(kvmfr.hostver, LG_VERSION_STR, sizeof(kvmfr.hostver) - 1);
    ss.write(reinterpret_cast<const char *>(&kvmfr), sizeof(kvmfr));
  }
@@ -1523,7 +1524,8 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex,

  if (status == LGMP_OK)
    m_frameScheduler.FramePublished(
-      scheduleGeneration, CFrameScheduler::Nanotime());
+      scheduleGeneration, m_frame[frameIndex]->frameSerial,
+      CFrameScheduler::Nanotime());

  if (status != LGMP_OK)
  {