[client] audio: minimize LGMP playback latency

Scope the legacy latency offset to classic SPICE and use the LGMP poll bound as its fixed arrival reserve. Keep learned jitter additive. Select provider clock feedback only when the current remote endpoint advertises it, avoiding a failed startup attempt on broker v1.
2026-11-12 14:57:20 +00:00 · 2026-09-02 04:39:23 +10:00
parent 2fa376b31b
commit 4c395bb488
8 changed files with 228 additions and 28 deletions
--- a/client/include/interface/audio.h
+++ b/client/include/interface/audio.h
@@ -131,10 +131,24 @@ typedef struct LG_AudioEventOps
 }
 LG_AudioEventOps;

+typedef enum LG_AudioPlaybackFlag
+{
+  /* Apply audio:latencyOffset when local playback rate control is active.
+   * This preserves the legacy safety margin required by classic SPICE. */
+  LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET = 1U << 0,
+}
+LG_AudioPlaybackFlag;
+
 typedef struct LG_AudioOps
 {
  const char * name;

+  /* A zero arrival reserve selects the client's default. Providers with a
+   * tighter, measured delivery bound may request a smaller value. Runtime
+   * delivery jitter is learned and added independently. */
+  uint32_t playbackFlags;
+  uint32_t playbackArrivalReserveUs;
+
  /* Registration must synchronously report the current status after releasing
   * any backend locks. Passing NULL unregisters the listener. Providers with
   * no status listener are assumed to remain available while their transport
@@ -169,6 +183,11 @@ typedef struct LG_AudioOps
   * restart playback with local rate control. */
  bool (*clockFeedback)(void * opaque, uint32_t generation,
      const LG_AudioClock * playbackClock, double targetRate);
+
+  /* Optional nonblocking capability check used before selecting provider
+   * rate control for a new stream. If omitted, clockFeedback is assumed to
+   * be available. */
+  bool (*clockFeedbackAvailable)(void * opaque);
 }
 LG_AudioOps;

--- a/client/src/audio.c
+++ b/client/src/audio.c
@@ -52,6 +52,8 @@
 #define PLAYBACK_MAX_RATE_SLEW_PER_SEC 0.005
 #define PLAYBACK_MAX_JITTER_SEC 0.1
 #define PLAYBACK_JITTER_DECAY_SEC 10.0
+#define PLAYBACK_DEFAULT_ARRIVAL_RESERVE_US 1000U
+#define PLAYBACK_MAX_ARRIVAL_RESERVE_US 100000U
 #define AUDIO_START_RETRY_MIN_NS INT64_C(5000000)
 #define AUDIO_START_RETRY_MAX_NS INT64_C(250000000)
 #define AUDIO_RETRY_RESET_NS INT64_C(1000000000)
@@ -105,6 +107,13 @@ typedef enum
 }
 PlaybackRateControl;

+typedef struct
+{
+  uint32_t flags;
+  uint32_t arrivalReserveUs;
+}
+PlaybackPolicy;
+
 typedef enum
 {
  PLAYBACK_DIAGNOSTIC_REGISTER_GRAPH = 1U << 0,
@@ -309,6 +318,7 @@ typedef struct
    LG_AudioFormat       requestedFormat;
    bool                 requestedFormatValid;
    bool                 requestedProviderRateControl;
+    PlaybackPolicy       requestedPolicy;
    bool                 forceSoftwareResampler;
    bool                 startPending;
    bool                 startInProgress;
@@ -339,6 +349,8 @@ typedef struct
    int64_t              startupPacketPeriod;
    PlaybackRateControl rateControl;
    bool                lastProviderRateControl;
+    PlaybackPolicy      policy;
+    PlaybackPolicy      lastPolicy;
    _Atomic(double) backendResampleRatio;
    atomic_bool     backendResamplerFailed;
    RingBuffer            buffer;
@@ -418,6 +430,48 @@ AudioState;

 static AudioState audio = { 0 };

+static PlaybackPolicy playbackPolicyFromOps(const LG_AudioOps * ops)
+{
+  return (PlaybackPolicy)
+  {
+    .flags            = ops->playbackFlags,
+    .arrivalReserveUs = ops->playbackArrivalReserveUs ?
+      ops->playbackArrivalReserveUs :
+      PLAYBACK_DEFAULT_ARRIVAL_RESERVE_US,
+  };
+}
+
+static bool playbackPolicyEqual(
+    const PlaybackPolicy * a, const PlaybackPolicy * b)
+{
+  return a->flags == b->flags &&
+    a->arrivalReserveUs == b->arrivalReserveUs;
+}
+
+static bool playbackProviderRateControlAvailable(
+    const LG_AudioOps * ops, void * opaque)
+{
+  return ops->clockFeedback &&
+    (!ops->clockFeedbackAvailable ||
+      ops->clockFeedbackAvailable(opaque));
+}
+
+static double playbackArrivalReserveFrames(double arrivalJitterSec,
+    int sampleRate, uint32_t fixedReserveUs)
+{
+  return (arrivalJitterSec + fixedReserveUs * 1.0e-6) * sampleRate;
+}
+
+static double playbackComputeLatencyOffsetFrames(bool providerRateControl,
+    const PlaybackPolicy * policy, int latencyOffsetMs, int sampleRate)
+{
+  if (providerRateControl || !(policy->flags &
+        LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET))
+    return 0.0;
+
+  return max(latencyOffsetMs, 0) * sampleRate / 1000.0;
+}
+
 static size_t audioSampleSize(LG_AudioSampleFormat format)
 {
  switch (format)
@@ -1420,11 +1474,12 @@ static bool playbackSetupDevice(const LG_AudioFormat * format,

 static bool playbackResume(const LG_AudioFormat * format,
    const LG_AudioClock * sourceClock, bool providerRateControl,
-    bool forceSoftwareResampler)
+    bool forceSoftwareResampler, const PlaybackPolicy * policy)
 {
  if (forceSoftwareResampler ||
      !audio.playback.lastFormatValid ||
      audio.playback.lastProviderRateControl != providerRateControl ||
+      !playbackPolicyEqual(&audio.playback.lastPolicy, policy) ||
      !audioFormatEqual(format, &audio.playback.lastFormat))
    return false;

@@ -1442,7 +1497,7 @@ static bool playbackResume(const LG_AudioFormat * format,

 static bool playbackStart(const LG_AudioFormat * format,
    const LG_AudioClock * sourceClock, bool providerRateControl,
-    bool forceSoftwareResampler)
+    bool forceSoftwareResampler, const PlaybackPolicy * policy)
 {
  if (!audio.audioDev)
    return false;
@@ -1459,7 +1514,7 @@ static bool playbackStart(const LG_AudioFormat * format,
  const int sampleRate = format->sampleRate;

  if (playbackResume(format, sourceClock, providerRateControl,
-        forceSoftwareResampler))
+        forceSoftwareResampler, policy))
    return true;

  const StreamState state = playbackGetState();
@@ -1471,6 +1526,8 @@ static bool playbackStart(const LG_AudioFormat * format,
  audio.playback.lastFormat              = *format;
  audio.playback.lastFormatValid         = true;
  audio.playback.lastProviderRateControl = providerRateControl;
+  audio.playback.policy                  = *policy;
+  audio.playback.lastPolicy              = *policy;

  audio.playback.channels   = channels;
  audio.playback.sampleRate = sampleRate;
@@ -1804,6 +1861,7 @@ static void playbackProcessStart(void)
  uint32_t       generation;
  bool           providerRateControl;
  bool           forceSoftwareResampler;
+  PlaybackPolicy policy;

  LG_LOCK(audio.playback.sourceLock);
  if (!audio.playback.startPending ||
@@ -1825,11 +1883,12 @@ static void playbackProcessStart(void)
    audio.playback.requestedProviderRateControl;
  forceSoftwareResampler           =
    audio.playback.forceSoftwareResampler;
+  policy                           = audio.playback.requestedPolicy;
  LG_UNLOCK(audio.playback.sourceLock);

  LG_LOCK(audio.playback.deviceLock);
  const bool started = playbackStart(&format, NULL,
-      providerRateControl, forceSoftwareResampler);
+      providerRateControl, forceSoftwareResampler, &policy);
  LG_UNLOCK(audio.playback.deviceLock);

  bool stopStale = false;
@@ -1843,7 +1902,8 @@ static void playbackProcessStart(void)
    providerRateControl ==
      audio.playback.requestedProviderRateControl &&
    forceSoftwareResampler ==
-      audio.playback.forceSoftwareResampler;
+      audio.playback.forceSoftwareResampler &&
+    playbackPolicyEqual(&policy, &audio.playback.requestedPolicy);

  audio.playback.startInProgress = false;
  if (started && current)
@@ -2552,15 +2612,14 @@ static PlaybackDataResult playbackData(const void * data, size_t frameCount,
    max((double)maxPeriodFrames,
        sourceData->sourcePacketDurationSec * audio.playback.sampleRate);
  /* The device period, delivery jitter, packet phase, and resampler delay
-   * define the minimum viable latency. Provider feedback directly controls
-   * the source rate, so latencyOffset only applies to local rate control. */
-  const double latencyOffsetFrames          = providerRateControl ? 0.0 :
-    max(g_params.audioLatencyOffset, 0) *
-      audio.playback.sampleRate / 1000.0;
-  const double arrivalJitterFrames          =
-    sourceData->arrivalJitterSec * audio.playback.sampleRate;
-  const double arrivalReserveFrames         =
-    arrivalJitterFrames + 0.001 * audio.playback.sampleRate;
+   * define the minimum viable latency. latencyOffset is a compatibility
+   * margin used only by providers which explicitly request it. */
+  const double latencyOffsetFrames          = playbackComputeLatencyOffsetFrames(
+      providerRateControl, &audio.playback.policy,
+      g_params.audioLatencyOffset, audio.playback.sampleRate);
+  const double arrivalReserveFrames         = playbackArrivalReserveFrames(
+      sourceData->arrivalJitterSec, audio.playback.sampleRate,
+      audio.playback.policy.arrivalReserveUs);
  const double minimumLowWaterReserveFrames =
    maxPeriodFrames * 0.1 + arrivalReserveFrames;
  const double minimumLowWaterFrames        =
@@ -3812,10 +3871,12 @@ static void eventPlaybackStart(void * opaque, uint32_t generation,
  if (!eventBegin(binding, &active))
    return;

-  const bool providerRateControl =
-    active.ops->clockFeedback &&
+  const bool           providerRateControl =
+    playbackProviderRateControlAvailable(active.ops, active.opaque) &&
    audio.feedback.wakeInitialized && audio.feedback.thread;
-  bool       wake                = false;
+  const PlaybackPolicy policy              =
+    playbackPolicyFromOps(active.ops);
+  bool                 wake                = false;

  LG_LOCK(audio.playback.sourceLock);
  ++audio.playback.requestSerial;
@@ -3823,6 +3884,7 @@ static void eventPlaybackStart(void * opaque, uint32_t generation,
  audio.playback.requestedFormatValid         =
    generation && audioFormatValid(format);
  audio.playback.requestedProviderRateControl = providerRateControl;
+  audio.playback.requestedPolicy              = policy;
  audio.playback.forceSoftwareResampler       = false;
  audio.playback.startPending                 = false;
  audio.playback.startFailures                = 0;
@@ -3840,7 +3902,7 @@ static void eventPlaybackStart(void * opaque, uint32_t generation,
    {
      LG_LOCK(audio.playback.deviceLock);
      resumed = playbackResume(format, NULL, providerRateControl,
-          false);
+          false, &policy);
      if (resumed)
      {
        if (atomic_load_explicit(&audio.playback.activeAttemptSerial,
@@ -3906,6 +3968,7 @@ static void eventPlaybackStop(void * opaque, uint32_t generation)
    audio.playback.requestedGeneration          = 0;
    audio.playback.requestedFormatValid         = false;
    audio.playback.requestedProviderRateControl = false;
+    audio.playback.requestedPolicy              = (PlaybackPolicy) { 0 };
    audio.playback.forceSoftwareResampler       = false;
    audio.playback.startPending                 = false;
    audio.playback.startFailures                = 0;
@@ -4094,7 +4157,14 @@ static const LG_AudioEventOps eventOps =

 static bool validOps(const LG_AudioOps * ops)
 {
-  return ops && ops->name && ops->attach && ops->detach;
+  const uint32_t playbackFlagMask =
+    LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET;
+  return ops && ops->name &&
+    !(ops->playbackFlags & ~playbackFlagMask) &&
+    ops->playbackArrivalReserveUs <=
+      PLAYBACK_MAX_ARRIVAL_RESERVE_US &&
+    (!ops->clockFeedbackAvailable || ops->clockFeedback) &&
+    ops->attach && ops->detach;
 }

 static AudioBinding makeBinding(const LG_AudioOps * ops, void * opaque)
@@ -4139,6 +4209,7 @@ static void stopStreams(void)
  audio.playback.requestedGeneration          = 0;
  audio.playback.requestedFormatValid         = false;
  audio.playback.requestedProviderRateControl = false;
+  audio.playback.requestedPolicy              = (PlaybackPolicy) { 0 };
  audio.playback.forceSoftwareResampler       = false;
  audio.playback.startPending                 = false;
  audio.playback.startFailures                = 0;
--- a/client/src/config.c
+++ b/client/src/config.c
@@ -517,7 +517,7 @@ static struct Option options[] =
  {
    .module         = "audio",
    .name           = "latencyOffset",
-    .description    = "Latency offset added to the calculated minimum in milliseconds",
+    .description    = "Extra classic SPICE playback latency in milliseconds",
    .type           = OPTION_TYPE_INT,
    .value.x_int    = 6
  },
--- a/client/tests/CMakeLists.txt
+++ b/client/tests/CMakeLists.txt
@@ -853,6 +853,7 @@ if(ENABLE_AUDIO)
    recovery
    stable-underrun
    resume
+    latency-policy
    consent
    quiesce
  )
--- a/client/tests/audio_test.c
+++ b/client/tests/audio_test.c
@@ -660,7 +660,8 @@ static void testPlaybackRecovery(void)
  startAudio();

  const LG_AudioFormat format = makeFormat(LG_AUDIO_FMT_S16_LE);
-  CHECK(playbackStart(&format, NULL, false, false));
+  const PlaybackPolicy policy = playbackPolicyFromOps(&ops);
+  CHECK(playbackStart(&format, NULL, false, false, &policy));
  CHECK(audio.playback.rateControl == PLAYBACK_RATE_BACKEND);
  playbackSetState(STREAM_STATE_RUN);

@@ -708,7 +709,8 @@ static void testPlaybackStableUnderrun(void)
  startAudio();

  const LG_AudioFormat format = makeFormat(LG_AUDIO_FMT_S16_LE);
-  CHECK(playbackStart(&format, NULL, false, false));
+  const PlaybackPolicy policy = playbackPolicyFromOps(&ops);
+  CHECK(playbackStart(&format, NULL, false, false, &policy));
  CHECK(audio.playback.rateControl == PLAYBACK_RATE_BACKEND);
  playbackSetState(STREAM_STATE_RUN);

@@ -892,6 +894,91 @@ static void testPlaybackResume(void)
  stopAudio();
 }

+static bool testClockFeedback(void * opaque, uint32_t generation,
+    const LG_AudioClock * clock, double targetRate)
+{
+  (void)opaque;
+  (void)generation;
+  (void)clock;
+  (void)targetRate;
+  return true;
+}
+
+static bool testClockFeedbackAvailable(void * opaque)
+{
+  return *(const bool *)opaque;
+}
+
+static void testLatencyPolicy(void)
+{
+  LG_AudioOps providerOps = ops;
+  PlaybackPolicy policy   = playbackPolicyFromOps(&providerOps);
+  CHECK(policy.flags == 0);
+  CHECK(policy.arrivalReserveUs ==
+      PLAYBACK_DEFAULT_ARRIVAL_RESERVE_US);
+  CHECK(playbackComputeLatencyOffsetFrames(
+        false, &policy, 6, 48000) == 0.0);
+
+  providerOps.playbackFlags =
+    LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET;
+  policy                    = playbackPolicyFromOps(&providerOps);
+  CHECK(playbackComputeLatencyOffsetFrames(
+        false, &policy, 6, 48000) == 288.0);
+  CHECK(playbackComputeLatencyOffsetFrames(
+        true, &policy, 6, 48000) == 0.0);
+
+  providerOps.playbackFlags             = 0;
+  providerOps.playbackArrivalReserveUs  = 250;
+  const PlaybackPolicy lowLatencyPolicy =
+    playbackPolicyFromOps(&providerOps);
+  CHECK(lowLatencyPolicy.arrivalReserveUs == 250);
+  CHECK(fabs(playbackArrivalReserveFrames(
+        0.0, 48000, lowLatencyPolicy.arrivalReserveUs) - 12.0) < 0.001);
+  CHECK(fabs(playbackArrivalReserveFrames(
+        0.010, 48000, lowLatencyPolicy.arrivalReserveUs) - 492.0) <
+      0.001);
+
+  CHECK(validOps(&providerOps));
+  providerOps.playbackFlags = 1U << 31;
+  CHECK(!validOps(&providerOps));
+  providerOps.playbackFlags            = 0;
+  providerOps.playbackArrivalReserveUs =
+    PLAYBACK_MAX_ARRIVAL_RESERVE_US + 1U;
+  CHECK(!validOps(&providerOps));
+
+  providerOps.playbackArrivalReserveUs = 0;
+  providerOps.clockFeedback            = testClockFeedback;
+  CHECK(playbackProviderRateControlAvailable(&providerOps, NULL));
+  providerOps.clockFeedbackAvailable   = testClockFeedbackAvailable;
+  bool feedbackAvailable               = false;
+  CHECK(!playbackProviderRateControlAvailable(
+        &providerOps, &feedbackAvailable));
+  feedbackAvailable = true;
+  CHECK(playbackProviderRateControlAvailable(
+        &providerOps, &feedbackAvailable));
+  providerOps.clockFeedback = NULL;
+  CHECK(!validOps(&providerOps));
+
+  reset();
+  startAudio();
+  const LG_AudioFormat format        = makeFormat(LG_AUDIO_FMT_S16_LE);
+  const PlaybackPolicy defaultPolicy = playbackPolicyFromOps(&ops);
+  CHECK(playbackStart(&format, NULL, false, false, &defaultPolicy));
+  CHECK(atomic_load(&b.setupN) == 1);
+
+  playbackSetState(STREAM_STATE_KEEP_ALIVE);
+  CHECK(playbackStart(&format, NULL, false, false, &defaultPolicy));
+  CHECK(atomic_load(&b.setupN) == 1);
+  CHECK(playbackGetState() == STREAM_STATE_RESUMING);
+
+  playbackSetState(STREAM_STATE_KEEP_ALIVE);
+  CHECK(playbackStart(
+        &format, NULL, false, false, &lowLatencyPolicy));
+  CHECK(atomic_load(&b.setupN) == 2);
+  CHECK(playbackGetState() == STREAM_STATE_SETUP_SOURCE);
+  stopAudio();
+}
+
 static void testConsent(void)
 {
  reset();
@@ -1022,6 +1109,7 @@ static const struct Test tests[] =
  { "recovery"       , testPlaybackRecovery        },
  { "stable-underrun", testPlaybackStableUnderrun  },
  { "resume"         , testPlaybackResume          },
+  { "latency-policy" , testLatencyPolicy           },
  { "consent"        , testConsent                 },
  { "quiesce"        , testQuiesce                 },
 };
--- a/client/tests/lgmp_audio_test.c
+++ b/client/tests/lgmp_audio_test.c
@@ -308,6 +308,10 @@ static bool fixtureInit(struct Fixture * fixture)
  CHECK(lgmpAudio_connect(fixture->audio, fixture->clientID));
  fixture->ops = lgmpAudio_getOps();
  CHECK(fixture->ops);
+  CHECK(fixture->ops->playbackFlags == 0);
+  CHECK(fixture->ops->playbackArrivalReserveUs == 250);
+  CHECK(fixture->ops->clockFeedbackAvailable);
+  CHECK(!fixture->ops->clockFeedbackAvailable(fixture->audio));
  fixture->endpointGeneration = UINT64_C(0x100000001);
  return true;
 }
@@ -930,6 +934,7 @@ static bool testDataPlane(void)
  };
  CHECK(!fixture.ops->clockFeedback(fixture.audio, playbackLocal,
        &playbackClock, 192000.0));
+  CHECK(!fixture.ops->clockFeedbackAvailable(fixture.audio));

  const uint64_t currentPlayWire =
    UINT64_C(0x300000000) + KVMFR_AUDIO_MAX_CHANNELS - 1;
@@ -946,6 +951,7 @@ static bool testDataPlane(void)
        captureWire, &captureFormat));
  CHECK(waitCount(&fixture, &fixture.trace.statusCount,
        clockStatusBefore + 1, WAIT_MS));
+  CHECK(fixture.ops->clockFeedbackAvailable(fixture.audio));
  CHECK(fixture.ops->clockFeedback(fixture.audio, playbackLocal,
        &playbackClock, 192000.0));
  CHECK(readC2HType(&fixture, KVMFR_AUDIO_MESSAGE_CLOCK_FEEDBACK,
--- a/client/transports/LGMP/audio.c
+++ b/client/transports/LGMP/audio.c
@@ -125,6 +125,7 @@ struct LGMPAudio
  atomic_bool       stop;
  atomic_bool       dataPlaneLost;
  atomic_bool       captureDiscontinuity;
+  atomic_bool       clockFeedbackAvailable;
  atomic_uint       statusInFlight;
  struct AudioCallbackWaitQueue statusWait;

@@ -526,6 +527,9 @@ static void markStatusNL(LGMPAudio * audio, bool available,
      oldGeneration != audio->endpointGeneration.local)
    audio->statusDirty = true;
  audio->statusAvailable = available;
+  atomic_store_explicit(&audio->clockFeedbackAvailable,
+      available && (audio->remoteStatus.capabilities &
+        KVMFR_AUDIO_CAP_CLOCK_FEEDBACK), memory_order_release);
 }

 static void publishStatus(LGMPAudio * audio)
@@ -1824,6 +1828,7 @@ bool lgmpAudio_create(PLGMPClient client, LGEvent * pointerWake,
  atomic_init(&audio->stop, false);
  atomic_init(&audio->dataPlaneLost, false);
  atomic_init(&audio->captureDiscontinuity, false);
+  atomic_init(&audio->clockFeedbackAvailable, false);
  atomic_init(&audio->statusInFlight, 0);
  atomic_init(&audio->statusWait.count, 0);
  *result = audio;
@@ -2322,14 +2327,23 @@ static bool clockFeedback(void * opaque, uint32_t generation,
  return result == WRITE_OK;
 }

+static bool clockFeedbackAvailable(void * opaque)
+{
+  LGMPAudio * audio = opaque;
+  return atomic_load_explicit(
+      &audio->clockFeedbackAvailable, memory_order_acquire);
+}
+
 static const LG_AudioOps l_audioOps =
 {
-  .name              = "LGMP",
-  .setStatusListener = setStatusListener,
-  .attach            = attach,
-  .detach            = detach,
-  .recordData        = recordData,
-  .clockFeedback     = clockFeedback,
+  .name                     = "LGMP",
+  .playbackArrivalReserveUs = AUDIO_ACTIVE_POLL_US,
+  .setStatusListener        = setStatusListener,
+  .attach                   = attach,
+  .detach                   = detach,
+  .recordData               = recordData,
+  .clockFeedback            = clockFeedback,
+  .clockFeedbackAvailable   = clockFeedbackAvailable,
 };

 const LG_AudioOps * lgmpAudio_getOps(void)
--- a/client/transports/SPICE/audio.c
+++ b/client/transports/SPICE/audio.c
@@ -547,6 +547,7 @@ static bool spiceRecordData(void * opaque, uint32_t generation,
 static const LG_AudioOps l_spiceAudioOps =
 {
  .name              = "SPICE",
+  .playbackFlags     = LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET,
  .setStatusListener = spiceSetStatusListener,
  .attach            = spiceAttach,
  .detach            = spiceDetach,