diff --git a/client/include/interface/audio.h b/client/include/interface/audio.h index 2b80ac6b..35cd9cc4 100644 --- a/client/include/interface/audio.h +++ b/client/include/interface/audio.h @@ -131,10 +131,24 @@ typedef struct LG_AudioEventOps } LG_AudioEventOps; +typedef enum LG_AudioPlaybackFlag +{ + /* Apply audio:latencyOffset when local playback rate control is active. + * This preserves the legacy safety margin required by classic SPICE. */ + LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET = 1U << 0, +} +LG_AudioPlaybackFlag; + typedef struct LG_AudioOps { const char * name; + /* A zero arrival reserve selects the client's default. Providers with a + * tighter, measured delivery bound may request a smaller value. Runtime + * delivery jitter is learned and added independently. */ + uint32_t playbackFlags; + uint32_t playbackArrivalReserveUs; + /* Registration must synchronously report the current status after releasing * any backend locks. Passing NULL unregisters the listener. Providers with * no status listener are assumed to remain available while their transport @@ -169,6 +183,11 @@ typedef struct LG_AudioOps * restart playback with local rate control. */ bool (*clockFeedback)(void * opaque, uint32_t generation, const LG_AudioClock * playbackClock, double targetRate); + + /* Optional nonblocking capability check used before selecting provider + * rate control for a new stream. If omitted, clockFeedback is assumed to + * be available. */ + bool (*clockFeedbackAvailable)(void * opaque); } LG_AudioOps; diff --git a/client/src/audio.c b/client/src/audio.c index 3eed0e83..a17277d0 100644 --- a/client/src/audio.c +++ b/client/src/audio.c @@ -52,6 +52,8 @@ #define PLAYBACK_MAX_RATE_SLEW_PER_SEC 0.005 #define PLAYBACK_MAX_JITTER_SEC 0.1 #define PLAYBACK_JITTER_DECAY_SEC 10.0 +#define PLAYBACK_DEFAULT_ARRIVAL_RESERVE_US 1000U +#define PLAYBACK_MAX_ARRIVAL_RESERVE_US 100000U #define AUDIO_START_RETRY_MIN_NS INT64_C(5000000) #define AUDIO_START_RETRY_MAX_NS INT64_C(250000000) #define AUDIO_RETRY_RESET_NS INT64_C(1000000000) @@ -105,6 +107,13 @@ typedef enum } PlaybackRateControl; +typedef struct +{ + uint32_t flags; + uint32_t arrivalReserveUs; +} +PlaybackPolicy; + typedef enum { PLAYBACK_DIAGNOSTIC_REGISTER_GRAPH = 1U << 0, @@ -309,6 +318,7 @@ typedef struct LG_AudioFormat requestedFormat; bool requestedFormatValid; bool requestedProviderRateControl; + PlaybackPolicy requestedPolicy; bool forceSoftwareResampler; bool startPending; bool startInProgress; @@ -339,6 +349,8 @@ typedef struct int64_t startupPacketPeriod; PlaybackRateControl rateControl; bool lastProviderRateControl; + PlaybackPolicy policy; + PlaybackPolicy lastPolicy; _Atomic(double) backendResampleRatio; atomic_bool backendResamplerFailed; RingBuffer buffer; @@ -418,6 +430,48 @@ AudioState; static AudioState audio = { 0 }; +static PlaybackPolicy playbackPolicyFromOps(const LG_AudioOps * ops) +{ + return (PlaybackPolicy) + { + .flags = ops->playbackFlags, + .arrivalReserveUs = ops->playbackArrivalReserveUs ? + ops->playbackArrivalReserveUs : + PLAYBACK_DEFAULT_ARRIVAL_RESERVE_US, + }; +} + +static bool playbackPolicyEqual( + const PlaybackPolicy * a, const PlaybackPolicy * b) +{ + return a->flags == b->flags && + a->arrivalReserveUs == b->arrivalReserveUs; +} + +static bool playbackProviderRateControlAvailable( + const LG_AudioOps * ops, void * opaque) +{ + return ops->clockFeedback && + (!ops->clockFeedbackAvailable || + ops->clockFeedbackAvailable(opaque)); +} + +static double playbackArrivalReserveFrames(double arrivalJitterSec, + int sampleRate, uint32_t fixedReserveUs) +{ + return (arrivalJitterSec + fixedReserveUs * 1.0e-6) * sampleRate; +} + +static double playbackComputeLatencyOffsetFrames(bool providerRateControl, + const PlaybackPolicy * policy, int latencyOffsetMs, int sampleRate) +{ + if (providerRateControl || !(policy->flags & + LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET)) + return 0.0; + + return max(latencyOffsetMs, 0) * sampleRate / 1000.0; +} + static size_t audioSampleSize(LG_AudioSampleFormat format) { switch (format) @@ -1420,11 +1474,12 @@ static bool playbackSetupDevice(const LG_AudioFormat * format, static bool playbackResume(const LG_AudioFormat * format, const LG_AudioClock * sourceClock, bool providerRateControl, - bool forceSoftwareResampler) + bool forceSoftwareResampler, const PlaybackPolicy * policy) { if (forceSoftwareResampler || !audio.playback.lastFormatValid || audio.playback.lastProviderRateControl != providerRateControl || + !playbackPolicyEqual(&audio.playback.lastPolicy, policy) || !audioFormatEqual(format, &audio.playback.lastFormat)) return false; @@ -1442,7 +1497,7 @@ static bool playbackResume(const LG_AudioFormat * format, static bool playbackStart(const LG_AudioFormat * format, const LG_AudioClock * sourceClock, bool providerRateControl, - bool forceSoftwareResampler) + bool forceSoftwareResampler, const PlaybackPolicy * policy) { if (!audio.audioDev) return false; @@ -1459,7 +1514,7 @@ static bool playbackStart(const LG_AudioFormat * format, const int sampleRate = format->sampleRate; if (playbackResume(format, sourceClock, providerRateControl, - forceSoftwareResampler)) + forceSoftwareResampler, policy)) return true; const StreamState state = playbackGetState(); @@ -1471,6 +1526,8 @@ static bool playbackStart(const LG_AudioFormat * format, audio.playback.lastFormat = *format; audio.playback.lastFormatValid = true; audio.playback.lastProviderRateControl = providerRateControl; + audio.playback.policy = *policy; + audio.playback.lastPolicy = *policy; audio.playback.channels = channels; audio.playback.sampleRate = sampleRate; @@ -1804,6 +1861,7 @@ static void playbackProcessStart(void) uint32_t generation; bool providerRateControl; bool forceSoftwareResampler; + PlaybackPolicy policy; LG_LOCK(audio.playback.sourceLock); if (!audio.playback.startPending || @@ -1825,11 +1883,12 @@ static void playbackProcessStart(void) audio.playback.requestedProviderRateControl; forceSoftwareResampler = audio.playback.forceSoftwareResampler; + policy = audio.playback.requestedPolicy; LG_UNLOCK(audio.playback.sourceLock); LG_LOCK(audio.playback.deviceLock); const bool started = playbackStart(&format, NULL, - providerRateControl, forceSoftwareResampler); + providerRateControl, forceSoftwareResampler, &policy); LG_UNLOCK(audio.playback.deviceLock); bool stopStale = false; @@ -1843,7 +1902,8 @@ static void playbackProcessStart(void) providerRateControl == audio.playback.requestedProviderRateControl && forceSoftwareResampler == - audio.playback.forceSoftwareResampler; + audio.playback.forceSoftwareResampler && + playbackPolicyEqual(&policy, &audio.playback.requestedPolicy); audio.playback.startInProgress = false; if (started && current) @@ -2552,15 +2612,14 @@ static PlaybackDataResult playbackData(const void * data, size_t frameCount, max((double)maxPeriodFrames, sourceData->sourcePacketDurationSec * audio.playback.sampleRate); /* The device period, delivery jitter, packet phase, and resampler delay - * define the minimum viable latency. Provider feedback directly controls - * the source rate, so latencyOffset only applies to local rate control. */ - const double latencyOffsetFrames = providerRateControl ? 0.0 : - max(g_params.audioLatencyOffset, 0) * - audio.playback.sampleRate / 1000.0; - const double arrivalJitterFrames = - sourceData->arrivalJitterSec * audio.playback.sampleRate; - const double arrivalReserveFrames = - arrivalJitterFrames + 0.001 * audio.playback.sampleRate; + * define the minimum viable latency. latencyOffset is a compatibility + * margin used only by providers which explicitly request it. */ + const double latencyOffsetFrames = playbackComputeLatencyOffsetFrames( + providerRateControl, &audio.playback.policy, + g_params.audioLatencyOffset, audio.playback.sampleRate); + const double arrivalReserveFrames = playbackArrivalReserveFrames( + sourceData->arrivalJitterSec, audio.playback.sampleRate, + audio.playback.policy.arrivalReserveUs); const double minimumLowWaterReserveFrames = maxPeriodFrames * 0.1 + arrivalReserveFrames; const double minimumLowWaterFrames = @@ -3812,10 +3871,12 @@ static void eventPlaybackStart(void * opaque, uint32_t generation, if (!eventBegin(binding, &active)) return; - const bool providerRateControl = - active.ops->clockFeedback && + const bool providerRateControl = + playbackProviderRateControlAvailable(active.ops, active.opaque) && audio.feedback.wakeInitialized && audio.feedback.thread; - bool wake = false; + const PlaybackPolicy policy = + playbackPolicyFromOps(active.ops); + bool wake = false; LG_LOCK(audio.playback.sourceLock); ++audio.playback.requestSerial; @@ -3823,6 +3884,7 @@ static void eventPlaybackStart(void * opaque, uint32_t generation, audio.playback.requestedFormatValid = generation && audioFormatValid(format); audio.playback.requestedProviderRateControl = providerRateControl; + audio.playback.requestedPolicy = policy; audio.playback.forceSoftwareResampler = false; audio.playback.startPending = false; audio.playback.startFailures = 0; @@ -3840,7 +3902,7 @@ static void eventPlaybackStart(void * opaque, uint32_t generation, { LG_LOCK(audio.playback.deviceLock); resumed = playbackResume(format, NULL, providerRateControl, - false); + false, &policy); if (resumed) { if (atomic_load_explicit(&audio.playback.activeAttemptSerial, @@ -3906,6 +3968,7 @@ static void eventPlaybackStop(void * opaque, uint32_t generation) audio.playback.requestedGeneration = 0; audio.playback.requestedFormatValid = false; audio.playback.requestedProviderRateControl = false; + audio.playback.requestedPolicy = (PlaybackPolicy) { 0 }; audio.playback.forceSoftwareResampler = false; audio.playback.startPending = false; audio.playback.startFailures = 0; @@ -4094,7 +4157,14 @@ static const LG_AudioEventOps eventOps = static bool validOps(const LG_AudioOps * ops) { - return ops && ops->name && ops->attach && ops->detach; + const uint32_t playbackFlagMask = + LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET; + return ops && ops->name && + !(ops->playbackFlags & ~playbackFlagMask) && + ops->playbackArrivalReserveUs <= + PLAYBACK_MAX_ARRIVAL_RESERVE_US && + (!ops->clockFeedbackAvailable || ops->clockFeedback) && + ops->attach && ops->detach; } static AudioBinding makeBinding(const LG_AudioOps * ops, void * opaque) @@ -4139,6 +4209,7 @@ static void stopStreams(void) audio.playback.requestedGeneration = 0; audio.playback.requestedFormatValid = false; audio.playback.requestedProviderRateControl = false; + audio.playback.requestedPolicy = (PlaybackPolicy) { 0 }; audio.playback.forceSoftwareResampler = false; audio.playback.startPending = false; audio.playback.startFailures = 0; diff --git a/client/src/config.c b/client/src/config.c index 4b43029f..f9eaaa80 100644 --- a/client/src/config.c +++ b/client/src/config.c @@ -517,7 +517,7 @@ static struct Option options[] = { .module = "audio", .name = "latencyOffset", - .description = "Latency offset added to the calculated minimum in milliseconds", + .description = "Extra classic SPICE playback latency in milliseconds", .type = OPTION_TYPE_INT, .value.x_int = 6 }, diff --git a/client/tests/CMakeLists.txt b/client/tests/CMakeLists.txt index 911f9d5c..826d95a0 100644 --- a/client/tests/CMakeLists.txt +++ b/client/tests/CMakeLists.txt @@ -853,6 +853,7 @@ if(ENABLE_AUDIO) recovery stable-underrun resume + latency-policy consent quiesce ) diff --git a/client/tests/audio_test.c b/client/tests/audio_test.c index 7489fd6f..08d93af8 100644 --- a/client/tests/audio_test.c +++ b/client/tests/audio_test.c @@ -660,7 +660,8 @@ static void testPlaybackRecovery(void) startAudio(); const LG_AudioFormat format = makeFormat(LG_AUDIO_FMT_S16_LE); - CHECK(playbackStart(&format, NULL, false, false)); + const PlaybackPolicy policy = playbackPolicyFromOps(&ops); + CHECK(playbackStart(&format, NULL, false, false, &policy)); CHECK(audio.playback.rateControl == PLAYBACK_RATE_BACKEND); playbackSetState(STREAM_STATE_RUN); @@ -708,7 +709,8 @@ static void testPlaybackStableUnderrun(void) startAudio(); const LG_AudioFormat format = makeFormat(LG_AUDIO_FMT_S16_LE); - CHECK(playbackStart(&format, NULL, false, false)); + const PlaybackPolicy policy = playbackPolicyFromOps(&ops); + CHECK(playbackStart(&format, NULL, false, false, &policy)); CHECK(audio.playback.rateControl == PLAYBACK_RATE_BACKEND); playbackSetState(STREAM_STATE_RUN); @@ -892,6 +894,91 @@ static void testPlaybackResume(void) stopAudio(); } +static bool testClockFeedback(void * opaque, uint32_t generation, + const LG_AudioClock * clock, double targetRate) +{ + (void)opaque; + (void)generation; + (void)clock; + (void)targetRate; + return true; +} + +static bool testClockFeedbackAvailable(void * opaque) +{ + return *(const bool *)opaque; +} + +static void testLatencyPolicy(void) +{ + LG_AudioOps providerOps = ops; + PlaybackPolicy policy = playbackPolicyFromOps(&providerOps); + CHECK(policy.flags == 0); + CHECK(policy.arrivalReserveUs == + PLAYBACK_DEFAULT_ARRIVAL_RESERVE_US); + CHECK(playbackComputeLatencyOffsetFrames( + false, &policy, 6, 48000) == 0.0); + + providerOps.playbackFlags = + LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET; + policy = playbackPolicyFromOps(&providerOps); + CHECK(playbackComputeLatencyOffsetFrames( + false, &policy, 6, 48000) == 288.0); + CHECK(playbackComputeLatencyOffsetFrames( + true, &policy, 6, 48000) == 0.0); + + providerOps.playbackFlags = 0; + providerOps.playbackArrivalReserveUs = 250; + const PlaybackPolicy lowLatencyPolicy = + playbackPolicyFromOps(&providerOps); + CHECK(lowLatencyPolicy.arrivalReserveUs == 250); + CHECK(fabs(playbackArrivalReserveFrames( + 0.0, 48000, lowLatencyPolicy.arrivalReserveUs) - 12.0) < 0.001); + CHECK(fabs(playbackArrivalReserveFrames( + 0.010, 48000, lowLatencyPolicy.arrivalReserveUs) - 492.0) < + 0.001); + + CHECK(validOps(&providerOps)); + providerOps.playbackFlags = 1U << 31; + CHECK(!validOps(&providerOps)); + providerOps.playbackFlags = 0; + providerOps.playbackArrivalReserveUs = + PLAYBACK_MAX_ARRIVAL_RESERVE_US + 1U; + CHECK(!validOps(&providerOps)); + + providerOps.playbackArrivalReserveUs = 0; + providerOps.clockFeedback = testClockFeedback; + CHECK(playbackProviderRateControlAvailable(&providerOps, NULL)); + providerOps.clockFeedbackAvailable = testClockFeedbackAvailable; + bool feedbackAvailable = false; + CHECK(!playbackProviderRateControlAvailable( + &providerOps, &feedbackAvailable)); + feedbackAvailable = true; + CHECK(playbackProviderRateControlAvailable( + &providerOps, &feedbackAvailable)); + providerOps.clockFeedback = NULL; + CHECK(!validOps(&providerOps)); + + reset(); + startAudio(); + const LG_AudioFormat format = makeFormat(LG_AUDIO_FMT_S16_LE); + const PlaybackPolicy defaultPolicy = playbackPolicyFromOps(&ops); + CHECK(playbackStart(&format, NULL, false, false, &defaultPolicy)); + CHECK(atomic_load(&b.setupN) == 1); + + playbackSetState(STREAM_STATE_KEEP_ALIVE); + CHECK(playbackStart(&format, NULL, false, false, &defaultPolicy)); + CHECK(atomic_load(&b.setupN) == 1); + CHECK(playbackGetState() == STREAM_STATE_RESUMING); + + playbackSetState(STREAM_STATE_KEEP_ALIVE); + CHECK(playbackStart( + &format, NULL, false, false, &lowLatencyPolicy)); + CHECK(atomic_load(&b.setupN) == 2); + CHECK(playbackGetState() == STREAM_STATE_SETUP_SOURCE); + stopAudio(); +} + static void testConsent(void) { reset(); @@ -1022,6 +1109,7 @@ static const struct Test tests[] = { "recovery" , testPlaybackRecovery }, { "stable-underrun", testPlaybackStableUnderrun }, { "resume" , testPlaybackResume }, + { "latency-policy" , testLatencyPolicy }, { "consent" , testConsent }, { "quiesce" , testQuiesce }, }; diff --git a/client/tests/lgmp_audio_test.c b/client/tests/lgmp_audio_test.c index 977ed137..0a1bbce6 100644 --- a/client/tests/lgmp_audio_test.c +++ b/client/tests/lgmp_audio_test.c @@ -308,6 +308,10 @@ static bool fixtureInit(struct Fixture * fixture) CHECK(lgmpAudio_connect(fixture->audio, fixture->clientID)); fixture->ops = lgmpAudio_getOps(); CHECK(fixture->ops); + CHECK(fixture->ops->playbackFlags == 0); + CHECK(fixture->ops->playbackArrivalReserveUs == 250); + CHECK(fixture->ops->clockFeedbackAvailable); + CHECK(!fixture->ops->clockFeedbackAvailable(fixture->audio)); fixture->endpointGeneration = UINT64_C(0x100000001); return true; } @@ -930,6 +934,7 @@ static bool testDataPlane(void) }; CHECK(!fixture.ops->clockFeedback(fixture.audio, playbackLocal, &playbackClock, 192000.0)); + CHECK(!fixture.ops->clockFeedbackAvailable(fixture.audio)); const uint64_t currentPlayWire = UINT64_C(0x300000000) + KVMFR_AUDIO_MAX_CHANNELS - 1; @@ -946,6 +951,7 @@ static bool testDataPlane(void) captureWire, &captureFormat)); CHECK(waitCount(&fixture, &fixture.trace.statusCount, clockStatusBefore + 1, WAIT_MS)); + CHECK(fixture.ops->clockFeedbackAvailable(fixture.audio)); CHECK(fixture.ops->clockFeedback(fixture.audio, playbackLocal, &playbackClock, 192000.0)); CHECK(readC2HType(&fixture, KVMFR_AUDIO_MESSAGE_CLOCK_FEEDBACK, diff --git a/client/transports/LGMP/audio.c b/client/transports/LGMP/audio.c index f81b555d..b71f6b21 100644 --- a/client/transports/LGMP/audio.c +++ b/client/transports/LGMP/audio.c @@ -125,6 +125,7 @@ struct LGMPAudio atomic_bool stop; atomic_bool dataPlaneLost; atomic_bool captureDiscontinuity; + atomic_bool clockFeedbackAvailable; atomic_uint statusInFlight; struct AudioCallbackWaitQueue statusWait; @@ -526,6 +527,9 @@ static void markStatusNL(LGMPAudio * audio, bool available, oldGeneration != audio->endpointGeneration.local) audio->statusDirty = true; audio->statusAvailable = available; + atomic_store_explicit(&audio->clockFeedbackAvailable, + available && (audio->remoteStatus.capabilities & + KVMFR_AUDIO_CAP_CLOCK_FEEDBACK), memory_order_release); } static void publishStatus(LGMPAudio * audio) @@ -1824,6 +1828,7 @@ bool lgmpAudio_create(PLGMPClient client, LGEvent * pointerWake, atomic_init(&audio->stop, false); atomic_init(&audio->dataPlaneLost, false); atomic_init(&audio->captureDiscontinuity, false); + atomic_init(&audio->clockFeedbackAvailable, false); atomic_init(&audio->statusInFlight, 0); atomic_init(&audio->statusWait.count, 0); *result = audio; @@ -2322,14 +2327,23 @@ static bool clockFeedback(void * opaque, uint32_t generation, return result == WRITE_OK; } +static bool clockFeedbackAvailable(void * opaque) +{ + LGMPAudio * audio = opaque; + return atomic_load_explicit( + &audio->clockFeedbackAvailable, memory_order_acquire); +} + static const LG_AudioOps l_audioOps = { - .name = "LGMP", - .setStatusListener = setStatusListener, - .attach = attach, - .detach = detach, - .recordData = recordData, - .clockFeedback = clockFeedback, + .name = "LGMP", + .playbackArrivalReserveUs = AUDIO_ACTIVE_POLL_US, + .setStatusListener = setStatusListener, + .attach = attach, + .detach = detach, + .recordData = recordData, + .clockFeedback = clockFeedback, + .clockFeedbackAvailable = clockFeedbackAvailable, }; const LG_AudioOps * lgmpAudio_getOps(void) diff --git a/client/transports/SPICE/audio.c b/client/transports/SPICE/audio.c index c5184ac3..df74f780 100644 --- a/client/transports/SPICE/audio.c +++ b/client/transports/SPICE/audio.c @@ -547,6 +547,7 @@ static bool spiceRecordData(void * opaque, uint32_t generation, static const LG_AudioOps l_spiceAudioOps = { .name = "SPICE", + .playbackFlags = LG_AUDIO_PLAYBACK_LEGACY_LATENCY_OFFSET, .setStatusListener = spiceSetStatusListener, .attach = spiceAttach, .detach = spiceDetach,