From 0f3c1db261ffe24a2e29ecd06d1b9d300b846690 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Mon, 10 Aug 2026 04:44:55 +1000 Subject: [PATCH] [client] usb audio: add asynchronous clock feedback Advertise an asynchronous UAC2 OUT endpoint with explicit feedback so Windows paces USB audio from the host playback clock. Drive feedback from measured backend consumption plus the existing buffer phase controller. Preserve the local and backend resampler paths for providers without active feedback. Keep audio diagnostics in the correct clock domains, report ring and backend latency separately, and avoid hot-path feedback wakeups. --- client/include/interface/audio.h | 13 +- client/src/audio.c | 286 +++++++++++++++++++++++-------- client/src/audio_usb.c | 24 ++- client/src/usb_audio.c | 227 ++++++++++++++++++------ client/src/usb_audio.h | 4 + 5 files changed, 420 insertions(+), 134 deletions(-) diff --git a/client/include/interface/audio.h b/client/include/interface/audio.h index 90025c86..9aae7aa8 100644 --- a/client/include/interface/audio.h +++ b/client/include/interface/audio.h @@ -156,12 +156,15 @@ typedef struct LG_AudioOps bool (*recordData)(void * opaque, uint32_t generation, const void * data, size_t frames, const LG_AudioClock * sourceClock); - /* Optional active synchronization feedback. This is called outside the - * realtime audio callback with the measured playback device clock. Its - * position uses the device's independent output-frame timeline and its time - * includes the backend's estimated presentation latency. */ + /* Optional active synchronization feedback. A provider implementing this + * owns playback rate control, so the client preserves its frame rate and + * does not activate a local resampler. This is called outside the realtime + * audio callback with the measured playback device clock. Its position uses + * the device's independent output-frame timeline and its time includes the + * backend's estimated presentation latency. targetRate is the source frame + * rate requested by the client's buffer controller. */ bool (*clockFeedback)(void * opaque, uint32_t generation, - const LG_AudioClock * playbackClock); + const LG_AudioClock * playbackClock, double targetRate); } LG_AudioOps; diff --git a/client/src/audio.c b/client/src/audio.c index f54c2ce0..2f9e0985 100644 --- a/client/src/audio.c +++ b/client/src/audio.c @@ -65,6 +65,7 @@ #define PLAYBACK_DEVICE_RATE_STABLE_SEC 2.0 #define PLAYBACK_DEVICE_RATE_STABLE_DELTA_PPM 50.0 #define PLAYBACK_DEVICE_RATE_MAX_ACQUIRE_SEC 20.0 +#define PLAYBACK_FEEDBACK_INTERVAL_NS INT64_C(1000000) typedef enum { @@ -78,6 +79,14 @@ typedef enum } StreamState; +typedef enum +{ + PLAYBACK_RATE_SOFTWARE, + PLAYBACK_RATE_BACKEND, + PLAYBACK_RATE_PROVIDER, +} +PlaybackRateControl; + #define STREAM_ACTIVE(state) \ (state == STREAM_STATE_RUN || \ state == STREAM_STATE_KEEP_ALIVE || \ @@ -163,6 +172,7 @@ typedef struct double ratioIntegral; double lastRatio; double lastClockRatio; + int64_t nextFeedbackTime; int64_t nextLogTime; unsigned int bufferOverruns; @@ -224,7 +234,8 @@ typedef struct int startupLowWaterFrames; int64_t startupPacketDeadline; int64_t startupPacketPeriod; - bool backendResampler; + PlaybackRateControl rateControl; + bool lastProviderRateControl; _Atomic(double) backendResampleRatio; atomic_bool backendResamplerFailed; RingBuffer buffer; @@ -274,6 +285,7 @@ typedef struct uint32_t bindingGeneration; uint32_t generation; LG_AudioClock clock; + double targetRate; } feedback; } @@ -590,10 +602,9 @@ static void playbackSourceRateReset(PlaybackSourceData * sourceData) sourceData->sourceRateValid = false; } -static void playbackSourceRateAdd( - PlaybackSourceData * sourceData, double nominalFrameSec) +static void playbackSourceRateAdd(PlaybackSourceData * sourceData, + int64_t timeMs, double nominalFrameSec) { - const int64_t timeMs = sourceData->mediaTimeMs; if (sourceData->rateLastSampleTimeMs != INT64_MIN && timeMs - sourceData->rateLastSampleTimeMs < PLAYBACK_RATE_SAMPLE_INTERVAL_MS) @@ -925,7 +936,7 @@ static int playbackPullFrames(uint8_t * dst, int frames) PlaybackDeviceData * data = &audio.playback.deviceData; double nextRatio = 1.0; - if (audio.playback.backendResampler) + if (audio.playback.rateControl == PLAYBACK_RATE_BACKEND) { nextRatio = atomic_load_explicit( &audio.playback.backendResampleRatio, memory_order_acquire); @@ -1045,15 +1056,16 @@ static int playbackPullFrames(uint8_t * dst, int frames) } static bool playbackSetupDevice(const LG_AudioFormat * format, - int requestedPeriodFrames, bool requestResampler) + int requestedPeriodFrames, bool requestResampler, + bool * backendResampler) { - audio.playback.backendResampler = false; audio.playback.deviceMaxPeriodFrames = 0; audio.playback.deviceStartFrames = 0; + *backendResampler = false; return audio.audioDev->playback.setup(format, requestedPeriodFrames, requestResampler, - &audio.playback.backendResampler, + backendResampler, &audio.playback.deviceMaxPeriodFrames, &audio.playback.deviceStartFrames, playbackPullFrames) && audio.playback.deviceMaxPeriodFrames > 0 && @@ -1061,7 +1073,7 @@ static bool playbackSetupDevice(const LG_AudioFormat * format, } static void playbackStart(const LG_AudioFormat * format, - const LG_AudioClock * sourceClock) + const LG_AudioClock * sourceClock, bool providerRateControl) { if (!audio.audioDev) return; @@ -1080,6 +1092,7 @@ static void playbackStart(const LG_AudioFormat * format, StreamState state = playbackGetState(); if (state == STREAM_STATE_KEEP_ALIVE && audio.playback.lastFormatValid && + audio.playback.lastProviderRateControl == providerRateControl && audioFormatEqual(format, &audio.playback.lastFormat)) { StreamState expected = STREAM_STATE_KEEP_ALIVE; @@ -1098,9 +1111,10 @@ static void playbackStart(const LG_AudioFormat * format, if (state != STREAM_STATE_STOP) playbackStop(); - audio.playback.format = *format; - audio.playback.lastFormat = *format; - audio.playback.lastFormatValid = true; + audio.playback.format = *format; + audio.playback.lastFormat = *format; + audio.playback.lastFormatValid = true; + audio.playback.lastProviderRateControl = providerRateControl; audio.playback.channels = channels; audio.playback.sampleRate = sampleRate; @@ -1122,6 +1136,7 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.sourceData.ratioIntegral = 0.0; audio.playback.sourceData.lastRatio = 1.0; audio.playback.sourceData.lastClockRatio = 1.0; + audio.playback.sourceData.nextFeedbackTime = 0; audio.playback.sourceData.bufferOverrunPending = false; audio.playback.sourceData.bufferOverruns = 0; audio.playback.sourceData.nextLogTime = @@ -1161,28 +1176,32 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.startupLowWaterFrames = 0; audio.playback.startupPacketDeadline = 0; audio.playback.startupPacketPeriod = 0; - const bool requestBackendResampler = + const bool requestBackendResampler = !providerRateControl && g_params.audioResampler != AUDIO_RESAMPLER_LIBSAMPLERATE; LG_AudioFormat deviceFormat = *format; /* The ring generates zero-filled silence. Keep unsigned PCM on the float * path because its silence level is biased rather than zero. */ - if (!requestBackendResampler || + if ((!requestBackendResampler && !providerRateControl) || deviceFormat.sampleFormat == LG_AUDIO_FMT_U8) deviceFormat.sampleFormat = LG_AUDIO_FMT_F32_NE; + bool backendResampler; bool deviceConfigured = playbackSetupDevice( - &deviceFormat, requestedPeriodFrames, requestBackendResampler); + &deviceFormat, requestedPeriodFrames, requestBackendResampler, + &backendResampler); - /* Native samples require the backend to own rate correction. If it cannot, - * reconnect using float samples for the libsamplerate path. This also - * provides a float fallback for formats unsupported by the backend. */ - if ((!deviceConfigured || !audio.playback.backendResampler) && + /* Native samples require either provider feedback or backend rate control. + * Otherwise reconnect using float samples for the libsamplerate path. This + * also provides a float fallback for formats unsupported by the backend. */ + if ((!deviceConfigured || + (!providerRateControl && !backendResampler)) && deviceFormat.sampleFormat != LG_AUDIO_FMT_F32_NE) { deviceFormat.sampleFormat = LG_AUDIO_FMT_F32_NE; deviceConfigured = playbackSetupDevice( - &deviceFormat, requestedPeriodFrames, requestBackendResampler); + &deviceFormat, requestedPeriodFrames, requestBackendResampler, + &backendResampler); } if (!deviceConfigured) @@ -1195,8 +1214,11 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.stride = channels * audioSampleSize(deviceFormat.sampleFormat); audio.playback.convertToFloat = - !audio.playback.backendResampler || + (!providerRateControl && !backendResampler) || deviceFormat.sampleFormat != format->sampleFormat; + audio.playback.rateControl = providerRateControl ? + PLAYBACK_RATE_PROVIDER : backendResampler ? + PLAYBACK_RATE_BACKEND : PLAYBACK_RATE_SOFTWARE; audio.playback.buffer = ringbuffer_newUnbounded( sampleRate, audio.playback.stride); @@ -1207,11 +1229,11 @@ static void playbackStart(const LG_AudioFormat * format, } if (g_params.audioResampler == AUDIO_RESAMPLER_BACKEND && - !audio.playback.backendResampler) + !providerRateControl && !backendResampler) DEBUG_WARN("%s could not activate backend resampling; " "using libsamplerate", audio.audioDev->name); - if (!audio.playback.backendResampler) + if (audio.playback.rateControl == PLAYBACK_RATE_SOFTWARE) { int srcError; audio.playback.sourceData.src = @@ -1226,9 +1248,20 @@ static void playbackStart(const LG_AudioFormat * format, else audio.playback.sourceData.src = NULL; - DEBUG_INFO("Using audio resampler: %s", - audio.playback.backendResampler ? - audio.audioDev->name : "libsamplerate"); + switch (audio.playback.rateControl) + { + case PLAYBACK_RATE_PROVIDER: + DEBUG_INFO("Using audio rate control: provider feedback"); + break; + + case PLAYBACK_RATE_BACKEND: + DEBUG_INFO("Using audio resampler: %s", audio.audioDev->name); + break; + + case PLAYBACK_RATE_SOFTWARE: + DEBUG_INFO("Using audio resampler: libsamplerate"); + break; + } // if a volume level was stored, set it before we return if (audio.playback.volumeChannels) @@ -1329,6 +1362,19 @@ static double computeDevicePosition(int64_t curTime) sourceData->devicePositionOffsetFrames; } +static double playbackProviderRate(const PlaybackSourceData * sourceData) +{ + const double nominalRate = audio.playback.sampleRate; + if (!sourceData->deviceClock.valid || + sourceData->deviceClock.frameSec <= 0.0) + return nominalRate; + + return clamp( + sourceData->lastRatio / sourceData->deviceClock.frameSec, + nominalRate * (1.0 - PLAYBACK_MAX_RATE_CORRECTION), + nominalRate * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)); +} + static bool playbackEnsureConversionBuffers( PlaybackSourceData * sourceData, int frames) { @@ -1347,7 +1393,7 @@ static bool playbackEnsureConversionBuffers( sourceData->framesInSize = frames; } - if (!audio.playback.backendResampler) + if (audio.playback.rateControl == PLAYBACK_RATE_SOFTWARE) { const int framesOut = (int)ceil(frames * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) + 64; @@ -1427,7 +1473,7 @@ static void playbackData(const void * data, size_t frameCount, if (state == STREAM_STATE_STOP || !audio.audioDev || frameCount == 0) return; - if (audio.playback.backendResampler && + if (audio.playback.rateControl == PLAYBACK_RATE_BACKEND && atomic_exchange_explicit( &audio.playback.backendResamplerFailed, false, memory_order_acq_rel)) @@ -1441,7 +1487,8 @@ static void playbackData(const void * data, size_t frameCount, /* Backend resampling changes how many source frames PipeWire requests per * device period. Use the command-normalized output clock for rate matching, * while deviceClock remains in the ring's source-frame domain for latency. */ - const PlaybackClock * rateClock = audio.playback.backendResampler ? + const PlaybackClock * rateClock = + audio.playback.rateControl == PLAYBACK_RATE_BACKEND ? &sourceData->outputClock : &sourceData->deviceClock; const int64_t now = nanotime(); const double nominalFrameSec = 1.0 / audio.playback.sampleRate; @@ -1506,7 +1553,13 @@ static void playbackData(const void * data, size_t frameCount, const bool sourceRateWasValid = sourceData->sourceRateValid; - playbackSourceRateAdd(sourceData, nominalFrameSec); + const bool providerRateControl = + audio.playback.rateControl == PLAYBACK_RATE_PROVIDER; + const int64_t sourceRateTimeMs = providerRateControl ? + (now - sourceData->mediaLocalOrigin) / INT64_C(1000000) : + sourceData->mediaTimeMs; + playbackSourceRateAdd( + sourceData, sourceRateTimeMs, nominalFrameSec); if (sourceClock && sourceClock->stable && sourceClock->rate > 0.0) { const double frameSec = 1.0 / sourceClock->rate; @@ -1524,7 +1577,7 @@ static void playbackData(const void * data, size_t frameCount, if (!playbackSourceClockUpdate(&sourceData->sourceClock, packetTime, sourceData->inputPosition, nominalFrameSec)) discontinuity = true; - if (sourceData->sourceRateValid) + if (sourceData->sourceRateValid && !providerRateControl) sourceData->sourceClock.frameSec = sourceData->sourceRateFrameSec; /* Track phase variation around its local baseline, not its absolute value. @@ -1587,7 +1640,7 @@ static void playbackData(const void * data, size_t frameCount, playbackClockUpdate(&sourceData->deviceClock, deviceTick.nextTime, deviceTick.nextPosition, nominalFrameSec); const bool outputClockUpdated = - !audio.playback.backendResampler || + audio.playback.rateControl != PLAYBACK_RATE_BACKEND || playbackClockUpdate(&sourceData->outputClock, deviceTick.nextTime, deviceTick.outputPosition, nominalFrameSec); @@ -1608,11 +1661,12 @@ static void playbackData(const void * data, size_t frameCount, * switching models would create a false phase step and drive the resampler * despite an already-correct ring level. Keep the source clock untouched: * changing it would also disturb source phase and jitter tracking. */ + const int64_t referenceTime = providerRateControl ? now : curTime; const double rawDevicePosition = - playbackClockPosition(&sourceData->deviceClock, curTime); + playbackClockPosition(&sourceData->deviceClock, referenceTime); sourceData->devicePositionOffsetFrames = - sourceData->devReadPosition - sourceReserveFrames - - rawDevicePosition; + sourceData->devReadPosition - rawDevicePosition - + (providerRateControl ? 0.0 : sourceReserveFrames); } const int maxPeriodFrames = @@ -1637,8 +1691,8 @@ static void playbackData(const void * data, size_t frameCount, const double targetBufferFrames = minimumBufferFrames + latencyOffsetFrames; const double resamplerDelayFrames = - audio.playback.backendResampler ? - 0.0 : PLAYBACK_RESAMPLER_DELAY_FRAMES; + audio.playback.rateControl == PLAYBACK_RATE_SOFTWARE ? + PLAYBACK_RESAMPLER_DELAY_FRAMES : 0.0; const double minimumLatencyFrames = minimumBufferFrames + resamplerDelayFrames; const double targetLatencyFrames = @@ -1646,9 +1700,27 @@ static void playbackData(const void * data, size_t frameCount, double devPosition = DBL_MIN; state = playbackGetState(); - if ((discontinuity || + if (providerRateControl && + (discontinuity || state == STREAM_STATE_KEEP_ALIVE || - state == STREAM_STATE_RESUMING) && + state == STREAM_STATE_RESUMING)) + { + const int occupancy = ringbuffer_getCount(audio.playback.buffer); + const int slewFrames = clamp( + llrint(targetLowWaterFrames - occupancy), + (int64_t)INT_MIN, (int64_t)INT_MAX); + const int actualSlew = playbackSlewBuffer(sourceData, slewFrames); + sourceData->outputPosition += actualSlew; + curPosition += actualSlew; + + sourceData->offsetError = 0.0; + sourceData->offsetErrorIntegral = 0.0; + sourceData->ratioIntegral = 0.0; + playbackSetState(STREAM_STATE_RUN); + } + else if ((discontinuity || + state == STREAM_STATE_KEEP_ALIVE || + state == STREAM_STATE_RESUMING) && sourceData->deviceClock.valid && sourceData->deviceClockStable) { @@ -1668,7 +1740,25 @@ static void playbackData(const void * data, size_t frameCount, double actualLatencyFrames = 0.0; double actualOffsetError = 0.0; - if (sourceData->deviceClock.valid) + if (providerRateControl) + { + const int occupancy = ringbuffer_getCount(audio.playback.buffer); + actualLatencyFrames = occupancy + sourceReserveFrames; + actualOffsetError = targetLowWaterFrames - occupancy; + + const double error = + actualOffsetError - sourceData->offsetError; + const double periodSec = frames * nominalFrameSec; + const double omega = + 2.0 * M_PI * PLAYBACK_OFFSET_FILTER_BANDWIDTH_HZ * periodSec; + const double b = M_SQRT2 * omega; + const double c = omega * omega; + + sourceData->offsetError += b * error + + sourceData->offsetErrorIntegral; + sourceData->offsetErrorIntegral += c * error; + } + else if (sourceData->deviceClock.valid) { if (sourceData->deviceClockStable) { @@ -1722,10 +1812,11 @@ static void playbackData(const void * data, size_t frameCount, 2.0 * naturalFrequency / audio.playback.sampleRate; const double ki = naturalFrequency * naturalFrequency / audio.playback.sampleRate; - if (sourceRateBecameValid) + if (sourceRateBecameValid && !providerRateControl) sourceData->ratioIntegral = 0.0; - if (sourceData->deviceClockStable && + if (!providerRateControl && + sourceData->deviceClockStable && sourceData->sourceRateValid && rateClock->updates >= 2) { @@ -1755,7 +1846,8 @@ static void playbackData(const void * data, size_t frameCount, double controllerKp = kp; double controllerKi = ki; double controllerError = phaseError; - double controllerBase = sourceData->lastClockRatio; + double controllerBase = providerRateControl ? + 1.0 : sourceData->lastClockRatio; if (acquiringDeviceClock) { @@ -1811,7 +1903,7 @@ static void playbackData(const void * data, size_t frameCount, sourceData->lastRatio + maxRatioStep); sourceData->lastRatio = ratio; - if (audio.playback.backendResampler) + if (audio.playback.rateControl == PLAYBACK_RATE_BACKEND) { atomic_store_explicit( &audio.playback.backendResampleRatio, ratio, @@ -1820,6 +1912,12 @@ static void playbackData(const void * data, size_t frameCount, playbackAppendFrames(sourceData, inputFrames, frames); sourceData->outputPosition += outputFrames; } + else if (audio.playback.rateControl == PLAYBACK_RATE_PROVIDER) + { + const int outputFrames = + playbackAppendFrames(sourceData, inputFrames, frames); + sourceData->outputPosition += outputFrames; + } else { int consumed = 0; @@ -1887,7 +1985,7 @@ static void playbackData(const void * data, size_t frameCount, const float graphMax = targetLatencyFrames * 1000.0 / audio.playback.sampleRate * 2; - audio.playback.graph = app_registerGraph("PLAYBACK", + audio.playback.graph = app_registerGraph("PLAYBACK RING", audio.playback.timings, 0.0f, graphMax, audioGraphFormatFn); if (!audio.playback.graph) @@ -1929,21 +2027,36 @@ static void playbackData(const void * data, size_t frameCount, if (now >= sourceData->nextLogTime) { + const double backendLatencyMs = + audio.audioDev->playback.latency ? + audio.audioDev->playback.latency() / 1000.0 : 0.0; const double sourcePpm = sourceData->sourceRateValid ? - (sourceData->sourceRateFrameSec / nominalFrameSec - 1.0) * 1.0e6 : + (nominalFrameSec / sourceData->sourceRateFrameSec - 1.0) * 1.0e6 : 0.0; const double devicePpm = rateClock->valid ? - (rateClock->frameSec / nominalFrameSec - 1.0) * 1.0e6 : + (nominalFrameSec / rateClock->frameSec - 1.0) * 1.0e6 : 0.0; + const bool providerControl = + audio.playback.rateControl == PLAYBACK_RATE_PROVIDER; + const double controlPpm = providerControl ? + (playbackProviderRate(sourceData) / + audio.playback.sampleRate - 1.0) * 1.0e6 : + (ratio - 1.0) * 1.0e6; + const char * controlName = providerControl ? "feedback" : + audio.playback.rateControl == PLAYBACK_RATE_BACKEND ? + "backend" : "software"; + const char * sourceRateName = providerControl ? "arrival" : "source"; const unsigned int underruns = atomic_exchange_explicit( &audio.playback.underruns, 0, memory_order_relaxed); DEBUG_INFO( - "Audio sync: %.2f/%.2f ms, ratio %+.1f ppm, " - "clocks %+.1f/%+.1f ppm, jitter %.2f ms, xruns %u/%u", + "Audio sync: ring %.2f/%.2f ms, backend %.2f ms, " + "%s %+.1f ppm, rates %s/device %+.1f/%+.1f ppm, " + "jitter %.2f ms, xruns %u/%u", softwareLatencyMs, targetLatencyFrames * 1000.0 / audio.playback.sampleRate, - (ratio - 1.0) * 1.0e6, sourcePpm, devicePpm, + backendLatencyMs, controlName, controlPpm, + sourceRateName, sourcePpm, devicePpm, sourceData->arrivalJitterSec * 1000.0, underruns, sourceData->bufferOverruns); @@ -1952,24 +2065,37 @@ static void playbackData(const void * data, size_t frameCount, } } -static bool playbackGetFeedback(LG_AudioClock * clock) +static bool playbackGetFeedback( + LG_AudioClock * clock, double * targetRate) { - const PlaybackSourceData * sourceData = &audio.playback.sourceData; - if (!clock || !sourceData->deviceClock.valid || - sourceData->deviceClock.position < 0.0 || + PlaybackSourceData * sourceData = &audio.playback.sourceData; + if (!clock || !targetRate || + audio.playback.rateControl != PLAYBACK_RATE_PROVIDER || + !sourceData->deviceClock.valid || sourceData->deviceClock.frameSec <= 0.0) return false; + const int64_t now = nanotime(); + if (now < sourceData->nextFeedbackTime) + return false; + + sourceData->nextFeedbackTime = now + PLAYBACK_FEEDBACK_INTERVAL_NS; + const double position = + playbackClockPosition(&sourceData->deviceClock, now); + if (position < 0.0) + return false; + const uint64_t latency = audio.audioDev->playback.latency ? audio.audioDev->playback.latency() : 0; *clock = (LG_AudioClock) { - .position = llrint(sourceData->deviceClock.position), - .time = sourceData->deviceClock.time + latency * 1000, + .position = llrint(position), + .time = now + latency * 1000, .rate = 1.0 / sourceData->deviceClock.frameSec, .stable = sourceData->deviceClockStable, }; + *targetRate = playbackProviderRate(sourceData); return true; } @@ -2249,18 +2375,19 @@ static bool bindingActiveNL(const AudioBinding * binding) static void queueFeedback(const LG_AudioOps * ops, void * opaque, uint32_t bindingGeneration, uint32_t generation, - const LG_AudioClock * clock) + const LG_AudioClock * clock, double targetRate) { if (!audio.feedback.event || !audio.feedback.thread || !clock) return; LG_LOCK(audio.feedback.lock); - audio.feedback.ops = ops; - audio.feedback.opaque = opaque; + audio.feedback.ops = ops; + audio.feedback.opaque = opaque; audio.feedback.bindingGeneration = bindingGeneration; - audio.feedback.generation = generation; - audio.feedback.clock = *clock; - audio.feedback.pending = true; + audio.feedback.generation = generation; + audio.feedback.clock = *clock; + audio.feedback.targetRate = targetRate; + audio.feedback.pending = true; LG_UNLOCK(audio.feedback.lock); lgSignalEvent(audio.feedback.event); } @@ -2276,7 +2403,9 @@ static void eventPlaybackStart(void * opaque, uint32_t generation, LG_LOCK(audio.playback.sourceLock); atomic_store_explicit(&audio.playback.streamGeneration, generation, memory_order_release); - playbackStart(format, sourceClock); + playbackStart(format, sourceClock, + binding->ops->clockFeedback && + audio.feedback.event && audio.feedback.thread); LG_UNLOCK(audio.playback.sourceLock); } LG_UNLOCK_SHARED(audio.activeLock); @@ -2356,9 +2485,11 @@ static void eventPlaybackData(void * opaque, uint32_t generation, playbackData(data, frames, sourceClock); LG_AudioClock feedback; - if (ops->clockFeedback && playbackGetFeedback(&feedback)) + double targetRate; + if (ops->clockFeedback && + playbackGetFeedback(&feedback, &targetRate)) queueFeedback(ops, providerOpaque, binding->generation, - generation, &feedback); + generation, &feedback, targetRate); } LG_UNLOCK(audio.playback.sourceLock); } @@ -2616,18 +2747,20 @@ static int feedbackThread(void * opaque) break; const LG_AudioOps * ops; - void * providerOpaque; - uint32_t bindingGeneration; - uint32_t generation; - LG_AudioClock clock; + void * providerOpaque; + uint32_t bindingGeneration; + uint32_t generation; + LG_AudioClock clock; + double targetRate; LG_LOCK(audio.feedback.lock); - const bool pending = audio.feedback.pending; - ops = audio.feedback.ops; - providerOpaque = audio.feedback.opaque; - bindingGeneration = audio.feedback.bindingGeneration; - generation = audio.feedback.generation; - clock = audio.feedback.clock; + const bool pending = audio.feedback.pending; + ops = audio.feedback.ops; + providerOpaque = audio.feedback.opaque; + bindingGeneration = audio.feedback.bindingGeneration; + generation = audio.feedback.generation; + clock = audio.feedback.clock; + targetRate = audio.feedback.targetRate; audio.feedback.pending = false; LG_UNLOCK(audio.feedback.lock); @@ -2640,7 +2773,8 @@ static int feedbackThread(void * opaque) audio.active.generation == bindingGeneration && atomic_load_explicit(&audio.playback.streamGeneration, memory_order_acquire) == generation) - ops->clockFeedback(providerOpaque, generation, &clock); + ops->clockFeedback( + providerOpaque, generation, &clock, targetRate); LG_UNLOCK_SHARED(audio.activeLock); } diff --git a/client/src/audio_usb.c b/client/src/audio_usb.c index f11b2fd9..007313f1 100644 --- a/client/src/audio_usb.c +++ b/client/src/audio_usb.c @@ -532,6 +532,28 @@ static void usbDetach(void * opaque) LG_UNLOCK(state->stateLock); } +static bool usbClockFeedback(void * opaque, uint32_t generation, + const LG_AudioClock * playbackClock, double targetRate) +{ + LGA_USBState * state = opaque; + + LG_LOCK(state->stateLock); + if (!state->attached || state->streamGeneration != generation) + { + LG_UNLOCK(state->stateLock); + return false; + } + + const double nominalRate = state->streamFormat.sampleRate; + const double rate = playbackClock && + targetRate >= nominalRate * 0.995 && + targetRate <= nominalRate * 1.005 ? + targetRate : nominalRate; + lgUsbAudio_setFeedbackRate(state->device, rate); + LG_UNLOCK(state->stateLock); + return true; +} + const LG_AudioOps LGA_USB = { .name = "USB Audio", @@ -539,7 +561,7 @@ const LG_AudioOps LGA_USB = .attach = usbAttach, .detach = usbDetach, .recordData = NULL, - .clockFeedback = NULL, + .clockFeedback = usbClockFeedback, }; LGA_USBState * lgaUsb_create(void) diff --git a/client/src/usb_audio.c b/client/src/usb_audio.c index bf289206..7680d790 100644 --- a/client/src/usb_audio.c +++ b/client/src/usb_audio.c @@ -25,27 +25,35 @@ #include #include +#include #include #include #include enum { - USB_AUDIO_VENDOR_ID = 0x043e, - USB_AUDIO_PRODUCT_ID = 0x0001, - USB_AUDIO_DEVICE_VERSION = 0x010a, - USB_AUDIO_CONFIGURATION = 1, - USB_AUDIO_CONTROL_IFACE = 0, - USB_AUDIO_STREAM_IFACE = 1, - USB_AUDIO_CLOCK_ID = 1, - USB_AUDIO_STREAM_ENDPOINT = 0x01, - USB_AUDIO_EP_INTERVAL = 1, - USB_AUDIO_SAMPLE_SIZE = 3, - USB_AUDIO_HS_PACKET_FRAMES = 25, - USB_AUDIO_FS_PACKET_FRAMES = 97, - USB_AUDIO_DESCRIPTOR_BASE = 81, - USB_AUDIO_STREAM_DESC_SIZE = 46, - USB_AUDIO_OTHER_SPEED_SIZE = + USB_AUDIO_VENDOR_ID = 0x043e, + USB_AUDIO_PRODUCT_ID = 0x0001, + USB_AUDIO_DEVICE_VERSION = 0x010b, + USB_AUDIO_CONFIGURATION = 1, + USB_AUDIO_CONTROL_IFACE = 0, + USB_AUDIO_STREAM_IFACE = 1, + USB_AUDIO_CLOCK_ID = 1, + USB_AUDIO_DATA_ENDPOINT = 0x01, + USB_AUDIO_FEEDBACK_ENDPOINT = 0x81, + USB_AUDIO_DATA_EP_INDEX = 1, + USB_AUDIO_FEEDBACK_EP_INDEX = 17, + USB_AUDIO_DATA_INTERVAL = 1, + USB_AUDIO_FEEDBACK_INTERVAL = 8, + USB_AUDIO_FEEDBACK_BINTERVAL = 4, + USB_AUDIO_FEEDBACK_SIZE = 4, + USB_AUDIO_FEEDBACK_MAX_QUEUE = 512, + USB_AUDIO_SAMPLE_SIZE = 3, + USB_AUDIO_HS_PACKET_FRAMES = 25, + USB_AUDIO_FS_PACKET_FRAMES = 97, + USB_AUDIO_DESCRIPTOR_BASE = 81, + USB_AUDIO_STREAM_DESC_SIZE = 53, + USB_AUDIO_OTHER_SPEED_SIZE = USB_AUDIO_DESCRIPTOR_BASE + USB_AUDIO_STREAM_DESC_SIZE, }; @@ -134,8 +142,9 @@ static const uint8_t l_deviceDescriptor[] = 0x03, 0x01, }; -#define USB_AUDIO_STREAM_DESCRIPTOR(alt, channels, mask, packetFrames) \ - 0x09, 0x04, 0x01, alt, 0x01, 0x01, 0x02, 0x20, 0x00, \ +#define USB_AUDIO_STREAM_DESCRIPTOR(alt, channels, mask, packetFrames, \ + feedbackSize, feedbackInterval) \ + 0x09, 0x04, 0x01, alt, 0x02, 0x01, 0x02, 0x20, 0x00, \ 0x10, 0x24, 0x01, 0x02, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, \ channels, \ (mask) & 0xff, \ @@ -144,15 +153,18 @@ static const uint8_t l_deviceDescriptor[] = (mask) >> 24 & 0xff, \ 0x00, \ 0x06, 0x24, 0x02, 0x01, 0x03, 0x18, \ - 0x07, 0x05, 0x01, 0x09, \ + 0x07, 0x05, USB_AUDIO_DATA_ENDPOINT, 0x05, \ USB_AUDIO_PACKET_SIZE(packetFrames, channels) & 0xff, \ USB_AUDIO_PACKET_SIZE(packetFrames, channels) >> 8 & 0xff, \ - USB_AUDIO_EP_INTERVAL, \ - 0x08, 0x25, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + USB_AUDIO_DATA_INTERVAL, \ + 0x08, 0x25, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x07, 0x05, USB_AUDIO_FEEDBACK_ENDPOINT, 0x11, \ + feedbackSize, 0x00, feedbackInterval, #define USB_AUDIO_HS_STREAM_DESCRIPTOR(alt, channels, mask) \ USB_AUDIO_STREAM_DESCRIPTOR( \ - alt, channels, mask, USB_AUDIO_HS_PACKET_FRAMES) + alt, channels, mask, USB_AUDIO_HS_PACKET_FRAMES, \ + USB_AUDIO_FEEDBACK_SIZE, USB_AUDIO_FEEDBACK_BINTERVAL) static const uint8_t l_configurationDescriptor[] = { @@ -186,7 +198,7 @@ static const uint8_t l_configurationDescriptor[] = static const uint8_t l_fullSpeedStreamDescriptor[] = { USB_AUDIO_STREAM_DESCRIPTOR(1, 2, USB_AUDIO_LAYOUT_STEREO, - USB_AUDIO_FS_PACKET_FRAMES) + USB_AUDIO_FS_PACKET_FRAMES, 3, 1) }; #undef USB_AUDIO_STREAM_DESCRIPTOR #undef USB_AUDIO_PACKET_SIZE @@ -235,7 +247,11 @@ struct LG_USBAudio uint8_t configuration; uint8_t streamAlt; uint32_t sampleRate; - bool streaming; + bool dataStreaming; + bool feedbackStreaming; + uint64_t feedbackPacketId; + uint8_t feedbackDataPackets; + atomic_uint feedbackValue; }; static LG_USBAudio * getAudio(void * opaque) @@ -266,6 +282,17 @@ static void writeLE32(uint8_t * data, uint32_t value) data[3] = value >> 24; } +static uint32_t encodeFeedbackRate(double sampleRate) +{ + return (uint32_t)(sampleRate * UINT32_C(65536) / 8000.0 + 0.5); +} + +static void resetFeedbackRate(LG_USBAudio * audio) +{ + atomic_store_explicit(&audio->feedbackValue, + encodeFeedbackRate(audio->sampleRate), memory_order_release); +} + static uint32_t nearestSampleRate(uint32_t sampleRate) { uint32_t selected = l_sampleRates[0]; @@ -320,21 +347,35 @@ static size_t writeSampleRateRange(uint8_t * buffer) static void stopPlayback(LG_USBAudio * audio) { - if (!audio->streaming) + if (!audio->dataStreaming) return; - audio->streaming = false; + audio->dataStreaming = false; + resetFeedbackRate(audio); if (audio->events && audio->events->stop) audio->events->stop(audio->eventOpaque); } +static void stopFeedback(LG_USBAudio * audio) +{ + audio->feedbackStreaming = false; + audio->feedbackDataPackets = 0; +} + +static void stopStreams(LG_USBAudio * audio) +{ + stopPlayback(audio); + stopFeedback(audio); +} + static void startPlayback(LG_USBAudio * audio) { const USBAudioLayout * layout = getLayout(audio->streamAlt); - if (audio->streaming || !layout) + if (audio->dataStreaming || !layout) return; - audio->streaming = true; + resetFeedbackRate(audio); + audio->dataStreaming = true; if (audio->events && audio->events->start) audio->events->start( audio->eventOpaque, audio->sampleRate, layout->channelMask); @@ -345,20 +386,22 @@ static void setSampleRate(LG_USBAudio * audio, uint32_t sampleRate) if (audio->sampleRate == sampleRate) return; - const bool restart = audio->streaming; + const bool restart = audio->dataStreaming; stopPlayback(audio); audio->sampleRate = sampleRate; + resetFeedbackRate(audio); if (restart) startPlayback(audio); } static void resetDevice(LG_USBAudio * audio) { - stopPlayback(audio); + stopStreams(audio); audio->configuration = 0; audio->streamAlt = 0; audio->sampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; + resetFeedbackRate(audio); } static void sendInterfaceInfo(LG_USBAudio * audio) @@ -387,11 +430,19 @@ static void sendEndpointInfo(LG_USBAudio * audio) if (audio->configuration == USB_AUDIO_CONFIGURATION && layout) { - info.type[USB_AUDIO_STREAM_ENDPOINT] = usb_redir_type_iso; - info.interval[USB_AUDIO_STREAM_ENDPOINT] = USB_AUDIO_EP_INTERVAL; - info.interface[USB_AUDIO_STREAM_ENDPOINT] = USB_AUDIO_STREAM_IFACE; - info.max_packet_size[USB_AUDIO_STREAM_ENDPOINT] = + info.type[USB_AUDIO_DATA_EP_INDEX] = usb_redir_type_iso; + info.interval[USB_AUDIO_DATA_EP_INDEX] = USB_AUDIO_DATA_INTERVAL; + info.interface[USB_AUDIO_DATA_EP_INDEX] = USB_AUDIO_STREAM_IFACE; + info.max_packet_size[USB_AUDIO_DATA_EP_INDEX] = layoutPacketSize(layout); + + info.type[USB_AUDIO_FEEDBACK_EP_INDEX] = usb_redir_type_iso; + info.interval[USB_AUDIO_FEEDBACK_EP_INDEX] = + USB_AUDIO_FEEDBACK_INTERVAL; + info.interface[USB_AUDIO_FEEDBACK_EP_INDEX] = + USB_AUDIO_STREAM_IFACE; + info.max_packet_size[USB_AUDIO_FEEDBACK_EP_INDEX] = + USB_AUDIO_FEEDBACK_SIZE; } usbredirparser_send_ep_info(audio->parser, &info); @@ -462,7 +513,7 @@ static void setAltSetting(void * opaque, uint64_t id, request->interface == USB_AUDIO_STREAM_IFACE && (request->alt == 0 || getLayout(request->alt))) { - stopPlayback(audio); + stopStreams(audio); audio->streamAlt = request->alt; status.status = usb_redir_success; } @@ -507,31 +558,80 @@ static void sendISOStatus(LG_USBAudio * audio, uint64_t id, usbredirparser_send_iso_stream_status(audio->parser, id, &status); } +static void sendFeedbackPackets(LG_USBAudio * audio, uint32_t count) +{ + uint8_t data[USB_AUDIO_FEEDBACK_SIZE]; + writeLE32(data, atomic_load_explicit( + &audio->feedbackValue, memory_order_acquire)); + struct usb_redir_iso_packet_header packet = + { + .endpoint = USB_AUDIO_FEEDBACK_ENDPOINT, + .status = usb_redir_success, + .length = sizeof(data), + }; + + for (uint32_t i = 0; i < count; ++i) + usbredirparser_send_iso_packet(audio->parser, + audio->feedbackPacketId++, + &packet, data, USB_AUDIO_FEEDBACK_SIZE); +} + static void startISOStream(void * opaque, uint64_t id, struct usb_redir_start_iso_stream_header * request) { - LG_USBAudio * audio = getAudio(opaque); - uint8_t result = usb_redir_stall; + LG_USBAudio * audio = getAudio(opaque); + uint8_t result = usb_redir_stall; + uint32_t feedbackPrefill = 0; - if (request->endpoint == USB_AUDIO_STREAM_ENDPOINT && - audio->configuration == USB_AUDIO_CONFIGURATION && + if (audio->configuration == USB_AUDIO_CONFIGURATION && getLayout(audio->streamAlt)) { - result = usb_redir_success; - startPlayback(audio); + switch (request->endpoint) + { + case USB_AUDIO_DATA_ENDPOINT: + result = usb_redir_success; + startPlayback(audio); + break; + + case USB_AUDIO_FEEDBACK_ENDPOINT: + resetFeedbackRate(audio); + audio->feedbackStreaming = true; + audio->feedbackDataPackets = 0; + audio->feedbackPacketId = 0; + feedbackPrefill = + (uint32_t)request->pkts_per_urb * request->no_urbs; + if (!feedbackPrefill) + feedbackPrefill = 1; + else if (feedbackPrefill > + USB_AUDIO_FEEDBACK_MAX_QUEUE) + feedbackPrefill = USB_AUDIO_FEEDBACK_MAX_QUEUE; + result = usb_redir_success; + break; + } } sendISOStatus(audio, id, request->endpoint, result); + if (feedbackPrefill) + sendFeedbackPackets(audio, feedbackPrefill); } static void stopISOStream(void * opaque, uint64_t id, struct usb_redir_stop_iso_stream_header * request) { LG_USBAudio * audio = getAudio(opaque); - const uint8_t result = request->endpoint == USB_AUDIO_STREAM_ENDPOINT ? - usb_redir_success : usb_redir_stall; - if (result == usb_redir_success) - stopPlayback(audio); + uint8_t result = usb_redir_stall; + switch (request->endpoint) + { + case USB_AUDIO_DATA_ENDPOINT: + stopPlayback(audio); + result = usb_redir_success; + break; + + case USB_AUDIO_FEEDBACK_ENDPOINT: + stopFeedback(audio); + result = usb_redir_success; + break; + } sendISOStatus(audio, id, request->endpoint, result); } @@ -556,7 +656,7 @@ static size_t stringDescriptor(uint8_t index, uint8_t * buffer, { case 1: text = "Looking Glass" ; break; case 2: text = "Looking Glass USB Audio"; break; - case 3: text = "LG-UAC2-0003" ; break; + case 3: text = "LG-UAC2-0004" ; break; default: return 0; } @@ -695,7 +795,8 @@ static void controlPacket(void * opaque, uint64_t id, valid = request->index == 0x00 || request->index == 0x80 || (audio->configuration == USB_AUDIO_CONFIGURATION && getLayout(audio->streamAlt) && - request->index == USB_AUDIO_STREAM_ENDPOINT); + (request->index == USB_AUDIO_DATA_ENDPOINT || + request->index == USB_AUDIO_FEEDBACK_ENDPOINT)); break; } @@ -756,7 +857,7 @@ static void controlPacket(void * opaque, uint64_t id, static void stallStream(LG_USBAudio * audio) { stopPlayback(audio); - sendISOStatus(audio, 0, USB_AUDIO_STREAM_ENDPOINT, usb_redir_stall); + sendISOStatus(audio, 0, USB_AUDIO_DATA_ENDPOINT, usb_redir_stall); } static void isoPacket(void * opaque, uint64_t id, @@ -767,11 +868,11 @@ static void isoPacket(void * opaque, uint64_t id, LG_USBAudio * audio = getAudio(opaque); const USBAudioLayout * layout = getLayout(audio->streamAlt); - if (audio->streaming) + if (audio->dataStreaming) { const uint16_t frameSize = layout ? layout->channelCount * USB_AUDIO_SAMPLE_SIZE : 0; - if (packet->endpoint != USB_AUDIO_STREAM_ENDPOINT || + if (packet->endpoint != USB_AUDIO_DATA_ENDPOINT || packet->status != usb_redir_success || dataLength < 0 || packet->length != dataLength || !layout || dataLength > layoutPacketSize(layout) || @@ -780,9 +881,19 @@ static void isoPacket(void * opaque, uint64_t id, DEBUG_WARN("Invalid USB audio isochronous packet"); stallStream(audio); } - else if (dataLength && audio->events && audio->events->data) - audio->events->data(audio->eventOpaque, data, - dataLength / frameSize); + else + { + if (dataLength && audio->events && audio->events->data) + audio->events->data(audio->eventOpaque, data, + dataLength / frameSize); + + if (audio->feedbackStreaming && + ++audio->feedbackDataPackets == USB_AUDIO_FEEDBACK_INTERVAL) + { + audio->feedbackDataPackets = 0; + sendFeedbackPackets(audio, 1); + } + } } if (data) @@ -847,9 +958,21 @@ LG_USBAudio * lgUsbAudio_create( audio->events = events; audio->eventOpaque = eventOpaque; + audio->sampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; + atomic_init(&audio->feedbackValue, + encodeFeedbackRate(audio->sampleRate)); return audio; } +void lgUsbAudio_setFeedbackRate(LG_USBAudio * audio, double sampleRate) +{ + if (!audio) + return; + + atomic_store_explicit(&audio->feedbackValue, + encodeFeedbackRate(sampleRate), memory_order_release); +} + void lgUsbAudio_destroy(LG_USBAudio * audio) { if (!audio) diff --git a/client/src/usb_audio.h b/client/src/usb_audio.h index a81e22cc..7280b394 100644 --- a/client/src/usb_audio.h +++ b/client/src/usb_audio.h @@ -47,6 +47,10 @@ LG_USBAudio * lgUsbAudio_create( /* Destroy the LG_USBRedir using this device before destroying the device. */ void lgUsbAudio_destroy(LG_USBAudio * audio); +/* Publish the requested source rate without touching usbredir from the audio + * feedback thread. */ +void lgUsbAudio_setFeedbackRate(LG_USBAudio * audio, double sampleRate); + const LG_USBRedirDeviceOps * lgUsbAudio_deviceOps(void); #endif