diff --git a/client/include/interface/audio.h b/client/include/interface/audio.h index 90025c86..9aae7aa8 100644 --- a/client/include/interface/audio.h +++ b/client/include/interface/audio.h @@ -156,12 +156,15 @@ typedef struct LG_AudioOps bool (*recordData)(void * opaque, uint32_t generation, const void * data, size_t frames, const LG_AudioClock * sourceClock); - /* Optional active synchronization feedback. This is called outside the - * realtime audio callback with the measured playback device clock. Its - * position uses the device's independent output-frame timeline and its time - * includes the backend's estimated presentation latency. */ + /* Optional active synchronization feedback. A provider implementing this + * owns playback rate control, so the client preserves its frame rate and + * does not activate a local resampler. This is called outside the realtime + * audio callback with the measured playback device clock. Its position uses + * the device's independent output-frame timeline and its time includes the + * backend's estimated presentation latency. targetRate is the source frame + * rate requested by the client's buffer controller. */ bool (*clockFeedback)(void * opaque, uint32_t generation, - const LG_AudioClock * playbackClock); + const LG_AudioClock * playbackClock, double targetRate); } LG_AudioOps; diff --git a/client/src/audio.c b/client/src/audio.c index f54c2ce0..2f9e0985 100644 --- a/client/src/audio.c +++ b/client/src/audio.c @@ -65,6 +65,7 @@ #define PLAYBACK_DEVICE_RATE_STABLE_SEC 2.0 #define PLAYBACK_DEVICE_RATE_STABLE_DELTA_PPM 50.0 #define PLAYBACK_DEVICE_RATE_MAX_ACQUIRE_SEC 20.0 +#define PLAYBACK_FEEDBACK_INTERVAL_NS INT64_C(1000000) typedef enum { @@ -78,6 +79,14 @@ typedef enum } StreamState; +typedef enum +{ + PLAYBACK_RATE_SOFTWARE, + PLAYBACK_RATE_BACKEND, + PLAYBACK_RATE_PROVIDER, +} +PlaybackRateControl; + #define STREAM_ACTIVE(state) \ (state == STREAM_STATE_RUN || \ state == STREAM_STATE_KEEP_ALIVE || \ @@ -163,6 +172,7 @@ typedef struct double ratioIntegral; double lastRatio; double lastClockRatio; + int64_t nextFeedbackTime; int64_t nextLogTime; unsigned int bufferOverruns; @@ -224,7 +234,8 @@ typedef struct int startupLowWaterFrames; int64_t startupPacketDeadline; int64_t startupPacketPeriod; - bool backendResampler; + PlaybackRateControl rateControl; + bool lastProviderRateControl; _Atomic(double) backendResampleRatio; atomic_bool backendResamplerFailed; RingBuffer buffer; @@ -274,6 +285,7 @@ typedef struct uint32_t bindingGeneration; uint32_t generation; LG_AudioClock clock; + double targetRate; } feedback; } @@ -590,10 +602,9 @@ static void playbackSourceRateReset(PlaybackSourceData * sourceData) sourceData->sourceRateValid = false; } -static void playbackSourceRateAdd( - PlaybackSourceData * sourceData, double nominalFrameSec) +static void playbackSourceRateAdd(PlaybackSourceData * sourceData, + int64_t timeMs, double nominalFrameSec) { - const int64_t timeMs = sourceData->mediaTimeMs; if (sourceData->rateLastSampleTimeMs != INT64_MIN && timeMs - sourceData->rateLastSampleTimeMs < PLAYBACK_RATE_SAMPLE_INTERVAL_MS) @@ -925,7 +936,7 @@ static int playbackPullFrames(uint8_t * dst, int frames) PlaybackDeviceData * data = &audio.playback.deviceData; double nextRatio = 1.0; - if (audio.playback.backendResampler) + if (audio.playback.rateControl == PLAYBACK_RATE_BACKEND) { nextRatio = atomic_load_explicit( &audio.playback.backendResampleRatio, memory_order_acquire); @@ -1045,15 +1056,16 @@ static int playbackPullFrames(uint8_t * dst, int frames) } static bool playbackSetupDevice(const LG_AudioFormat * format, - int requestedPeriodFrames, bool requestResampler) + int requestedPeriodFrames, bool requestResampler, + bool * backendResampler) { - audio.playback.backendResampler = false; audio.playback.deviceMaxPeriodFrames = 0; audio.playback.deviceStartFrames = 0; + *backendResampler = false; return audio.audioDev->playback.setup(format, requestedPeriodFrames, requestResampler, - &audio.playback.backendResampler, + backendResampler, &audio.playback.deviceMaxPeriodFrames, &audio.playback.deviceStartFrames, playbackPullFrames) && audio.playback.deviceMaxPeriodFrames > 0 && @@ -1061,7 +1073,7 @@ static bool playbackSetupDevice(const LG_AudioFormat * format, } static void playbackStart(const LG_AudioFormat * format, - const LG_AudioClock * sourceClock) + const LG_AudioClock * sourceClock, bool providerRateControl) { if (!audio.audioDev) return; @@ -1080,6 +1092,7 @@ static void playbackStart(const LG_AudioFormat * format, StreamState state = playbackGetState(); if (state == STREAM_STATE_KEEP_ALIVE && audio.playback.lastFormatValid && + audio.playback.lastProviderRateControl == providerRateControl && audioFormatEqual(format, &audio.playback.lastFormat)) { StreamState expected = STREAM_STATE_KEEP_ALIVE; @@ -1098,9 +1111,10 @@ static void playbackStart(const LG_AudioFormat * format, if (state != STREAM_STATE_STOP) playbackStop(); - audio.playback.format = *format; - audio.playback.lastFormat = *format; - audio.playback.lastFormatValid = true; + audio.playback.format = *format; + audio.playback.lastFormat = *format; + audio.playback.lastFormatValid = true; + audio.playback.lastProviderRateControl = providerRateControl; audio.playback.channels = channels; audio.playback.sampleRate = sampleRate; @@ -1122,6 +1136,7 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.sourceData.ratioIntegral = 0.0; audio.playback.sourceData.lastRatio = 1.0; audio.playback.sourceData.lastClockRatio = 1.0; + audio.playback.sourceData.nextFeedbackTime = 0; audio.playback.sourceData.bufferOverrunPending = false; audio.playback.sourceData.bufferOverruns = 0; audio.playback.sourceData.nextLogTime = @@ -1161,28 +1176,32 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.startupLowWaterFrames = 0; audio.playback.startupPacketDeadline = 0; audio.playback.startupPacketPeriod = 0; - const bool requestBackendResampler = + const bool requestBackendResampler = !providerRateControl && g_params.audioResampler != AUDIO_RESAMPLER_LIBSAMPLERATE; LG_AudioFormat deviceFormat = *format; /* The ring generates zero-filled silence. Keep unsigned PCM on the float * path because its silence level is biased rather than zero. */ - if (!requestBackendResampler || + if ((!requestBackendResampler && !providerRateControl) || deviceFormat.sampleFormat == LG_AUDIO_FMT_U8) deviceFormat.sampleFormat = LG_AUDIO_FMT_F32_NE; + bool backendResampler; bool deviceConfigured = playbackSetupDevice( - &deviceFormat, requestedPeriodFrames, requestBackendResampler); + &deviceFormat, requestedPeriodFrames, requestBackendResampler, + &backendResampler); - /* Native samples require the backend to own rate correction. If it cannot, - * reconnect using float samples for the libsamplerate path. This also - * provides a float fallback for formats unsupported by the backend. */ - if ((!deviceConfigured || !audio.playback.backendResampler) && + /* Native samples require either provider feedback or backend rate control. + * Otherwise reconnect using float samples for the libsamplerate path. This + * also provides a float fallback for formats unsupported by the backend. */ + if ((!deviceConfigured || + (!providerRateControl && !backendResampler)) && deviceFormat.sampleFormat != LG_AUDIO_FMT_F32_NE) { deviceFormat.sampleFormat = LG_AUDIO_FMT_F32_NE; deviceConfigured = playbackSetupDevice( - &deviceFormat, requestedPeriodFrames, requestBackendResampler); + &deviceFormat, requestedPeriodFrames, requestBackendResampler, + &backendResampler); } if (!deviceConfigured) @@ -1195,8 +1214,11 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.stride = channels * audioSampleSize(deviceFormat.sampleFormat); audio.playback.convertToFloat = - !audio.playback.backendResampler || + (!providerRateControl && !backendResampler) || deviceFormat.sampleFormat != format->sampleFormat; + audio.playback.rateControl = providerRateControl ? + PLAYBACK_RATE_PROVIDER : backendResampler ? + PLAYBACK_RATE_BACKEND : PLAYBACK_RATE_SOFTWARE; audio.playback.buffer = ringbuffer_newUnbounded( sampleRate, audio.playback.stride); @@ -1207,11 +1229,11 @@ static void playbackStart(const LG_AudioFormat * format, } if (g_params.audioResampler == AUDIO_RESAMPLER_BACKEND && - !audio.playback.backendResampler) + !providerRateControl && !backendResampler) DEBUG_WARN("%s could not activate backend resampling; " "using libsamplerate", audio.audioDev->name); - if (!audio.playback.backendResampler) + if (audio.playback.rateControl == PLAYBACK_RATE_SOFTWARE) { int srcError; audio.playback.sourceData.src = @@ -1226,9 +1248,20 @@ static void playbackStart(const LG_AudioFormat * format, else audio.playback.sourceData.src = NULL; - DEBUG_INFO("Using audio resampler: %s", - audio.playback.backendResampler ? - audio.audioDev->name : "libsamplerate"); + switch (audio.playback.rateControl) + { + case PLAYBACK_RATE_PROVIDER: + DEBUG_INFO("Using audio rate control: provider feedback"); + break; + + case PLAYBACK_RATE_BACKEND: + DEBUG_INFO("Using audio resampler: %s", audio.audioDev->name); + break; + + case PLAYBACK_RATE_SOFTWARE: + DEBUG_INFO("Using audio resampler: libsamplerate"); + break; + } // if a volume level was stored, set it before we return if (audio.playback.volumeChannels) @@ -1329,6 +1362,19 @@ static double computeDevicePosition(int64_t curTime) sourceData->devicePositionOffsetFrames; } +static double playbackProviderRate(const PlaybackSourceData * sourceData) +{ + const double nominalRate = audio.playback.sampleRate; + if (!sourceData->deviceClock.valid || + sourceData->deviceClock.frameSec <= 0.0) + return nominalRate; + + return clamp( + sourceData->lastRatio / sourceData->deviceClock.frameSec, + nominalRate * (1.0 - PLAYBACK_MAX_RATE_CORRECTION), + nominalRate * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)); +} + static bool playbackEnsureConversionBuffers( PlaybackSourceData * sourceData, int frames) { @@ -1347,7 +1393,7 @@ static bool playbackEnsureConversionBuffers( sourceData->framesInSize = frames; } - if (!audio.playback.backendResampler) + if (audio.playback.rateControl == PLAYBACK_RATE_SOFTWARE) { const int framesOut = (int)ceil(frames * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) + 64; @@ -1427,7 +1473,7 @@ static void playbackData(const void * data, size_t frameCount, if (state == STREAM_STATE_STOP || !audio.audioDev || frameCount == 0) return; - if (audio.playback.backendResampler && + if (audio.playback.rateControl == PLAYBACK_RATE_BACKEND && atomic_exchange_explicit( &audio.playback.backendResamplerFailed, false, memory_order_acq_rel)) @@ -1441,7 +1487,8 @@ static void playbackData(const void * data, size_t frameCount, /* Backend resampling changes how many source frames PipeWire requests per * device period. Use the command-normalized output clock for rate matching, * while deviceClock remains in the ring's source-frame domain for latency. */ - const PlaybackClock * rateClock = audio.playback.backendResampler ? + const PlaybackClock * rateClock = + audio.playback.rateControl == PLAYBACK_RATE_BACKEND ? &sourceData->outputClock : &sourceData->deviceClock; const int64_t now = nanotime(); const double nominalFrameSec = 1.0 / audio.playback.sampleRate; @@ -1506,7 +1553,13 @@ static void playbackData(const void * data, size_t frameCount, const bool sourceRateWasValid = sourceData->sourceRateValid; - playbackSourceRateAdd(sourceData, nominalFrameSec); + const bool providerRateControl = + audio.playback.rateControl == PLAYBACK_RATE_PROVIDER; + const int64_t sourceRateTimeMs = providerRateControl ? + (now - sourceData->mediaLocalOrigin) / INT64_C(1000000) : + sourceData->mediaTimeMs; + playbackSourceRateAdd( + sourceData, sourceRateTimeMs, nominalFrameSec); if (sourceClock && sourceClock->stable && sourceClock->rate > 0.0) { const double frameSec = 1.0 / sourceClock->rate; @@ -1524,7 +1577,7 @@ static void playbackData(const void * data, size_t frameCount, if (!playbackSourceClockUpdate(&sourceData->sourceClock, packetTime, sourceData->inputPosition, nominalFrameSec)) discontinuity = true; - if (sourceData->sourceRateValid) + if (sourceData->sourceRateValid && !providerRateControl) sourceData->sourceClock.frameSec = sourceData->sourceRateFrameSec; /* Track phase variation around its local baseline, not its absolute value. @@ -1587,7 +1640,7 @@ static void playbackData(const void * data, size_t frameCount, playbackClockUpdate(&sourceData->deviceClock, deviceTick.nextTime, deviceTick.nextPosition, nominalFrameSec); const bool outputClockUpdated = - !audio.playback.backendResampler || + audio.playback.rateControl != PLAYBACK_RATE_BACKEND || playbackClockUpdate(&sourceData->outputClock, deviceTick.nextTime, deviceTick.outputPosition, nominalFrameSec); @@ -1608,11 +1661,12 @@ static void playbackData(const void * data, size_t frameCount, * switching models would create a false phase step and drive the resampler * despite an already-correct ring level. Keep the source clock untouched: * changing it would also disturb source phase and jitter tracking. */ + const int64_t referenceTime = providerRateControl ? now : curTime; const double rawDevicePosition = - playbackClockPosition(&sourceData->deviceClock, curTime); + playbackClockPosition(&sourceData->deviceClock, referenceTime); sourceData->devicePositionOffsetFrames = - sourceData->devReadPosition - sourceReserveFrames - - rawDevicePosition; + sourceData->devReadPosition - rawDevicePosition - + (providerRateControl ? 0.0 : sourceReserveFrames); } const int maxPeriodFrames = @@ -1637,8 +1691,8 @@ static void playbackData(const void * data, size_t frameCount, const double targetBufferFrames = minimumBufferFrames + latencyOffsetFrames; const double resamplerDelayFrames = - audio.playback.backendResampler ? - 0.0 : PLAYBACK_RESAMPLER_DELAY_FRAMES; + audio.playback.rateControl == PLAYBACK_RATE_SOFTWARE ? + PLAYBACK_RESAMPLER_DELAY_FRAMES : 0.0; const double minimumLatencyFrames = minimumBufferFrames + resamplerDelayFrames; const double targetLatencyFrames = @@ -1646,9 +1700,27 @@ static void playbackData(const void * data, size_t frameCount, double devPosition = DBL_MIN; state = playbackGetState(); - if ((discontinuity || + if (providerRateControl && + (discontinuity || state == STREAM_STATE_KEEP_ALIVE || - state == STREAM_STATE_RESUMING) && + state == STREAM_STATE_RESUMING)) + { + const int occupancy = ringbuffer_getCount(audio.playback.buffer); + const int slewFrames = clamp( + llrint(targetLowWaterFrames - occupancy), + (int64_t)INT_MIN, (int64_t)INT_MAX); + const int actualSlew = playbackSlewBuffer(sourceData, slewFrames); + sourceData->outputPosition += actualSlew; + curPosition += actualSlew; + + sourceData->offsetError = 0.0; + sourceData->offsetErrorIntegral = 0.0; + sourceData->ratioIntegral = 0.0; + playbackSetState(STREAM_STATE_RUN); + } + else if ((discontinuity || + state == STREAM_STATE_KEEP_ALIVE || + state == STREAM_STATE_RESUMING) && sourceData->deviceClock.valid && sourceData->deviceClockStable) { @@ -1668,7 +1740,25 @@ static void playbackData(const void * data, size_t frameCount, double actualLatencyFrames = 0.0; double actualOffsetError = 0.0; - if (sourceData->deviceClock.valid) + if (providerRateControl) + { + const int occupancy = ringbuffer_getCount(audio.playback.buffer); + actualLatencyFrames = occupancy + sourceReserveFrames; + actualOffsetError = targetLowWaterFrames - occupancy; + + const double error = + actualOffsetError - sourceData->offsetError; + const double periodSec = frames * nominalFrameSec; + const double omega = + 2.0 * M_PI * PLAYBACK_OFFSET_FILTER_BANDWIDTH_HZ * periodSec; + const double b = M_SQRT2 * omega; + const double c = omega * omega; + + sourceData->offsetError += b * error + + sourceData->offsetErrorIntegral; + sourceData->offsetErrorIntegral += c * error; + } + else if (sourceData->deviceClock.valid) { if (sourceData->deviceClockStable) { @@ -1722,10 +1812,11 @@ static void playbackData(const void * data, size_t frameCount, 2.0 * naturalFrequency / audio.playback.sampleRate; const double ki = naturalFrequency * naturalFrequency / audio.playback.sampleRate; - if (sourceRateBecameValid) + if (sourceRateBecameValid && !providerRateControl) sourceData->ratioIntegral = 0.0; - if (sourceData->deviceClockStable && + if (!providerRateControl && + sourceData->deviceClockStable && sourceData->sourceRateValid && rateClock->updates >= 2) { @@ -1755,7 +1846,8 @@ static void playbackData(const void * data, size_t frameCount, double controllerKp = kp; double controllerKi = ki; double controllerError = phaseError; - double controllerBase = sourceData->lastClockRatio; + double controllerBase = providerRateControl ? + 1.0 : sourceData->lastClockRatio; if (acquiringDeviceClock) { @@ -1811,7 +1903,7 @@ static void playbackData(const void * data, size_t frameCount, sourceData->lastRatio + maxRatioStep); sourceData->lastRatio = ratio; - if (audio.playback.backendResampler) + if (audio.playback.rateControl == PLAYBACK_RATE_BACKEND) { atomic_store_explicit( &audio.playback.backendResampleRatio, ratio, @@ -1820,6 +1912,12 @@ static void playbackData(const void * data, size_t frameCount, playbackAppendFrames(sourceData, inputFrames, frames); sourceData->outputPosition += outputFrames; } + else if (audio.playback.rateControl == PLAYBACK_RATE_PROVIDER) + { + const int outputFrames = + playbackAppendFrames(sourceData, inputFrames, frames); + sourceData->outputPosition += outputFrames; + } else { int consumed = 0; @@ -1887,7 +1985,7 @@ static void playbackData(const void * data, size_t frameCount, const float graphMax = targetLatencyFrames * 1000.0 / audio.playback.sampleRate * 2; - audio.playback.graph = app_registerGraph("PLAYBACK", + audio.playback.graph = app_registerGraph("PLAYBACK RING", audio.playback.timings, 0.0f, graphMax, audioGraphFormatFn); if (!audio.playback.graph) @@ -1929,21 +2027,36 @@ static void playbackData(const void * data, size_t frameCount, if (now >= sourceData->nextLogTime) { + const double backendLatencyMs = + audio.audioDev->playback.latency ? + audio.audioDev->playback.latency() / 1000.0 : 0.0; const double sourcePpm = sourceData->sourceRateValid ? - (sourceData->sourceRateFrameSec / nominalFrameSec - 1.0) * 1.0e6 : + (nominalFrameSec / sourceData->sourceRateFrameSec - 1.0) * 1.0e6 : 0.0; const double devicePpm = rateClock->valid ? - (rateClock->frameSec / nominalFrameSec - 1.0) * 1.0e6 : + (nominalFrameSec / rateClock->frameSec - 1.0) * 1.0e6 : 0.0; + const bool providerControl = + audio.playback.rateControl == PLAYBACK_RATE_PROVIDER; + const double controlPpm = providerControl ? + (playbackProviderRate(sourceData) / + audio.playback.sampleRate - 1.0) * 1.0e6 : + (ratio - 1.0) * 1.0e6; + const char * controlName = providerControl ? "feedback" : + audio.playback.rateControl == PLAYBACK_RATE_BACKEND ? + "backend" : "software"; + const char * sourceRateName = providerControl ? "arrival" : "source"; const unsigned int underruns = atomic_exchange_explicit( &audio.playback.underruns, 0, memory_order_relaxed); DEBUG_INFO( - "Audio sync: %.2f/%.2f ms, ratio %+.1f ppm, " - "clocks %+.1f/%+.1f ppm, jitter %.2f ms, xruns %u/%u", + "Audio sync: ring %.2f/%.2f ms, backend %.2f ms, " + "%s %+.1f ppm, rates %s/device %+.1f/%+.1f ppm, " + "jitter %.2f ms, xruns %u/%u", softwareLatencyMs, targetLatencyFrames * 1000.0 / audio.playback.sampleRate, - (ratio - 1.0) * 1.0e6, sourcePpm, devicePpm, + backendLatencyMs, controlName, controlPpm, + sourceRateName, sourcePpm, devicePpm, sourceData->arrivalJitterSec * 1000.0, underruns, sourceData->bufferOverruns); @@ -1952,24 +2065,37 @@ static void playbackData(const void * data, size_t frameCount, } } -static bool playbackGetFeedback(LG_AudioClock * clock) +static bool playbackGetFeedback( + LG_AudioClock * clock, double * targetRate) { - const PlaybackSourceData * sourceData = &audio.playback.sourceData; - if (!clock || !sourceData->deviceClock.valid || - sourceData->deviceClock.position < 0.0 || + PlaybackSourceData * sourceData = &audio.playback.sourceData; + if (!clock || !targetRate || + audio.playback.rateControl != PLAYBACK_RATE_PROVIDER || + !sourceData->deviceClock.valid || sourceData->deviceClock.frameSec <= 0.0) return false; + const int64_t now = nanotime(); + if (now < sourceData->nextFeedbackTime) + return false; + + sourceData->nextFeedbackTime = now + PLAYBACK_FEEDBACK_INTERVAL_NS; + const double position = + playbackClockPosition(&sourceData->deviceClock, now); + if (position < 0.0) + return false; + const uint64_t latency = audio.audioDev->playback.latency ? audio.audioDev->playback.latency() : 0; *clock = (LG_AudioClock) { - .position = llrint(sourceData->deviceClock.position), - .time = sourceData->deviceClock.time + latency * 1000, + .position = llrint(position), + .time = now + latency * 1000, .rate = 1.0 / sourceData->deviceClock.frameSec, .stable = sourceData->deviceClockStable, }; + *targetRate = playbackProviderRate(sourceData); return true; } @@ -2249,18 +2375,19 @@ static bool bindingActiveNL(const AudioBinding * binding) static void queueFeedback(const LG_AudioOps * ops, void * opaque, uint32_t bindingGeneration, uint32_t generation, - const LG_AudioClock * clock) + const LG_AudioClock * clock, double targetRate) { if (!audio.feedback.event || !audio.feedback.thread || !clock) return; LG_LOCK(audio.feedback.lock); - audio.feedback.ops = ops; - audio.feedback.opaque = opaque; + audio.feedback.ops = ops; + audio.feedback.opaque = opaque; audio.feedback.bindingGeneration = bindingGeneration; - audio.feedback.generation = generation; - audio.feedback.clock = *clock; - audio.feedback.pending = true; + audio.feedback.generation = generation; + audio.feedback.clock = *clock; + audio.feedback.targetRate = targetRate; + audio.feedback.pending = true; LG_UNLOCK(audio.feedback.lock); lgSignalEvent(audio.feedback.event); } @@ -2276,7 +2403,9 @@ static void eventPlaybackStart(void * opaque, uint32_t generation, LG_LOCK(audio.playback.sourceLock); atomic_store_explicit(&audio.playback.streamGeneration, generation, memory_order_release); - playbackStart(format, sourceClock); + playbackStart(format, sourceClock, + binding->ops->clockFeedback && + audio.feedback.event && audio.feedback.thread); LG_UNLOCK(audio.playback.sourceLock); } LG_UNLOCK_SHARED(audio.activeLock); @@ -2356,9 +2485,11 @@ static void eventPlaybackData(void * opaque, uint32_t generation, playbackData(data, frames, sourceClock); LG_AudioClock feedback; - if (ops->clockFeedback && playbackGetFeedback(&feedback)) + double targetRate; + if (ops->clockFeedback && + playbackGetFeedback(&feedback, &targetRate)) queueFeedback(ops, providerOpaque, binding->generation, - generation, &feedback); + generation, &feedback, targetRate); } LG_UNLOCK(audio.playback.sourceLock); } @@ -2616,18 +2747,20 @@ static int feedbackThread(void * opaque) break; const LG_AudioOps * ops; - void * providerOpaque; - uint32_t bindingGeneration; - uint32_t generation; - LG_AudioClock clock; + void * providerOpaque; + uint32_t bindingGeneration; + uint32_t generation; + LG_AudioClock clock; + double targetRate; LG_LOCK(audio.feedback.lock); - const bool pending = audio.feedback.pending; - ops = audio.feedback.ops; - providerOpaque = audio.feedback.opaque; - bindingGeneration = audio.feedback.bindingGeneration; - generation = audio.feedback.generation; - clock = audio.feedback.clock; + const bool pending = audio.feedback.pending; + ops = audio.feedback.ops; + providerOpaque = audio.feedback.opaque; + bindingGeneration = audio.feedback.bindingGeneration; + generation = audio.feedback.generation; + clock = audio.feedback.clock; + targetRate = audio.feedback.targetRate; audio.feedback.pending = false; LG_UNLOCK(audio.feedback.lock); @@ -2640,7 +2773,8 @@ static int feedbackThread(void * opaque) audio.active.generation == bindingGeneration && atomic_load_explicit(&audio.playback.streamGeneration, memory_order_acquire) == generation) - ops->clockFeedback(providerOpaque, generation, &clock); + ops->clockFeedback( + providerOpaque, generation, &clock, targetRate); LG_UNLOCK_SHARED(audio.activeLock); } diff --git a/client/src/audio_usb.c b/client/src/audio_usb.c index f11b2fd9..007313f1 100644 --- a/client/src/audio_usb.c +++ b/client/src/audio_usb.c @@ -532,6 +532,28 @@ static void usbDetach(void * opaque) LG_UNLOCK(state->stateLock); } +static bool usbClockFeedback(void * opaque, uint32_t generation, + const LG_AudioClock * playbackClock, double targetRate) +{ + LGA_USBState * state = opaque; + + LG_LOCK(state->stateLock); + if (!state->attached || state->streamGeneration != generation) + { + LG_UNLOCK(state->stateLock); + return false; + } + + const double nominalRate = state->streamFormat.sampleRate; + const double rate = playbackClock && + targetRate >= nominalRate * 0.995 && + targetRate <= nominalRate * 1.005 ? + targetRate : nominalRate; + lgUsbAudio_setFeedbackRate(state->device, rate); + LG_UNLOCK(state->stateLock); + return true; +} + const LG_AudioOps LGA_USB = { .name = "USB Audio", @@ -539,7 +561,7 @@ const LG_AudioOps LGA_USB = .attach = usbAttach, .detach = usbDetach, .recordData = NULL, - .clockFeedback = NULL, + .clockFeedback = usbClockFeedback, }; LGA_USBState * lgaUsb_create(void) diff --git a/client/src/usb_audio.c b/client/src/usb_audio.c index bf289206..7680d790 100644 --- a/client/src/usb_audio.c +++ b/client/src/usb_audio.c @@ -25,27 +25,35 @@ #include #include +#include #include #include #include enum { - USB_AUDIO_VENDOR_ID = 0x043e, - USB_AUDIO_PRODUCT_ID = 0x0001, - USB_AUDIO_DEVICE_VERSION = 0x010a, - USB_AUDIO_CONFIGURATION = 1, - USB_AUDIO_CONTROL_IFACE = 0, - USB_AUDIO_STREAM_IFACE = 1, - USB_AUDIO_CLOCK_ID = 1, - USB_AUDIO_STREAM_ENDPOINT = 0x01, - USB_AUDIO_EP_INTERVAL = 1, - USB_AUDIO_SAMPLE_SIZE = 3, - USB_AUDIO_HS_PACKET_FRAMES = 25, - USB_AUDIO_FS_PACKET_FRAMES = 97, - USB_AUDIO_DESCRIPTOR_BASE = 81, - USB_AUDIO_STREAM_DESC_SIZE = 46, - USB_AUDIO_OTHER_SPEED_SIZE = + USB_AUDIO_VENDOR_ID = 0x043e, + USB_AUDIO_PRODUCT_ID = 0x0001, + USB_AUDIO_DEVICE_VERSION = 0x010b, + USB_AUDIO_CONFIGURATION = 1, + USB_AUDIO_CONTROL_IFACE = 0, + USB_AUDIO_STREAM_IFACE = 1, + USB_AUDIO_CLOCK_ID = 1, + USB_AUDIO_DATA_ENDPOINT = 0x01, + USB_AUDIO_FEEDBACK_ENDPOINT = 0x81, + USB_AUDIO_DATA_EP_INDEX = 1, + USB_AUDIO_FEEDBACK_EP_INDEX = 17, + USB_AUDIO_DATA_INTERVAL = 1, + USB_AUDIO_FEEDBACK_INTERVAL = 8, + USB_AUDIO_FEEDBACK_BINTERVAL = 4, + USB_AUDIO_FEEDBACK_SIZE = 4, + USB_AUDIO_FEEDBACK_MAX_QUEUE = 512, + USB_AUDIO_SAMPLE_SIZE = 3, + USB_AUDIO_HS_PACKET_FRAMES = 25, + USB_AUDIO_FS_PACKET_FRAMES = 97, + USB_AUDIO_DESCRIPTOR_BASE = 81, + USB_AUDIO_STREAM_DESC_SIZE = 53, + USB_AUDIO_OTHER_SPEED_SIZE = USB_AUDIO_DESCRIPTOR_BASE + USB_AUDIO_STREAM_DESC_SIZE, }; @@ -134,8 +142,9 @@ static const uint8_t l_deviceDescriptor[] = 0x03, 0x01, }; -#define USB_AUDIO_STREAM_DESCRIPTOR(alt, channels, mask, packetFrames) \ - 0x09, 0x04, 0x01, alt, 0x01, 0x01, 0x02, 0x20, 0x00, \ +#define USB_AUDIO_STREAM_DESCRIPTOR(alt, channels, mask, packetFrames, \ + feedbackSize, feedbackInterval) \ + 0x09, 0x04, 0x01, alt, 0x02, 0x01, 0x02, 0x20, 0x00, \ 0x10, 0x24, 0x01, 0x02, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, \ channels, \ (mask) & 0xff, \ @@ -144,15 +153,18 @@ static const uint8_t l_deviceDescriptor[] = (mask) >> 24 & 0xff, \ 0x00, \ 0x06, 0x24, 0x02, 0x01, 0x03, 0x18, \ - 0x07, 0x05, 0x01, 0x09, \ + 0x07, 0x05, USB_AUDIO_DATA_ENDPOINT, 0x05, \ USB_AUDIO_PACKET_SIZE(packetFrames, channels) & 0xff, \ USB_AUDIO_PACKET_SIZE(packetFrames, channels) >> 8 & 0xff, \ - USB_AUDIO_EP_INTERVAL, \ - 0x08, 0x25, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + USB_AUDIO_DATA_INTERVAL, \ + 0x08, 0x25, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, \ + 0x07, 0x05, USB_AUDIO_FEEDBACK_ENDPOINT, 0x11, \ + feedbackSize, 0x00, feedbackInterval, #define USB_AUDIO_HS_STREAM_DESCRIPTOR(alt, channels, mask) \ USB_AUDIO_STREAM_DESCRIPTOR( \ - alt, channels, mask, USB_AUDIO_HS_PACKET_FRAMES) + alt, channels, mask, USB_AUDIO_HS_PACKET_FRAMES, \ + USB_AUDIO_FEEDBACK_SIZE, USB_AUDIO_FEEDBACK_BINTERVAL) static const uint8_t l_configurationDescriptor[] = { @@ -186,7 +198,7 @@ static const uint8_t l_configurationDescriptor[] = static const uint8_t l_fullSpeedStreamDescriptor[] = { USB_AUDIO_STREAM_DESCRIPTOR(1, 2, USB_AUDIO_LAYOUT_STEREO, - USB_AUDIO_FS_PACKET_FRAMES) + USB_AUDIO_FS_PACKET_FRAMES, 3, 1) }; #undef USB_AUDIO_STREAM_DESCRIPTOR #undef USB_AUDIO_PACKET_SIZE @@ -235,7 +247,11 @@ struct LG_USBAudio uint8_t configuration; uint8_t streamAlt; uint32_t sampleRate; - bool streaming; + bool dataStreaming; + bool feedbackStreaming; + uint64_t feedbackPacketId; + uint8_t feedbackDataPackets; + atomic_uint feedbackValue; }; static LG_USBAudio * getAudio(void * opaque) @@ -266,6 +282,17 @@ static void writeLE32(uint8_t * data, uint32_t value) data[3] = value >> 24; } +static uint32_t encodeFeedbackRate(double sampleRate) +{ + return (uint32_t)(sampleRate * UINT32_C(65536) / 8000.0 + 0.5); +} + +static void resetFeedbackRate(LG_USBAudio * audio) +{ + atomic_store_explicit(&audio->feedbackValue, + encodeFeedbackRate(audio->sampleRate), memory_order_release); +} + static uint32_t nearestSampleRate(uint32_t sampleRate) { uint32_t selected = l_sampleRates[0]; @@ -320,21 +347,35 @@ static size_t writeSampleRateRange(uint8_t * buffer) static void stopPlayback(LG_USBAudio * audio) { - if (!audio->streaming) + if (!audio->dataStreaming) return; - audio->streaming = false; + audio->dataStreaming = false; + resetFeedbackRate(audio); if (audio->events && audio->events->stop) audio->events->stop(audio->eventOpaque); } +static void stopFeedback(LG_USBAudio * audio) +{ + audio->feedbackStreaming = false; + audio->feedbackDataPackets = 0; +} + +static void stopStreams(LG_USBAudio * audio) +{ + stopPlayback(audio); + stopFeedback(audio); +} + static void startPlayback(LG_USBAudio * audio) { const USBAudioLayout * layout = getLayout(audio->streamAlt); - if (audio->streaming || !layout) + if (audio->dataStreaming || !layout) return; - audio->streaming = true; + resetFeedbackRate(audio); + audio->dataStreaming = true; if (audio->events && audio->events->start) audio->events->start( audio->eventOpaque, audio->sampleRate, layout->channelMask); @@ -345,20 +386,22 @@ static void setSampleRate(LG_USBAudio * audio, uint32_t sampleRate) if (audio->sampleRate == sampleRate) return; - const bool restart = audio->streaming; + const bool restart = audio->dataStreaming; stopPlayback(audio); audio->sampleRate = sampleRate; + resetFeedbackRate(audio); if (restart) startPlayback(audio); } static void resetDevice(LG_USBAudio * audio) { - stopPlayback(audio); + stopStreams(audio); audio->configuration = 0; audio->streamAlt = 0; audio->sampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; + resetFeedbackRate(audio); } static void sendInterfaceInfo(LG_USBAudio * audio) @@ -387,11 +430,19 @@ static void sendEndpointInfo(LG_USBAudio * audio) if (audio->configuration == USB_AUDIO_CONFIGURATION && layout) { - info.type[USB_AUDIO_STREAM_ENDPOINT] = usb_redir_type_iso; - info.interval[USB_AUDIO_STREAM_ENDPOINT] = USB_AUDIO_EP_INTERVAL; - info.interface[USB_AUDIO_STREAM_ENDPOINT] = USB_AUDIO_STREAM_IFACE; - info.max_packet_size[USB_AUDIO_STREAM_ENDPOINT] = + info.type[USB_AUDIO_DATA_EP_INDEX] = usb_redir_type_iso; + info.interval[USB_AUDIO_DATA_EP_INDEX] = USB_AUDIO_DATA_INTERVAL; + info.interface[USB_AUDIO_DATA_EP_INDEX] = USB_AUDIO_STREAM_IFACE; + info.max_packet_size[USB_AUDIO_DATA_EP_INDEX] = layoutPacketSize(layout); + + info.type[USB_AUDIO_FEEDBACK_EP_INDEX] = usb_redir_type_iso; + info.interval[USB_AUDIO_FEEDBACK_EP_INDEX] = + USB_AUDIO_FEEDBACK_INTERVAL; + info.interface[USB_AUDIO_FEEDBACK_EP_INDEX] = + USB_AUDIO_STREAM_IFACE; + info.max_packet_size[USB_AUDIO_FEEDBACK_EP_INDEX] = + USB_AUDIO_FEEDBACK_SIZE; } usbredirparser_send_ep_info(audio->parser, &info); @@ -462,7 +513,7 @@ static void setAltSetting(void * opaque, uint64_t id, request->interface == USB_AUDIO_STREAM_IFACE && (request->alt == 0 || getLayout(request->alt))) { - stopPlayback(audio); + stopStreams(audio); audio->streamAlt = request->alt; status.status = usb_redir_success; } @@ -507,31 +558,80 @@ static void sendISOStatus(LG_USBAudio * audio, uint64_t id, usbredirparser_send_iso_stream_status(audio->parser, id, &status); } +static void sendFeedbackPackets(LG_USBAudio * audio, uint32_t count) +{ + uint8_t data[USB_AUDIO_FEEDBACK_SIZE]; + writeLE32(data, atomic_load_explicit( + &audio->feedbackValue, memory_order_acquire)); + struct usb_redir_iso_packet_header packet = + { + .endpoint = USB_AUDIO_FEEDBACK_ENDPOINT, + .status = usb_redir_success, + .length = sizeof(data), + }; + + for (uint32_t i = 0; i < count; ++i) + usbredirparser_send_iso_packet(audio->parser, + audio->feedbackPacketId++, + &packet, data, USB_AUDIO_FEEDBACK_SIZE); +} + static void startISOStream(void * opaque, uint64_t id, struct usb_redir_start_iso_stream_header * request) { - LG_USBAudio * audio = getAudio(opaque); - uint8_t result = usb_redir_stall; + LG_USBAudio * audio = getAudio(opaque); + uint8_t result = usb_redir_stall; + uint32_t feedbackPrefill = 0; - if (request->endpoint == USB_AUDIO_STREAM_ENDPOINT && - audio->configuration == USB_AUDIO_CONFIGURATION && + if (audio->configuration == USB_AUDIO_CONFIGURATION && getLayout(audio->streamAlt)) { - result = usb_redir_success; - startPlayback(audio); + switch (request->endpoint) + { + case USB_AUDIO_DATA_ENDPOINT: + result = usb_redir_success; + startPlayback(audio); + break; + + case USB_AUDIO_FEEDBACK_ENDPOINT: + resetFeedbackRate(audio); + audio->feedbackStreaming = true; + audio->feedbackDataPackets = 0; + audio->feedbackPacketId = 0; + feedbackPrefill = + (uint32_t)request->pkts_per_urb * request->no_urbs; + if (!feedbackPrefill) + feedbackPrefill = 1; + else if (feedbackPrefill > + USB_AUDIO_FEEDBACK_MAX_QUEUE) + feedbackPrefill = USB_AUDIO_FEEDBACK_MAX_QUEUE; + result = usb_redir_success; + break; + } } sendISOStatus(audio, id, request->endpoint, result); + if (feedbackPrefill) + sendFeedbackPackets(audio, feedbackPrefill); } static void stopISOStream(void * opaque, uint64_t id, struct usb_redir_stop_iso_stream_header * request) { LG_USBAudio * audio = getAudio(opaque); - const uint8_t result = request->endpoint == USB_AUDIO_STREAM_ENDPOINT ? - usb_redir_success : usb_redir_stall; - if (result == usb_redir_success) - stopPlayback(audio); + uint8_t result = usb_redir_stall; + switch (request->endpoint) + { + case USB_AUDIO_DATA_ENDPOINT: + stopPlayback(audio); + result = usb_redir_success; + break; + + case USB_AUDIO_FEEDBACK_ENDPOINT: + stopFeedback(audio); + result = usb_redir_success; + break; + } sendISOStatus(audio, id, request->endpoint, result); } @@ -556,7 +656,7 @@ static size_t stringDescriptor(uint8_t index, uint8_t * buffer, { case 1: text = "Looking Glass" ; break; case 2: text = "Looking Glass USB Audio"; break; - case 3: text = "LG-UAC2-0003" ; break; + case 3: text = "LG-UAC2-0004" ; break; default: return 0; } @@ -695,7 +795,8 @@ static void controlPacket(void * opaque, uint64_t id, valid = request->index == 0x00 || request->index == 0x80 || (audio->configuration == USB_AUDIO_CONFIGURATION && getLayout(audio->streamAlt) && - request->index == USB_AUDIO_STREAM_ENDPOINT); + (request->index == USB_AUDIO_DATA_ENDPOINT || + request->index == USB_AUDIO_FEEDBACK_ENDPOINT)); break; } @@ -756,7 +857,7 @@ static void controlPacket(void * opaque, uint64_t id, static void stallStream(LG_USBAudio * audio) { stopPlayback(audio); - sendISOStatus(audio, 0, USB_AUDIO_STREAM_ENDPOINT, usb_redir_stall); + sendISOStatus(audio, 0, USB_AUDIO_DATA_ENDPOINT, usb_redir_stall); } static void isoPacket(void * opaque, uint64_t id, @@ -767,11 +868,11 @@ static void isoPacket(void * opaque, uint64_t id, LG_USBAudio * audio = getAudio(opaque); const USBAudioLayout * layout = getLayout(audio->streamAlt); - if (audio->streaming) + if (audio->dataStreaming) { const uint16_t frameSize = layout ? layout->channelCount * USB_AUDIO_SAMPLE_SIZE : 0; - if (packet->endpoint != USB_AUDIO_STREAM_ENDPOINT || + if (packet->endpoint != USB_AUDIO_DATA_ENDPOINT || packet->status != usb_redir_success || dataLength < 0 || packet->length != dataLength || !layout || dataLength > layoutPacketSize(layout) || @@ -780,9 +881,19 @@ static void isoPacket(void * opaque, uint64_t id, DEBUG_WARN("Invalid USB audio isochronous packet"); stallStream(audio); } - else if (dataLength && audio->events && audio->events->data) - audio->events->data(audio->eventOpaque, data, - dataLength / frameSize); + else + { + if (dataLength && audio->events && audio->events->data) + audio->events->data(audio->eventOpaque, data, + dataLength / frameSize); + + if (audio->feedbackStreaming && + ++audio->feedbackDataPackets == USB_AUDIO_FEEDBACK_INTERVAL) + { + audio->feedbackDataPackets = 0; + sendFeedbackPackets(audio, 1); + } + } } if (data) @@ -847,9 +958,21 @@ LG_USBAudio * lgUsbAudio_create( audio->events = events; audio->eventOpaque = eventOpaque; + audio->sampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; + atomic_init(&audio->feedbackValue, + encodeFeedbackRate(audio->sampleRate)); return audio; } +void lgUsbAudio_setFeedbackRate(LG_USBAudio * audio, double sampleRate) +{ + if (!audio) + return; + + atomic_store_explicit(&audio->feedbackValue, + encodeFeedbackRate(sampleRate), memory_order_release); +} + void lgUsbAudio_destroy(LG_USBAudio * audio) { if (!audio) diff --git a/client/src/usb_audio.h b/client/src/usb_audio.h index a81e22cc..7280b394 100644 --- a/client/src/usb_audio.h +++ b/client/src/usb_audio.h @@ -47,6 +47,10 @@ LG_USBAudio * lgUsbAudio_create( /* Destroy the LG_USBRedir using this device before destroying the device. */ void lgUsbAudio_destroy(LG_USBAudio * audio); +/* Publish the requested source rate without touching usbredir from the audio + * feedback thread. */ +void lgUsbAudio_setFeedbackRate(LG_USBAudio * audio, double sampleRate); + const LG_USBRedirDeviceOps * lgUsbAudio_deviceOps(void); #endif