From 87aa61510c87e776c88b82b0b59f419b72c377f8 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Mon, 10 Aug 2026 16:36:12 +1000 Subject: [PATCH] [client] audio: harden low-latency streaming Rework audio provider and backend lifecycles so playback and capture callbacks quiesce without blocking real-time threads. Move activation, teardown, controls, retries, and diagnostics onto bounded workers. Harden USB audio cadence, feedback, and capture recovery. Preserve source clocks through recording and pace packets from the device clock. Bound queues, waits, conversion buffers, and packet sizes. Make PipeWire and PulseAudio stream control thread-safe and recoverable. Correct latency clock domains, coalesce rate updates, preserve recent capture under overload, and keep logging outside real-time callbacks. --- client/audiodevs/PipeWire/pipewire.c | 1205 +++++++++-- client/audiodevs/PulseAudio/pulseaudio.c | 704 +++++-- client/include/interface/audio.h | 7 +- client/include/interface/audiodev.h | 26 +- client/src/audio.c | 2428 +++++++++++++++++----- client/src/audio_spice.c | 136 +- client/src/audio_usb.c | 550 +++-- client/src/audio_usb.h | 6 +- client/src/main.c | 14 +- client/src/overlay/status.c | 10 +- client/src/usb_audio.c | 808 +++++-- client/src/usb_audio.h | 17 +- client/src/usbredir.c | 101 +- 13 files changed, 4795 insertions(+), 1217 deletions(-) diff --git a/client/audiodevs/PipeWire/pipewire.c b/client/audiodevs/PipeWire/pipewire.c index ceaa895f..62d77367 100644 --- a/client/audiodevs/PipeWire/pipewire.c +++ b/client/audiodevs/PipeWire/pipewire.c @@ -21,6 +21,7 @@ #include "interface/audiodev.h" #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -38,29 +40,54 @@ #include "common/option.h" #include "common/ringbuffer.h" #include "common/thread.h" +#include "common/time.h" typedef enum { STREAM_STATE_INACTIVE, - STREAM_STATE_ACTIVE, - STREAM_STATE_DRAINING + STREAM_STATE_ACTIVE } StreamState; +typedef enum +{ + RECORD_LATEST_FREE, + RECORD_LATEST_WRITING, + RECORD_LATEST_READY, + RECORD_LATEST_READING +} +RecordLatestState; + +typedef struct +{ + int frames; + bool clockValid; + LG_AudioClock clock; +} +RecordBlock; + +#define RECORD_LATEST_SLOTS 2 +#define RECORD_CLOCK_TOLERANCE_FRAMES 2.0 +#define RECORD_ERROR_REPORT_INTERVAL_NS INT64_C(5000000000) +#define PIPEWIRE_CONNECT_TIMEOUT_NS INT64_C(5000000000) + struct PipeWire { struct pw_loop * loop; struct pw_context * context; + struct pw_core * core; struct pw_thread_loop * thread; struct { struct pw_stream * stream; - struct spa_io_rate_match * rateMatch; - _Atomic(int64_t) presentationDeadline; + _Atomic(struct spa_io_rate_match *) rateMatch; + _Atomic(int64_t) latencyNs; atomic_bool latencyUpdateRequested; atomic_uint bufferErrors; + atomic_uint pullErrors; atomic_uint timingErrors; + atomic_int streamError; #if PW_CHECK_VERSION(1, 4, 0) double appliedResampleRatio; atomic_int resampleError; @@ -71,32 +98,63 @@ struct PipeWire LG_AudioFormat format; int stride; LG_AudioPullFn pullFn; + LG_AudioFailureFn failureFn; + uint32_t failureCookie; int maxPeriodFrames; int startFrames; - StreamState state; + atomic_uint state; } playback; struct { struct pw_stream * stream; + enum pw_stream_state connectionState; LG_AudioFormat format; int stride; LG_AudioPushFn pushFn; + LG_AudioFailureFn failureFn; + uint32_t failureCookie; RingBuffer sendQueue; + RingBuffer sendBlocks; uint8_t * sendBuffer; int sendBufferFrames; + int maxQueuedFrames; sem_t sendWake; bool sendWakeInitialized; LGThread * sendThread; atomic_bool sendStop; + atomic_bool sendWakePending; + atomic_bool latestMode; + atomic_uint_fast64_t latestSerial; + uint8_t * latestBuffer; + struct + { + atomic_uint state; + atomic_uint_fast64_t serial; + int frames; + bool clockValid; + LG_AudioClock clock; + } + latest[RECORD_LATEST_SLOTS]; atomic_uint bufferErrors; atomic_uint droppedFrames; + atomic_uint rejectedBatches; + atomic_uint rejectedFrames; atomic_uint signalErrors; + atomic_int recycleError; + uint64_t clockPosition; + uint64_t clockLastPosition; + uint64_t clockLastTicks; + int64_t clockLastTime; + struct spa_fraction clockLastRate; + bool clockLastValid; + bool clockLastTimingValid; + bool clockDiscontinuityPending; bool active; } record; @@ -104,6 +162,17 @@ struct PipeWire static struct PipeWire pw = {0}; +static inline void pipewire_storeError(atomic_int * target, int result) +{ + if (result >= 0) + return; + + int expected = 0; + atomic_compare_exchange_strong_explicit( + target, &expected, result, + memory_order_relaxed, memory_order_relaxed); +} + static bool pipewire_audioFormatEqual(const LG_AudioFormat * a, const LG_AudioFormat * b) { @@ -198,8 +267,12 @@ static void pipewire_reportPlaybackErrors(void) { const unsigned int bufferErrors = atomic_exchange_explicit( &pw.playback.bufferErrors, 0, memory_order_relaxed); + const unsigned int pullErrors = atomic_exchange_explicit( + &pw.playback.pullErrors, 0, memory_order_relaxed); const unsigned int timingErrors = atomic_exchange_explicit( &pw.playback.timingErrors, 0, memory_order_relaxed); + const int streamError = atomic_exchange_explicit( + &pw.playback.streamError, 0, memory_order_relaxed); #if PW_CHECK_VERSION(1, 4, 0) const int resampleError = atomic_exchange_explicit( &pw.playback.resampleError, 0, memory_order_relaxed); @@ -208,9 +281,15 @@ static void pipewire_reportPlaybackErrors(void) if (bufferErrors) DEBUG_WARN("PipeWire playback ran out of buffers %u time(s)", bufferErrors); + if (pullErrors) + DEBUG_WARN("PipeWire playback returned an invalid frame count %u time(s)", + pullErrors); if (timingErrors) DEBUG_WARN("PipeWire playback timing query failed %u time(s)", timingErrors); + if (streamError) + DEBUG_WARN("PipeWire playback stream operation failed: %s", + spa_strerror(streamError)); #if PW_CHECK_VERSION(1, 4, 0) if (resampleError) DEBUG_WARN("PipeWire resampler rate update failed: %s", @@ -224,25 +303,49 @@ static void pipewire_reportRecordErrors(void) &pw.record.bufferErrors, 0, memory_order_relaxed); const unsigned int droppedFrames = atomic_exchange_explicit( &pw.record.droppedFrames, 0, memory_order_relaxed); + const unsigned int rejectedBatches = atomic_exchange_explicit( + &pw.record.rejectedBatches, 0, memory_order_relaxed); + const unsigned int rejectedFrames = atomic_exchange_explicit( + &pw.record.rejectedFrames, 0, memory_order_relaxed); const unsigned int signalErrors = atomic_exchange_explicit( &pw.record.signalErrors, 0, memory_order_relaxed); + const int recycleError = atomic_exchange_explicit( + &pw.record.recycleError, 0, memory_order_relaxed); if (bufferErrors) DEBUG_WARN("PipeWire recording encountered %u buffer error(s)", bufferErrors); if (droppedFrames) DEBUG_WARN("PipeWire recording dropped %u frame(s)", droppedFrames); + if (rejectedFrames) + DEBUG_WARN("PipeWire recording provider rejected %u frame(s) " + "in %u batch(es)", rejectedFrames, rejectedBatches); if (signalErrors) DEBUG_WARN("PipeWire recording worker notification failed %u time(s)", signalErrors); + if (recycleError) + DEBUG_WARN("PipeWire recording buffer recycle failed: %s", + spa_strerror(recycleError)); } +static void pipewire_reportRecordErrorsDue(uint64_t * nextReport) +{ + const uint64_t now = nanotime(); + if (now < *nextReport) + return; + + pipewire_reportRecordErrors(); + *nextReport = now + RECORD_ERROR_REPORT_INTERVAL_NS; +} + +#if !PW_CHECK_VERSION(1, 1, 0) static int64_t pipewire_monotonicTime(void) { struct timespec time; clock_gettime(CLOCK_MONOTONIC, &time); return SPA_TIMESPEC_TO_NSEC(&time); } +#endif static inline void pipewire_updatePlaybackLatency(void) { @@ -266,17 +369,27 @@ static inline void pipewire_updatePlaybackLatency(void) if (time.rate.num == 0 || time.rate.denom == 0) return; -#if PW_CHECK_VERSION(0, 3, 50) - const double latencyTicks = - time.delay + (double)time.queued + time.buffered; +#if PW_CHECK_VERSION(1, 1, 0) + const int64_t now = (int64_t)pw_stream_get_nsec(pw.playback.stream); #else - const double latencyTicks = - time.delay + (double)time.queued / pw.playback.stride; + const int64_t now = pipewire_monotonicTime(); #endif - const int64_t latencyNs = llrint(max(0.0, latencyTicks) * - time.rate.num * SPA_NSEC_PER_SEC / time.rate.denom); - atomic_store_explicit(&pw.playback.presentationDeadline, - time.now + latencyNs, memory_order_release); + const int64_t elapsedNs = max(INT64_C(0), now - time.now); + const double graphLatencyNs = + (double)time.delay * time.rate.num * + SPA_NSEC_PER_SEC / time.rate.denom - elapsedNs; +#if PW_CHECK_VERSION(0, 3, 50) + const double streamFrames = time.queued + time.buffered; +#else + const double streamFrames = + (double)time.queued / pw.playback.stride; +#endif + const double streamLatencyNs = streamFrames * SPA_NSEC_PER_SEC / + pw.playback.format.sampleRate; + const int64_t latencyNs = + llrint(max(0.0, graphLatencyNs + streamLatencyNs)); + atomic_store_explicit( + &pw.playback.latencyNs, latencyNs, memory_order_release); } #if PW_CHECK_VERSION(1, 4, 0) @@ -302,24 +415,20 @@ static bool pipewire_playbackSetRate(double * ratio) return true; } -static bool pipewire_configurePlaybackResampler(bool enable) +static bool pipewire_configurePlaybackResampler( + bool enable, bool * reusable) { - pw.playback.resamplerEnabled = false; - pw.playback.appliedResampleRatio = 0.0; - - if (!enable) - { - pw_stream_set_rate(pw.playback.stream, 0.0); - return false; - } - - const int result = pw_stream_set_rate(pw.playback.stream, 1.0); + const bool wasEnabled = pw.playback.resamplerEnabled; + const int result = pw_stream_set_rate( + pw.playback.stream, enable ? 1.0 : 0.0); + if (reusable) + *reusable = result >= 0 || !wasEnabled; if (result < 0) return false; - pw.playback.resamplerEnabled = true; - pw.playback.appliedResampleRatio = 1.0; - return true; + pw.playback.resamplerEnabled = enable; + pw.playback.appliedResampleRatio = enable ? 1.0 : 0.0; + return enable; } #endif @@ -329,7 +438,10 @@ static void pipewire_onPlaybackIoChanged(void * userdata, uint32_t id, switch (id) { case SPA_IO_RateMatch: - pw.playback.rateMatch = data; + atomic_store_explicit(&pw.playback.rateMatch, + data && size >= offsetof(struct spa_io_rate_match, size) + + sizeof(((struct spa_io_rate_match *)0)->size) ? data : NULL, + memory_order_release); break; } } @@ -338,10 +450,18 @@ static void pipewire_onPlaybackStateChanged(void * userdata, enum pw_stream_state old, enum pw_stream_state state, const char * error) { + (void)error; pw.playback.connectionState = state; - if (state == PW_STREAM_STATE_ERROR) - DEBUG_ERROR("PipeWire playback stream failed: %s", - error ? error : "unknown error"); + if ((state == PW_STREAM_STATE_ERROR || + state == PW_STREAM_STATE_UNCONNECTED) && + pw.playback.failureFn) + { + LG_AudioFailureFn failureFn = pw.playback.failureFn; + const uint32_t failureCookie = pw.playback.failureCookie; + pw.playback.failureFn = NULL; + pw.playback.failureCookie = 0; + failureFn(failureCookie); + } pw_thread_loop_signal(pw.thread, false); } @@ -359,34 +479,48 @@ static void pipewire_onPlaybackProcess(void * userdata) struct spa_buffer * sbuf = pbuf->buffer; uint8_t * dst; - if (sbuf->n_datas == 0 || !sbuf->datas[0].chunk || + if (!sbuf || sbuf->n_datas == 0 || !sbuf->datas || + !sbuf->datas[0].chunk || !(dst = sbuf->datas[0].data)) { #if PW_CHECK_VERSION(1, 4, 0) - pw_stream_return_buffer(pw.playback.stream, pbuf); + const int recycleResult = + pw_stream_return_buffer(pw.playback.stream, pbuf); #else - pw_stream_queue_buffer(pw.playback.stream, pbuf); + const int recycleResult = + pw_stream_queue_buffer(pw.playback.stream, pbuf); #endif + pipewire_storeError(&pw.playback.streamError, recycleResult); return; } int frames = sbuf->datas[0].maxsize / pw.playback.stride; - if (pw.playback.rateMatch && pw.playback.rateMatch->size > 0) - frames = min(frames, pw.playback.rateMatch->size); + struct spa_io_rate_match * rateMatch = atomic_load_explicit( + &pw.playback.rateMatch, memory_order_acquire); + if (rateMatch && rateMatch->size > 0) + frames = min(frames, rateMatch->size); #if PW_CHECK_VERSION(0, 3, 50) else if (pbuf->requested > 0) frames = min(frames, pbuf->requested); #endif - frames = pw.playback.pullFn(dst, frames); + const int requestedFrames = frames; + frames = pw.playback.pullFn(dst, requestedFrames); + if (frames < 0 || frames > requestedFrames) + { + atomic_fetch_add_explicit( + &pw.playback.pullErrors, 1, memory_order_relaxed); + frames = clamp(frames, 0, requestedFrames); + } + pipewire_updatePlaybackLatency(); if (!frames) { pbuf->size = 0; sbuf->datas[0].chunk->offset = 0; sbuf->datas[0].chunk->stride = pw.playback.stride; sbuf->datas[0].chunk->size = 0; - pw_stream_queue_buffer(pw.playback.stream, pbuf); - pipewire_updatePlaybackLatency(); + pipewire_storeError(&pw.playback.streamError, + pw_stream_queue_buffer(pw.playback.stream, pbuf)); return; } @@ -395,14 +529,8 @@ static void pipewire_onPlaybackProcess(void * userdata) sbuf->datas[0].chunk->stride = pw.playback.stride; sbuf->datas[0].chunk->size = frames * pw.playback.stride; - pw_stream_queue_buffer(pw.playback.stream, pbuf); - pipewire_updatePlaybackLatency(); -} - -static void pipewire_onPlaybackDrained(void * userdata) -{ - pw_stream_set_active(pw.playback.stream, false); - pw.playback.state = STREAM_STATE_INACTIVE; + pipewire_storeError(&pw.playback.streamError, + pw_stream_queue_buffer(pw.playback.stream, pbuf)); } static struct Option pipewire_options[] = @@ -428,11 +556,35 @@ static void pipewire_earlyInit(void) option_register(pipewire_options); } +static bool pipewire_waitForStream(enum pw_stream_state * state) +{ + int result = 0; +#if PW_CHECK_VERSION(0, 3, 7) + struct timespec deadline; + result = pw_thread_loop_get_time( + pw.thread, &deadline, PIPEWIRE_CONNECT_TIMEOUT_NS); + if (result < 0) + return false; + + while (*state == PW_STREAM_STATE_CONNECTING && result >= 0) + result = pw_thread_loop_timed_wait_full(pw.thread, &deadline); +#else + while (*state == PW_STREAM_STATE_CONNECTING && result >= 0) + result = pw_thread_loop_timed_wait(pw.thread, 5); +#endif + return result >= 0; +} + static bool pipewire_init(void) { pw_init(NULL, NULL); pw.loop = pw_loop_new(NULL); + if (!pw.loop) + { + DEBUG_ERROR("Failed to create a PipeWire loop"); + goto err; + } #if PW_CHECK_VERSION(1, 3, 81) pw.context = pw_context_new( pw.loop, @@ -455,8 +607,8 @@ static bool pipewire_init(void) } /* this is just to test for PipeWire availabillity */ - struct pw_core * core = pw_context_connect(pw.context, NULL, 0); - if (!core) + pw.core = pw_context_connect(pw.context, NULL, 0); + if (!pw.core) goto err_context; /* PipeWire is available so create the loop thread and start it */ @@ -464,17 +616,32 @@ static bool pipewire_init(void) if (!pw.thread) { DEBUG_ERROR("Failed to create the thread loop"); - goto err_context; + goto err_core; } - pw_thread_loop_start(pw.thread); + const int result = pw_thread_loop_start(pw.thread); + if (result < 0) + { + DEBUG_ERROR("Failed to start the PipeWire thread loop: %s", + spa_strerror(result)); + pw_thread_loop_destroy(pw.thread); + pw.thread = NULL; + goto err_core; + } return true; +err_core: + pw_core_disconnect(pw.core); + pw.core = NULL; + err_context: pw_context_destroy(pw.context); + pw.context = NULL; err: - pw_loop_destroy(pw.loop); + if (pw.loop) + pw_loop_destroy(pw.loop); + pw.loop = NULL; pw_deinit(); return false; } @@ -488,12 +655,17 @@ static void pipewire_playbackStopStream(void) } pw_thread_loop_lock(pw.thread); - pw_stream_destroy(pw.playback.stream); - pw.playback.stream = NULL; - pw.playback.rateMatch = NULL; - pw.playback.resamplerEnabled = false; atomic_store_explicit( - &pw.playback.presentationDeadline, 0, memory_order_release); + &pw.playback.rateMatch, NULL, memory_order_release); + pw.playback.failureFn = NULL; + pw.playback.failureCookie = 0; + pw_stream_destroy(pw.playback.stream); + pw.playback.stream = NULL; + pw.playback.resamplerEnabled = false; + atomic_store_explicit(&pw.playback.state, + STREAM_STATE_INACTIVE, memory_order_release); + atomic_store_explicit( + &pw.playback.latencyNs, 0, memory_order_release); atomic_store_explicit( &pw.playback.latencyUpdateRequested, false, memory_order_relaxed); pw_thread_loop_unlock(pw.thread); @@ -524,23 +696,36 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, .version = PW_VERSION_STREAM_EVENTS, .state_changed = pipewire_onPlaybackStateChanged, .io_changed = pipewire_onPlaybackIoChanged, - .process = pipewire_onPlaybackProcess, - .drained = pipewire_onPlaybackDrained + .process = pipewire_onPlaybackProcess }; + bool reuse = false; + pw_thread_loop_lock(pw.thread); if (pw.playback.stream && + pw.playback.connectionState == PW_STREAM_STATE_PAUSED && + atomic_load_explicit( + &pw.playback.state, memory_order_acquire) == STREAM_STATE_INACTIVE && pipewire_audioFormatEqual(&pw.playback.format, format)) { + pw.playback.pullFn = pullFn; + bool resamplerReusable = true; #if PW_CHECK_VERSION(1, 4, 0) atomic_store_explicit( &pw.playback.resampleError, 0, memory_order_relaxed); *resamplerEnabled = - pipewire_configurePlaybackResampler(requestResampler); + pipewire_configurePlaybackResampler( + requestResampler, &resamplerReusable); #endif - *maxPeriodFrames = pw.playback.maxPeriodFrames; - *startFrames = pw.playback.startFrames; - return true; + if (resamplerReusable) + { + *maxPeriodFrames = pw.playback.maxPeriodFrames; + *startFrames = pw.playback.startFrames; + reuse = true; + } } + pw_thread_loop_unlock(pw.thread); + if (reuse) + return true; pipewire_playbackStopStream(); @@ -553,6 +738,8 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, pw.playback.pullFn = pullFn; pw_thread_loop_lock(pw.thread); + atomic_store_explicit( + &pw.playback.rateMatch, NULL, memory_order_release); struct pw_properties * props = pw_properties_new( @@ -602,17 +789,19 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, // is using the audio device, but we can treat this value as a maximum const struct pw_properties * properties = pw_stream_get_properties(pw.playback.stream); - const char *actualNodeLatency = properties ? + const char * actualNodeLatency = properties ? pw_properties_get(properties, PW_KEY_NODE_LATENCY) : NULL; unsigned num, denom; + uint64_t actualPeriodFrames = 0; if (!actualNodeLatency || - sscanf(actualNodeLatency, "%u/%u", &num, &denom) != 2 || - num == 0 || num > INT_MAX || denom != sampleRate) + sscanf(actualNodeLatency, "%u/%u", &num, &denom) != 2 || + num == 0 || denom == 0 || + (actualPeriodFrames = + ((uint64_t)num * sampleRate + denom - 1) / denom) > INT_MAX) { DEBUG_WARN( - "PIPEWIRE_LATENCY value '%s' is invalid or does not match stream sample " - "rate; using %d/%d", + "PIPEWIRE_LATENCY value '%s' is invalid; using %d/%d", actualNodeLatency ? actualNodeLatency : "(unset)", requestedPeriodFrames, sampleRate); @@ -625,11 +814,12 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, pw.playback.maxPeriodFrames = requestedPeriodFrames; } else - pw.playback.maxPeriodFrames = num; + pw.playback.maxPeriodFrames = (int)actualPeriodFrames; // If the previous quantum size was very small, PipeWire can request two full // periods almost immediately at the start of playback - pw.playback.startFrames = pw.playback.maxPeriodFrames * 2; + pw.playback.startFrames = (int)min( + (int64_t)pw.playback.maxPeriodFrames * 2, (int64_t)INT_MAX); *maxPeriodFrames = pw.playback.maxPeriodFrames; *startFrames = pw.playback.startFrames; @@ -653,40 +843,46 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, if (result < 0) { DEBUG_ERROR("Failed to connect playback stream: %s", spa_strerror(result)); + atomic_store_explicit( + &pw.playback.rateMatch, NULL, memory_order_release); pw_stream_destroy(pw.playback.stream); pw.playback.stream = NULL; - pw.playback.rateMatch = NULL; pw_thread_loop_unlock(pw.thread); return false; } - while (pw.playback.connectionState == PW_STREAM_STATE_CONNECTING) - pw_thread_loop_wait(pw.thread); - - if (pw.playback.connectionState != PW_STREAM_STATE_PAUSED) + if (!pipewire_waitForStream(&pw.playback.connectionState) || + pw.playback.connectionState != PW_STREAM_STATE_PAUSED) { DEBUG_ERROR("PipeWire playback stream did not become ready"); + atomic_store_explicit( + &pw.playback.rateMatch, NULL, memory_order_release); pw_stream_destroy(pw.playback.stream); pw.playback.stream = NULL; - pw.playback.rateMatch = NULL; pw_thread_loop_unlock(pw.thread); return false; } - pw.playback.state = STREAM_STATE_INACTIVE; + atomic_store_explicit(&pw.playback.state, + STREAM_STATE_INACTIVE, memory_order_release); atomic_store_explicit( - &pw.playback.presentationDeadline, 0, memory_order_release); + &pw.playback.latencyNs, 0, memory_order_release); atomic_store_explicit( &pw.playback.latencyUpdateRequested, false, memory_order_relaxed); atomic_store_explicit( &pw.playback.bufferErrors, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.playback.pullErrors, 0, memory_order_relaxed); atomic_store_explicit( &pw.playback.timingErrors, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.playback.streamError, 0, memory_order_relaxed); #if PW_CHECK_VERSION(1, 4, 0) atomic_store_explicit( &pw.playback.resampleError, 0, memory_order_relaxed); *resamplerEnabled = - pipewire_configurePlaybackResampler(requestResampler); + pipewire_configurePlaybackResampler( + requestResampler, NULL); #else (void)requestResampler; #endif @@ -694,113 +890,386 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, return true; } -static void pipewire_playbackStart(void) +static int pipewire_playbackControl(bool active, + LG_AudioFailureFn failureFn, uint32_t failureCookie) { - if (!pw.playback.stream) - return; - + int error = 0; pw_thread_loop_lock(pw.thread); - if (pw.playback.state != STREAM_STATE_ACTIVE) + if (!active) { - switch (pw.playback.state) + pw.playback.failureFn = NULL; + pw.playback.failureCookie = 0; + } + + if (!pw.playback.stream) + error = active ? -ENODEV : 0; + else if (active) + { + if (pw.playback.connectionState == PW_STREAM_STATE_ERROR || + pw.playback.connectionState == PW_STREAM_STATE_UNCONNECTED) + error = -EPIPE; + else { - case STREAM_STATE_INACTIVE: - pw_stream_set_active(pw.playback.stream, true); - pw.playback.state = STREAM_STATE_ACTIVE; - break; + pw.playback.failureFn = failureFn; + pw.playback.failureCookie = failureCookie; + if (atomic_load_explicit( + &pw.playback.state, memory_order_acquire) != STREAM_STATE_ACTIVE) + { + /* A stopped stream must not replay data queued before reactivation. */ + error = pw_stream_flush(pw.playback.stream, false); + if (error >= 0) + error = pw_stream_set_active(pw.playback.stream, true); + } + if (error >= 0) + { + atomic_store_explicit(&pw.playback.state, + STREAM_STATE_ACTIVE, memory_order_release); + } + } - case STREAM_STATE_DRAINING: - // We are in the middle of draining the PipeWire buffers; we need to - // wait for this to complete before allowing the new playback to start - break; - - default: - DEBUG_UNREACHABLE(); + if (error < 0 || + pw.playback.connectionState == PW_STREAM_STATE_ERROR || + pw.playback.connectionState == PW_STREAM_STATE_UNCONNECTED) + { + pw.playback.failureFn = NULL; + pw.playback.failureCookie = 0; + if (error >= 0) + error = -EPIPE; } } + else if (atomic_load_explicit( + &pw.playback.state, memory_order_acquire) == STREAM_STATE_ACTIVE) + { + const int activeError = + pw_stream_set_active(pw.playback.stream, false); + const int flushError = + pw_stream_flush(pw.playback.stream, false); + + error = activeError < 0 ? activeError : flushError; + if (activeError >= 0) + atomic_store_explicit(&pw.playback.state, + STREAM_STATE_INACTIVE, memory_order_release); + } + pw_thread_loop_unlock(pw.thread); + if (error < 0) + { + int expected = 0; + atomic_compare_exchange_strong_explicit( + &pw.playback.streamError, &expected, error, + memory_order_relaxed, memory_order_relaxed); + } + return error; +} + +static bool pipewire_playbackStart( + LG_AudioFailureFn failureFn, uint32_t failureCookie) +{ + return pipewire_playbackControl( + true, failureFn, failureCookie) >= 0; } static void pipewire_playbackStop(void) { - const bool inThread = pw_thread_loop_in_thread(pw.thread); - if (!inThread) - pw_thread_loop_lock(pw.thread); - - if (pw.playback.state != STREAM_STATE_ACTIVE) - goto done; - - if (inThread) + if (pipewire_playbackControl(false, NULL, 0) < 0) { - pw_stream_set_active(pw.playback.stream, false); - pw.playback.state = STREAM_STATE_INACTIVE; + pipewire_playbackStopStream(); + return; } - else - { - pw_stream_flush(pw.playback.stream, true); - pw.playback.state = STREAM_STATE_DRAINING; - } - -done: - if (!inThread) - pw_thread_loop_unlock(pw.thread); + atomic_store_explicit( + &pw.playback.latencyNs, 0, memory_order_release); + atomic_store_explicit( + &pw.playback.latencyUpdateRequested, false, memory_order_relaxed); } static void pipewire_playbackVolume(int channels, const uint16_t volume[]) { - if (channels != pw.playback.format.channelCount) + if (channels <= 0 || channels > LG_AUDIO_MAX_CHANNELS) return; float param[channels]; for(int i = 0; i < channels; ++i) - param[i] = 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787; + param[i] = max(0.0, + 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787); + int result = 0; pw_thread_loop_lock(pw.thread); - pw_stream_set_control(pw.playback.stream, SPA_PROP_channelVolumes, - channels, param, 0); + if (pw.playback.stream && + channels == pw.playback.format.channelCount) + result = pw_stream_set_control(pw.playback.stream, + SPA_PROP_channelVolumes, channels, param, 0); pw_thread_loop_unlock(pw.thread); + + if (result < 0) + DEBUG_WARN("Failed to set PipeWire playback volume: %s", + spa_strerror(result)); } static void pipewire_playbackMute(bool mute) { + int result = 0; pw_thread_loop_lock(pw.thread); - float val = mute ? 1.0f : 0.0f; - pw_stream_set_control(pw.playback.stream, SPA_PROP_mute, 1, &val, 0); + if (pw.playback.stream) + { + float val = mute ? 1.0f : 0.0f; + result = pw_stream_set_control( + pw.playback.stream, SPA_PROP_mute, 1, &val, 0); + } pw_thread_loop_unlock(pw.thread); + + if (result < 0) + DEBUG_WARN("Failed to set PipeWire playback mute: %s", + spa_strerror(result)); } static uint64_t pipewire_playbackLatency(void) { - pipewire_reportPlaybackErrors(); atomic_store_explicit( &pw.playback.latencyUpdateRequested, true, memory_order_release); - const int64_t deadline = atomic_load_explicit( - &pw.playback.presentationDeadline, memory_order_acquire); - if (deadline <= 0) + const int64_t latencyNs = atomic_load_explicit( + &pw.playback.latencyNs, memory_order_acquire); + if (latencyNs <= 0) return 0; - return max(INT64_C(0), deadline - pipewire_monotonicTime()) / 1000; + return latencyNs / 1000; +} + +static void pipewire_recordSignalSender(void) +{ + if (atomic_exchange_explicit( + &pw.record.sendWakePending, true, memory_order_acq_rel)) + return; + + if (sem_post(&pw.record.sendWake) < 0) + { + atomic_fetch_add_explicit( + &pw.record.signalErrors, 1, memory_order_relaxed); + if (errno != EOVERFLOW) + atomic_store_explicit( + &pw.record.sendWakePending, false, memory_order_release); + } +} + +static void pipewire_recordRejected(int frames) +{ + atomic_fetch_add_explicit( + &pw.record.rejectedBatches, 1, memory_order_relaxed); + atomic_fetch_add_explicit( + &pw.record.rejectedFrames, frames, memory_order_relaxed); +} + +static void pipewire_recordAdvanceClock( + LG_AudioClock * clock, int frames) +{ + const double rate = clock->rate > 0.0 ? + clock->rate : pw.record.format.sampleRate; + clock->position += frames; + if (rate > 0.0) + clock->time += (int64_t)llrint( + (double)frames * SPA_NSEC_PER_SEC / rate); + clock->discontinuity = false; +} + +static bool pipewire_recordClockJump( + double elapsedNs, double expectedNs) +{ + const double tolerance = + RECORD_CLOCK_TOLERANCE_FRAMES * SPA_NSEC_PER_SEC / + pw.record.format.sampleRate; + return elapsedNs <= 0 || + fabs(elapsedNs - expectedNs) > tolerance; +} + +static RecordBlock pipewire_recordMakeBlock( + struct pw_buffer * pbuf, const struct spa_buffer * sbuf, int frames) +{ + RecordBlock block = + { + .frames = frames + }; + struct pw_time timing = {0}; +#if PW_CHECK_VERSION(0, 3, 50) + const int timingResult = pw_stream_get_time_n( + pw.record.stream, &timing, sizeof(timing)); +#else + const int timingResult = pw_stream_get_time( + pw.record.stream, &timing); +#endif + const bool timingValid = timingResult >= 0 && + timing.rate.num != 0 && timing.rate.denom != 0; + + int64_t clockTime = 0; +#if PW_CHECK_VERSION(1, 0, 5) + if (pbuf->time) + clockTime = (int64_t)pbuf->time; +#endif + if (!clockTime && timingResult >= 0) + clockTime = timing.now; + + const uint64_t position = pw.record.clockPosition; + pw.record.clockPosition += frames; + + bool discontinuity = pw.record.clockDiscontinuityPending; + const struct spa_meta_header * header = spa_buffer_find_meta_data( + sbuf, SPA_META_Header, sizeof(*header)); + if (header && + (header->flags & (SPA_META_HEADER_FLAG_DISCONT | + SPA_META_HEADER_FLAG_CORRUPTED))) + discontinuity = true; + if (sbuf->datas[0].chunk->flags & SPA_CHUNK_FLAG_CORRUPTED) + discontinuity = true; + + block.clockValid = clockTime > 0; + if (block.clockValid) + { + const bool clockStable = pw.record.clockLastValid; + if (pw.record.clockLastValid) + { + const uint64_t elapsedFrames = + position - pw.record.clockLastPosition; + const double expectedNs = + (double)elapsedFrames * SPA_NSEC_PER_SEC / + pw.record.format.sampleRate; + if (pipewire_recordClockJump( + clockTime - pw.record.clockLastTime, expectedNs)) + discontinuity = true; + } + + if (timingValid && pw.record.clockLastTimingValid) + { + if (timing.rate.num != pw.record.clockLastRate.num || + timing.rate.denom != pw.record.clockLastRate.denom || + timing.ticks <= pw.record.clockLastTicks) + discontinuity = true; + else + { + const double elapsedTickNs = + (double)(timing.ticks - pw.record.clockLastTicks) * + timing.rate.num * SPA_NSEC_PER_SEC / timing.rate.denom; + const double expectedNs = + (double)(position - pw.record.clockLastPosition) * + SPA_NSEC_PER_SEC / pw.record.format.sampleRate; + if (pipewire_recordClockJump(elapsedTickNs, expectedNs)) + discontinuity = true; + } + } + + block.clock = (LG_AudioClock) + { + .position = position, + .time = clockTime, + .rate = 0.0, + .stable = clockStable && !discontinuity, + .discontinuity = discontinuity + }; + pw.record.clockDiscontinuityPending = false; + pw.record.clockLastPosition = position; + pw.record.clockLastTime = clockTime; + pw.record.clockLastValid = true; + } + else + { + pw.record.clockDiscontinuityPending |= discontinuity; + pw.record.clockLastValid = false; + } + + if (timingValid) + { + pw.record.clockLastTicks = timing.ticks; + pw.record.clockLastRate = timing.rate; + pw.record.clockLastTimingValid = block.clockValid; + } + else + pw.record.clockLastTimingValid = false; + + return block; +} + +static bool pipewire_recordLatestPending(void) +{ + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + if (atomic_load_explicit( + &pw.record.latest[i].state, memory_order_acquire) != + RECORD_LATEST_FREE) + return true; + + return false; +} + +static bool pipewire_recordSendLatest(void) +{ + int chosen = -1; + uint64_t chosenSerial = 0; + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + if (atomic_load_explicit( + &pw.record.latest[i].state, memory_order_acquire) != + RECORD_LATEST_READY) + continue; + + const uint64_t serial = atomic_load_explicit( + &pw.record.latest[i].serial, memory_order_relaxed); + if (chosen < 0 || serial > chosenSerial) + { + chosen = i; + chosenSerial = serial; + } + } + + if (chosen < 0) + return false; + + unsigned int expected = RECORD_LATEST_READY; + if (!atomic_compare_exchange_strong_explicit( + &pw.record.latest[chosen].state, &expected, + RECORD_LATEST_READING, memory_order_acq_rel, memory_order_acquire)) + return true; + + /* Only the newest overload block is useful. Drop older blocks before + * delivering it so recovery cannot build another backlog. */ + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + if (i == chosen || + atomic_load_explicit( + &pw.record.latest[i].serial, memory_order_relaxed) > chosenSerial) + continue; + + expected = RECORD_LATEST_READY; + if (atomic_compare_exchange_strong_explicit( + &pw.record.latest[i].state, &expected, + RECORD_LATEST_READING, + memory_order_acq_rel, memory_order_acquire)) + { + const int frames = pw.record.latest[i].frames; + atomic_store_explicit(&pw.record.latest[i].state, + RECORD_LATEST_FREE, memory_order_release); + atomic_fetch_add_explicit(&pw.record.droppedFrames, + frames, memory_order_relaxed); + } + } + + const int frames = pw.record.latest[chosen].frames; + const LG_AudioClock * clock = + pw.record.latest[chosen].clockValid ? + &pw.record.latest[chosen].clock : NULL; + if (!pw.record.pushFn( + pw.record.latestBuffer + + (size_t)chosen * pw.record.sendBufferFrames * pw.record.stride, + frames, clock)) + pipewire_recordRejected(frames); + atomic_store_explicit(&pw.record.latest[chosen].state, + RECORD_LATEST_FREE, memory_order_release); + return true; } static int pipewire_recordSendThread(void * opaque) { + uint64_t nextErrorReport = nanotime() + + RECORD_ERROR_REPORT_INTERVAL_NS; + for (;;) { - int available; - while ((available = ringbuffer_getCount(pw.record.sendQueue)) > 0) - { - const int frames = min(available, pw.record.sendBufferFrames); - const int consumed = ringbuffer_consume( - pw.record.sendQueue, pw.record.sendBuffer, frames); - DEBUG_ASSERT(consumed == frames); - pw.record.pushFn(pw.record.sendBuffer, frames); - } - - pipewire_reportRecordErrors(); - if (atomic_load_explicit(&pw.record.sendStop, memory_order_acquire)) - break; - int result; do result = sem_wait(&pw.record.sendWake); @@ -811,6 +1280,87 @@ static int pipewire_recordSendThread(void * opaque) DEBUG_ERROR("Failed to wait for the PipeWire recording worker"); break; } + + /* Clear before draining. A producer racing before this point is covered by + * this wake, while a producer racing after it publishes the sole next + * wake. */ + atomic_store_explicit( + &pw.record.sendWakePending, false, memory_order_release); + if (atomic_load_explicit(&pw.record.sendStop, memory_order_acquire)) + break; + + for (;;) + { + pipewire_reportRecordErrorsDue(&nextErrorReport); + + if (atomic_load_explicit( + &pw.record.sendStop, memory_order_acquire)) + break; + + if (atomic_load_explicit( + &pw.record.latestMode, memory_order_acquire)) + { + const int queued = ringbuffer_getCount(pw.record.sendQueue); + if (queued > 0) + { + ringbuffer_consume(pw.record.sendQueue, NULL, queued); + atomic_fetch_add_explicit( + &pw.record.droppedFrames, queued, memory_order_relaxed); + } + const int queuedBlocks = ringbuffer_getCount(pw.record.sendBlocks); + if (queuedBlocks > 0) + ringbuffer_consume( + pw.record.sendBlocks, NULL, queuedBlocks); + + while (!atomic_load_explicit( + &pw.record.sendStop, memory_order_acquire) && + pipewire_recordSendLatest()) + ; + + atomic_store_explicit( + &pw.record.latestMode, false, memory_order_release); + if (pipewire_recordLatestPending()) + { + atomic_store_explicit( + &pw.record.latestMode, true, memory_order_release); + continue; + } + } + + RecordBlock block; + if (ringbuffer_consume( + pw.record.sendBlocks, &block, 1) != 1) + break; + + LG_AudioClock clock = block.clock; + int remaining = block.frames; + while (remaining > 0 && !atomic_load_explicit( + &pw.record.sendStop, memory_order_acquire)) + { + const int frames = min( + remaining, pw.record.sendBufferFrames); + const int consumed = ringbuffer_consume( + pw.record.sendQueue, pw.record.sendBuffer, frames); + DEBUG_ASSERT(consumed == frames); + if (consumed != frames) + { + atomic_fetch_add_explicit( + &pw.record.bufferErrors, 1, memory_order_relaxed); + break; + } + + if (!pw.record.pushFn(pw.record.sendBuffer, frames, + block.clockValid ? &clock : NULL)) + pipewire_recordRejected(frames); + remaining -= frames; + if (block.clockValid) + pipewire_recordAdvanceClock(&clock, frames); + } + } + + if (atomic_load_explicit( + &pw.record.sendStop, memory_order_acquire)) + break; } pipewire_reportRecordErrors(); @@ -819,12 +1369,12 @@ static int pipewire_recordSendThread(void * opaque) static void pipewire_recordStopSender(void) { + const bool hadThread = pw.record.sendThread != NULL; if (pw.record.sendThread) { atomic_store_explicit( &pw.record.sendStop, true, memory_order_release); - if (sem_post(&pw.record.sendWake) < 0) - DEBUG_ERROR("Failed to stop the PipeWire recording worker"); + pipewire_recordSignalSender(); lgJoinThread(pw.record.sendThread, NULL); pw.record.sendThread = NULL; } @@ -836,10 +1386,15 @@ static void pipewire_recordStopSender(void) } ringbuffer_free(&pw.record.sendQueue); + ringbuffer_free(&pw.record.sendBlocks); free(pw.record.sendBuffer); - pw.record.sendBuffer = NULL; + free(pw.record.latestBuffer); + pw.record.sendBuffer = NULL; + pw.record.latestBuffer = NULL; pw.record.sendBufferFrames = 0; - pipewire_reportRecordErrors(); + pw.record.maxQueuedFrames = 0; + if (!hadThread) + pipewire_reportRecordErrors(); } static bool pipewire_recordStartSender(int sampleRate) @@ -851,10 +1406,16 @@ static bool pipewire_recordStartSender(int sampleRate) pw.record.sendQueue = ringbuffer_new( queueFrames, pw.record.stride); + pw.record.sendBlocks = ringbuffer_new( + queueFrames, sizeof(RecordBlock)); pw.record.sendBuffer = malloc( (size_t)sendFrames * pw.record.stride); + pw.record.latestBuffer = malloc( + (size_t)RECORD_LATEST_SLOTS * sendFrames * pw.record.stride); pw.record.sendBufferFrames = sendFrames; - if (!pw.record.sendQueue || !pw.record.sendBuffer) + pw.record.maxQueuedFrames = max(sampleRate / 50, 1); + if (!pw.record.sendQueue || !pw.record.sendBlocks || + !pw.record.sendBuffer || !pw.record.latestBuffer) { DEBUG_ERROR("Failed to allocate the PipeWire recording queue"); pipewire_recordStopSender(); @@ -871,12 +1432,42 @@ static bool pipewire_recordStartSender(int sampleRate) atomic_store_explicit( &pw.record.sendStop, false, memory_order_relaxed); + atomic_store_explicit( + &pw.record.sendWakePending, false, memory_order_relaxed); + atomic_store_explicit( + &pw.record.latestMode, false, memory_order_relaxed); + atomic_store_explicit( + &pw.record.latestSerial, 0, memory_order_relaxed); + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + atomic_store_explicit(&pw.record.latest[i].state, + RECORD_LATEST_FREE, memory_order_relaxed); + atomic_store_explicit( + &pw.record.latest[i].serial, 0, memory_order_relaxed); + pw.record.latest[i].frames = 0; + pw.record.latest[i].clockValid = false; + pw.record.latest[i].clock = (LG_AudioClock) {0}; + } + pw.record.clockPosition = 0; + pw.record.clockLastPosition = 0; + pw.record.clockLastTicks = 0; + pw.record.clockLastTime = 0; + pw.record.clockLastRate = (struct spa_fraction) {0}; + pw.record.clockLastValid = false; + pw.record.clockLastTimingValid = false; + pw.record.clockDiscontinuityPending = false; atomic_store_explicit( &pw.record.bufferErrors, 0, memory_order_relaxed); atomic_store_explicit( &pw.record.droppedFrames, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.record.rejectedBatches, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.record.rejectedFrames, 0, memory_order_relaxed); atomic_store_explicit( &pw.record.signalErrors, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.record.recycleError, 0, memory_order_relaxed); if (!lgCreateThread("pwRecordSend", pipewire_recordSendThread, NULL, &pw.record.sendThread)) { @@ -889,38 +1480,143 @@ static bool pipewire_recordStartSender(int sampleRate) static void pipewire_recordStopStream(void) { + pw_thread_loop_lock(pw.thread); + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; if (pw.record.stream) { - pw_thread_loop_lock(pw.thread); pw_stream_destroy(pw.record.stream); pw.record.stream = NULL; - pw_thread_loop_unlock(pw.thread); } - pw.record.active = false; + pw_thread_loop_unlock(pw.thread); + pipewire_recordStopSender(); + pw.record.pushFn = NULL; } -static void pipewire_recordQueueFrames(const void * data, int frames) +static void pipewire_recordQueueFrames( + const void * data, const RecordBlock * block) { + const int frames = block->frames; + int chosen = -1; + if (atomic_load_explicit( + &pw.record.latestMode, memory_order_acquire)) + goto latest; + const int occupancy = ringbuffer_getCount(pw.record.sendQueue); const int available = max(0, ringbuffer_getLength(pw.record.sendQueue) - occupancy); - const int append = min(frames, available); + const bool exceedsBacklog = occupancy > 0 && + (occupancy >= pw.record.maxQueuedFrames || + frames > pw.record.maxQueuedFrames - occupancy); + if (frames > available || exceedsBacklog) + goto latest; + const int advanced = - ringbuffer_append(pw.record.sendQueue, data, append); - DEBUG_ASSERT(advanced == append); + ringbuffer_append(pw.record.sendQueue, data, frames); + DEBUG_ASSERT(advanced == frames); + if (advanced == frames) + { + const int blocks = ringbuffer_append( + pw.record.sendBlocks, block, 1); + DEBUG_ASSERT(blocks == 1); + if (blocks == 1) + { + pipewire_recordSignalSender(); + return; + } + } - if (append != frames) - atomic_fetch_add_explicit( - &pw.record.droppedFrames, frames - append, memory_order_relaxed); +latest: + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + unsigned int expected = RECORD_LATEST_FREE; + if (atomic_compare_exchange_strong_explicit( + &pw.record.latest[i].state, &expected, + RECORD_LATEST_WRITING, memory_order_acq_rel, memory_order_acquire)) + { + chosen = i; + break; + } + } - /* Posting only when the producer observes an empty queue can lose a wake - * while the consumer drains the previous batch. Always notify for appended - * data; the sender coalesces all queued frames when it wakes. */ - if (append > 0 && sem_post(&pw.record.sendWake) < 0) + if (chosen < 0) + { + int oldest = -1; + uint64_t oldestSerial = UINT64_MAX; + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + if (atomic_load_explicit( + &pw.record.latest[i].state, memory_order_acquire) != + RECORD_LATEST_READY) + continue; + + const uint64_t serial = atomic_load_explicit( + &pw.record.latest[i].serial, memory_order_relaxed); + if (serial < oldestSerial) + { + oldest = i; + oldestSerial = serial; + } + } + + if (oldest >= 0) + { + unsigned int expected = RECORD_LATEST_READY; + if (atomic_compare_exchange_strong_explicit( + &pw.record.latest[oldest].state, &expected, + RECORD_LATEST_WRITING, + memory_order_acq_rel, memory_order_acquire)) + { + chosen = oldest; + const int dropped = pw.record.latest[oldest].frames; + atomic_fetch_add_explicit(&pw.record.droppedFrames, + dropped, memory_order_relaxed); + } + } + } + + if (chosen < 0) + { atomic_fetch_add_explicit( - &pw.record.signalErrors, 1, memory_order_relaxed); + &pw.record.droppedFrames, frames, memory_order_relaxed); + pw.record.clockDiscontinuityPending = true; + atomic_store_explicit( + &pw.record.latestMode, true, memory_order_release); + pipewire_recordSignalSender(); + return; + } + + const int keep = min(frames, pw.record.sendBufferFrames); + memcpy( + pw.record.latestBuffer + + (size_t)chosen * pw.record.sendBufferFrames * pw.record.stride, + (const uint8_t *)data + (size_t)(frames - keep) * pw.record.stride, + (size_t)keep * pw.record.stride); + pw.record.latest[chosen].frames = keep; + pw.record.latest[chosen].clockValid = block->clockValid; + if (block->clockValid) + { + pw.record.latest[chosen].clock = block->clock; + pipewire_recordAdvanceClock( + &pw.record.latest[chosen].clock, frames - keep); + pw.record.latest[chosen].clock.discontinuity = true; + pw.record.latest[chosen].clock.stable = false; + } + atomic_store_explicit(&pw.record.latest[chosen].serial, + atomic_fetch_add_explicit( + &pw.record.latestSerial, 1, memory_order_relaxed) + 1, + memory_order_relaxed); + atomic_store_explicit(&pw.record.latest[chosen].state, + RECORD_LATEST_READY, memory_order_release); + atomic_store_explicit( + &pw.record.latestMode, true, memory_order_release); + + if (keep != frames) + atomic_fetch_add_explicit( + &pw.record.droppedFrames, frames - keep, memory_order_relaxed); + pipewire_recordSignalSender(); } static void pipewire_onRecordProcess(void * userdata) @@ -931,6 +1627,7 @@ static void pipewire_onRecordProcess(void * userdata) { atomic_fetch_add_explicit( &pw.record.bufferErrors, 1, memory_order_relaxed); + pw.record.clockDiscontinuityPending = true; return; } @@ -941,65 +1638,111 @@ static void pipewire_onRecordProcess(void * userdata) { atomic_fetch_add_explicit( &pw.record.bufferErrors, 1, memory_order_relaxed); + pw.record.clockDiscontinuityPending = true; #if PW_CHECK_VERSION(1, 4, 0) - pw_stream_return_buffer(pw.record.stream, pbuf); + const int recycleResult = + pw_stream_return_buffer(pw.record.stream, pbuf); #else - pw_stream_queue_buffer(pw.record.stream, pbuf); + const int recycleResult = + pw_stream_queue_buffer(pw.record.stream, pbuf); #endif + pipewire_storeError(&pw.record.recycleError, recycleResult); return; } const uint32_t offset = sbuf->datas[0].chunk->offset; - const uint32_t bytes = min( + const uint32_t flags = sbuf->datas[0].chunk->flags; + const bool empty = flags & SPA_CHUNK_FLAG_EMPTY; + if (flags & (SPA_CHUNK_FLAG_EMPTY | SPA_CHUNK_FLAG_CORRUPTED)) + pw.record.clockDiscontinuityPending = true; + + const uint32_t bytes = empty ? 0 : min( sbuf->datas[0].chunk->size, sbuf->datas[0].maxsize - offset); const int frames = bytes / pw.record.stride; + if (bytes % pw.record.stride) + pw.record.clockDiscontinuityPending = true; if (frames > 0) + { + const RecordBlock block = + pipewire_recordMakeBlock(pbuf, sbuf, frames); pipewire_recordQueueFrames( - (uint8_t *)sbuf->datas[0].data + offset, frames); + (uint8_t *)sbuf->datas[0].data + offset, &block); + } - pw_stream_queue_buffer(pw.record.stream, pbuf); + pipewire_storeError(&pw.record.recycleError, + pw_stream_queue_buffer(pw.record.stream, pbuf)); } -static void pipewire_recordStart(const LG_AudioFormat * format, - LG_AudioPushFn pushFn) +static void pipewire_onRecordStateChanged(void * userdata, + enum pw_stream_state old, enum pw_stream_state state, + const char * error) { + (void)error; + pw.record.connectionState = state; + if (state == PW_STREAM_STATE_ERROR || + state == PW_STREAM_STATE_UNCONNECTED) + { + LG_AudioFailureFn failureFn = pw.record.failureFn; + const uint32_t failureCookie = pw.record.failureCookie; + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; + + if (failureFn && failureCookie) + failureFn(failureCookie); + } + pw_thread_loop_signal(pw.thread, false); +} + +static int pipewire_recordControl(bool active) +{ + int error = 0; + pw_thread_loop_lock(pw.thread); + if (!pw.record.stream) + error = active ? -ENODEV : 0; + else if (pw.record.active != active) + { + error = pw_stream_set_active(pw.record.stream, active); + if (error >= 0) + pw.record.active = active; + } + pw_thread_loop_unlock(pw.thread); + return error; +} + +static bool pipewire_recordStart(const LG_AudioFormat * format, + LG_AudioPushFn pushFn, LG_AudioFailureFn failureFn, + uint32_t failureCookie) +{ + pipewire_recordStopStream(); + const int channels = format->channelCount; const int sampleRate = format->sampleRate; const int sampleSize = pipewire_sampleSize(format->sampleFormat); + const enum spa_audio_format sampleFormat = + pipewire_sampleFormat(format->sampleFormat); + if (!sampleSize || sampleFormat == SPA_AUDIO_FORMAT_UNKNOWN) + return false; + const int periodFrames = max(sampleRate / 1000, 1); char requestedNodeLatency[32]; snprintf(requestedNodeLatency, sizeof(requestedNodeLatency), "%d/%d", periodFrames, sampleRate); - const struct spa_pod * params[1]; + const struct spa_pod * params[2]; uint8_t buffer[1024]; struct spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer)); static const struct pw_stream_events events = { - .version = PW_VERSION_STREAM_EVENTS, - .process = pipewire_onRecordProcess + .version = PW_VERSION_STREAM_EVENTS, + .state_changed = pipewire_onRecordStateChanged, + .process = pipewire_onRecordProcess }; - if (pw.record.stream && - pipewire_audioFormatEqual(&pw.record.format, format)) - { - if (!pw.record.active) - { - pw_thread_loop_lock(pw.thread); - pw_stream_set_active(pw.record.stream, true); - pw.record.active = true; - pw_thread_loop_unlock(pw.thread); - } - return; - } - - pipewire_recordStopStream(); - pw.record.format = *format; pw.record.stride = sampleSize * channels; pw.record.pushFn = pushFn; if (!pipewire_recordStartSender(sampleRate)) - return; + return false; struct pw_properties * props = pw_properties_new( @@ -1014,7 +1757,7 @@ static void pipewire_recordStart(const LG_AudioFormat * format, { DEBUG_ERROR("Failed to create recording stream properties"); pipewire_recordStopSender(); - return; + return false; } const char * device = option_get_string("pipewire", "recDevice"); @@ -1028,6 +1771,8 @@ static void pipewire_recordStart(const LG_AudioFormat * format, } pw_thread_loop_lock(pw.thread); + pw.record.failureFn = failureFn; + pw.record.failureCookie = failureCookie; pw.record.stream = pw_stream_new_simple( pw.loop, "Looking Glass", @@ -1038,40 +1783,71 @@ static void pipewire_recordStart(const LG_AudioFormat * format, if (!pw.record.stream) { + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; pw_thread_loop_unlock(pw.thread); DEBUG_ERROR("Failed to create the stream"); pipewire_recordStopSender(); - return; + return false; } struct spa_audio_info_raw info = - pipewire_audioInfo(format, - pipewire_sampleFormat(format->sampleFormat)); + pipewire_audioInfo(format, sampleFormat); params[0] = spa_format_audio_raw_build( &b, SPA_PARAM_EnumFormat, &info); + params[1] = spa_pod_builder_add_object(&b, + SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, + SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header), + SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_header))); - const int result = pw_stream_connect( + pw.record.connectionState = PW_STREAM_STATE_CONNECTING; + int result = pw_stream_connect( pw.record.stream, PW_DIRECTION_INPUT, PW_ID_ANY, PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_MAP_BUFFERS | - PW_STREAM_FLAG_RT_PROCESS, - params, 1); + PW_STREAM_FLAG_RT_PROCESS | + PW_STREAM_FLAG_INACTIVE, + params, 2); if (result < 0) { DEBUG_ERROR("Failed to connect recording stream: %s", spa_strerror(result)); + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; pw_stream_destroy(pw.record.stream); pw.record.stream = NULL; pw_thread_loop_unlock(pw.thread); pipewire_recordStopSender(); - return; + return false; + } + + if (!pipewire_waitForStream(&pw.record.connectionState) || + pw.record.connectionState != PW_STREAM_STATE_PAUSED) + { + DEBUG_ERROR("PipeWire recording stream did not become ready"); + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; + pw_stream_destroy(pw.record.stream); + pw.record.stream = NULL; + pw_thread_loop_unlock(pw.thread); + pipewire_recordStopSender(); + return false; } pw_thread_loop_unlock(pw.thread); - pw.record.active = true; + result = pipewire_recordControl(true); + if (result < 0) + { + DEBUG_ERROR("Failed to activate recording stream: %s", + spa_strerror(result)); + pipewire_recordStopStream(); + return false; + } + + return true; } static void pipewire_recordStop(void) @@ -1081,31 +1857,56 @@ static void pipewire_recordStop(void) static void pipewire_recordVolume(int channels, const uint16_t volume[]) { - if (channels != pw.record.format.channelCount) + if (channels <= 0 || channels > LG_AUDIO_MAX_CHANNELS) return; float param[channels]; for(int i = 0; i < channels; ++i) - param[i] = 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787; + param[i] = max(0.0, + 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787); + int result = 0; pw_thread_loop_lock(pw.thread); - pw_stream_set_control(pw.record.stream, SPA_PROP_channelVolumes, - channels, param, 0); + if (pw.record.stream && + channels == pw.record.format.channelCount) + result = pw_stream_set_control(pw.record.stream, + SPA_PROP_channelVolumes, channels, param, 0); pw_thread_loop_unlock(pw.thread); + + if (result < 0) + DEBUG_WARN("Failed to set PipeWire recording volume: %s", + spa_strerror(result)); } static void pipewire_recordMute(bool mute) { + int result = 0; pw_thread_loop_lock(pw.thread); - float val = mute ? 1.0f : 0.0f; - pw_stream_set_control(pw.record.stream, SPA_PROP_mute, 1, &val, 0); + if (pw.record.stream) + { + float val = mute ? 1.0f : 0.0f; + result = pw_stream_set_control( + pw.record.stream, SPA_PROP_mute, 1, &val, 0); + } pw_thread_loop_unlock(pw.thread); + + if (result < 0) + DEBUG_WARN("Failed to set PipeWire recording mute: %s", + spa_strerror(result)); } static void pipewire_free(void) { pipewire_playbackStopStream(); pipewire_recordStopStream(); + + pw_thread_loop_lock(pw.thread); + if (pw.core) + { + pw_core_disconnect(pw.core); + pw.core = NULL; + } + pw_thread_loop_unlock(pw.thread); pw_thread_loop_stop(pw.thread); pw_thread_loop_destroy(pw.thread); pw_context_destroy(pw.context); diff --git a/client/audiodevs/PulseAudio/pulseaudio.c b/client/audiodevs/PulseAudio/pulseaudio.c index 06d4c5dc..6630e6b0 100644 --- a/client/audiodevs/PulseAudio/pulseaudio.c +++ b/client/audiodevs/PulseAudio/pulseaudio.c @@ -28,18 +28,23 @@ #include "common/debug.h" #include "common/time.h" +#define PULSEAUDIO_ERROR_REPORT_INTERVAL_NS INT64_C(1000000000) +#define PULSEAUDIO_OPERATION_TIMEOUT_US UINT64_C(250000) +#define PULSEAUDIO_READY_TIMEOUT_US UINT64_C(5000000) +#define PULSEAUDIO_RATE_UPDATE_INTERVAL_NS INT64_C(50000000) +#define PULSEAUDIO_RATE_UPDATE_MAX_HOLD_NS INT64_C(250000000) +#define PULSEAUDIO_RATE_UPDATE_DEADBAND_HZ UINT32_C(1) + struct PulseAudio { pa_threaded_mainloop * loop; pa_mainloop_api * api; pa_context * context; - pa_operation * contextSub; + bool loopStarted; pa_stream * sink; - int sinkIndex; + uint32_t sinkIndex; bool sinkCorked; - bool sinkMuted; - bool sinkStarting; bool sinkResamplerEnabled; bool sinkRateFailed; int sinkMaxPeriodFrames; @@ -50,9 +55,26 @@ struct PulseAudio uint32_t sinkAppliedRate; uint32_t sinkPendingRate; uint32_t sinkRequestedRate; + uint32_t sinkDeferredRate; + int64_t sinkNextRateUpdate; + int64_t sinkRateDeferredSince; + bool sinkRateUpdateArmed; pa_operation * sinkRateOperation; + pa_operation * sinkStartOperation; + pa_time_event * sinkStartTimer; + uint32_t sinkStartSerial; LG_AudioPullFn sinkPullFn; - _Atomic(int64_t) sinkPresentationDeadline; + LG_AudioFailureFn sinkFailureFn; + uint32_t sinkFailureCookie; + _Atomic(int64_t) sinkLatencyNs; + atomic_bool sinkLatencyUpdateRequested; + atomic_bool sinkErrorsPending; + _Atomic(int64_t) sinkNextErrorReport; + atomic_uint sinkUnderflows; + atomic_uint sinkOverflows; + atomic_uint sinkWriteErrors; + atomic_uint sinkRateErrors; + atomic_uint sinkControlErrors; }; static struct PulseAudio pa = {0}; @@ -136,13 +158,83 @@ static pa_channel_position_t pulseaudio_channel( return (pa_channel_position_t)(PA_CHANNEL_POSITION_AUX0 + index); } -static void pulseaudio_unrefOperation(pa_operation * operation) +static void pulseaudio_noteError(atomic_uint * counter) { - if (operation) + atomic_fetch_add_explicit(counter, 1, memory_order_relaxed); + atomic_store_explicit( + &pa.sinkErrorsPending, true, memory_order_release); +} + +static void pulseaudio_trackControlOperation(pa_operation * operation) +{ + if (!operation) + pulseaudio_noteError(&pa.sinkControlErrors); + else pa_operation_unref(operation); } -static bool pulseaudio_submitRateUpdate(void); +static void pulseaudio_streamControl_cb( + pa_stream * stream, int success, void * userdata) +{ + if (!success) + pulseaudio_noteError(&pa.sinkControlErrors); +} + +static void pulseaudio_contextControl_cb( + pa_context * context, int success, void * userdata) +{ + if (!success) + pulseaudio_noteError(&pa.sinkControlErrors); +} + +static void pulseaudio_reportErrors(bool force) +{ + if (!atomic_load_explicit( + &pa.sinkErrorsPending, memory_order_acquire)) + return; + + const int64_t now = (int64_t)nanotime(); + if (!force) + { + int64_t next = atomic_load_explicit( + &pa.sinkNextErrorReport, memory_order_relaxed); + if (now < next || !atomic_compare_exchange_strong_explicit( + &pa.sinkNextErrorReport, &next, + now + PULSEAUDIO_ERROR_REPORT_INTERVAL_NS, + memory_order_relaxed, memory_order_relaxed)) + return; + } + + if (!atomic_exchange_explicit( + &pa.sinkErrorsPending, false, memory_order_acq_rel)) + return; + + const unsigned int underflows = atomic_exchange_explicit( + &pa.sinkUnderflows, 0, memory_order_relaxed); + const unsigned int overflows = atomic_exchange_explicit( + &pa.sinkOverflows, 0, memory_order_relaxed); + const unsigned int writeErrors = atomic_exchange_explicit( + &pa.sinkWriteErrors, 0, memory_order_relaxed); + const unsigned int rateErrors = atomic_exchange_explicit( + &pa.sinkRateErrors, 0, memory_order_relaxed); + const unsigned int controlErrors = atomic_exchange_explicit( + &pa.sinkControlErrors, 0, memory_order_relaxed); + + if (underflows) + DEBUG_WARN("PulseAudio playback underflowed %u time(s)", underflows); + if (overflows) + DEBUG_WARN("PulseAudio playback overflowed %u time(s)", overflows); + if (writeErrors) + DEBUG_WARN("PulseAudio playback write failed %u time(s)", writeErrors); + if (rateErrors) + DEBUG_WARN("PulseAudio sample rate update failed %u time(s)", + rateErrors); + if (controlErrors) + DEBUG_WARN("PulseAudio control operation failed %u time(s)", + controlErrors); +} + +static bool pulseaudio_submitRateUpdate(bool force); static void pulseaudio_rateUpdate_cb(pa_stream * stream, int success, void * userdata) @@ -156,33 +248,46 @@ static void pulseaudio_rateUpdate_cb(pa_stream * stream, int success, if (!success) { - DEBUG_ERROR("Failed to update PulseAudio sample rate: %s", - pa_strerror(pa_context_errno(pa.context))); + pulseaudio_noteError(&pa.sinkRateErrors); pa.sinkRateFailed = true; return; } pa.sinkAppliedRate = pa.sinkPendingRate; if (pa.sinkCorked) - pulseaudio_submitRateUpdate(); + pulseaudio_submitRateUpdate(true); } -static bool pulseaudio_submitRateUpdate(void) +static bool pulseaudio_submitRateUpdate(bool force) { if (pa.sinkRateFailed) return false; - if (pa.sinkRateOperation || - pa.sinkRequestedRate == pa.sinkAppliedRate) + if (pa.sinkRateOperation) return true; - pa.sinkPendingRate = pa.sinkRequestedRate; + if (pa.sinkRequestedRate == pa.sinkAppliedRate) + { + pa.sinkDeferredRate = pa.sinkRequestedRate; + pa.sinkRateUpdateArmed = false; + pa.sinkRateDeferredSince = 0; + return true; + } + + if (!force && !pa.sinkRateUpdateArmed) + return true; + + pa.sinkPendingRate = pa.sinkRequestedRate; + pa.sinkDeferredRate = pa.sinkPendingRate; + pa.sinkRateUpdateArmed = false; + pa.sinkRateDeferredSince = 0; + pa.sinkNextRateUpdate = + (int64_t)nanotime() + PULSEAUDIO_RATE_UPDATE_INTERVAL_NS; pa.sinkRateOperation = pa_stream_update_sample_rate(pa.sink, pa.sinkPendingRate, pulseaudio_rateUpdate_cb, NULL); if (!pa.sinkRateOperation) { - DEBUG_ERROR("Failed to request a PulseAudio sample rate update: %s", - pa_strerror(pa_context_errno(pa.context))); + pulseaudio_noteError(&pa.sinkRateErrors); pa.sinkRateFailed = true; return false; } @@ -190,31 +295,43 @@ static bool pulseaudio_submitRateUpdate(void) return true; } -static void pulseaudio_sink_input_cb(pa_context *c, const pa_sink_input_info *i, - int eol, void *userdata) +static void pulseaudio_cancelStartTimer_nl(void) { - if (eol < 0 || eol == 1) + if (!pa.sinkStartTimer) return; - pa.sinkIndex = i->index; + pa_time_event * timer = pa.sinkStartTimer; + pa.sinkStartTimer = NULL; + pa.api->time_free(timer); } -static void pulseaudio_subscribe_cb(pa_context *c, - pa_subscription_event_type_t t, uint32_t index, void *userdata) +static void pulseaudio_cancelStartOperation_nl(void) { - switch (t & PA_SUBSCRIPTION_EVENT_FACILITY_MASK) - { - case PA_SUBSCRIPTION_EVENT_SINK_INPUT: - if ((t & PA_SUBSCRIPTION_EVENT_TYPE_MASK) == PA_SUBSCRIPTION_EVENT_REMOVE) - pa.sinkIndex = 0; - else - { - pa_operation *o = pa_context_get_sink_input_info(c, index, - pulseaudio_sink_input_cb, NULL); - pulseaudio_unrefOperation(o); - } - break; - } + if (!pa.sinkStartOperation) + return; + + pa_operation * operation = pa.sinkStartOperation; + pa.sinkStartOperation = NULL; + pa_operation_cancel(operation); + pa_operation_unref(operation); +} + +static void pulseaudio_sinkDisarm_nl(void) +{ + ++pa.sinkStartSerial; + pulseaudio_cancelStartTimer_nl(); + pulseaudio_cancelStartOperation_nl(); + pa.sinkFailureFn = NULL; + pa.sinkFailureCookie = 0; +} + +static void pulseaudio_sinkFailed_nl(void) +{ + LG_AudioFailureFn failureFn = pa.sinkFailureFn; + const uint32_t failureCookie = pa.sinkFailureCookie; + pulseaudio_sinkDisarm_nl(); + if (failureFn) + failureFn(failureCookie); } static void pulseaudio_ctx_state_change_cb(pa_context * c, void * userdata) @@ -228,28 +345,120 @@ static void pulseaudio_ctx_state_change_cb(pa_context * c, void * userdata) case PA_CONTEXT_READY: DEBUG_INFO("Connected to PulseAudio server"); - pa_context_set_subscribe_callback(c, pulseaudio_subscribe_cb, NULL); - pa_context_subscribe(c, PA_SUBSCRIPTION_MASK_SINK_INPUT, NULL, NULL); pa_threaded_mainloop_signal(pa.loop, 0); break; case PA_CONTEXT_TERMINATED: - if (pa.contextSub) - { - pa_operation_unref(pa.contextSub); - pa.contextSub = NULL; - } + if (c == pa.context) + pulseaudio_sinkFailed_nl(); + pa_threaded_mainloop_signal(pa.loop, 0); break; case PA_CONTEXT_FAILED: default: + if (c == pa.context) + pulseaudio_sinkFailed_nl(); DEBUG_ERROR("context error: %s", pa_strerror(pa_context_errno(c))); + pa_threaded_mainloop_signal(pa.loop, 0); break; } } +static void pulseaudio_context_close_nl(void) +{ + if (!pa.context) + return; + + pa_context_set_state_callback(pa.context, NULL, NULL); + pa_context_disconnect(pa.context); + pa_context_unref(pa.context); + pa.context = NULL; +} + +struct PulseWait +{ + bool timedOut; +}; + +static void pulseaudio_timeout_cb(pa_mainloop_api * api, + pa_time_event * event, const struct timeval * tv, void * userdata) +{ + (void)api; + (void)event; + (void)tv; + struct PulseWait * wait = userdata; + wait->timedOut = true; + pa_threaded_mainloop_signal(pa.loop, 0); +} + +/* The PulseAudio threaded mainloop must be locked. */ +static bool pulseaudio_contextConnect_nl(void) +{ + pa_proplist * propList = pa_proplist_new(); + if (!propList) + { + DEBUG_ERROR("Failed to create the PulseAudio property list"); + return false; + } + pa_proplist_sets(propList, PA_PROP_MEDIA_ROLE, "video"); + + pa.context = pa_context_new_with_proplist( + pa.api, "Looking Glass", propList); + pa_proplist_free(propList); + if (!pa.context) + { + DEBUG_ERROR("Failed to create the PulseAudio context"); + return false; + } + + pa_context_set_state_callback(pa.context, + pulseaudio_ctx_state_change_cb, NULL); + if (pa_context_connect( + pa.context, NULL, PA_CONTEXT_NOAUTOSPAWN, NULL) < 0) + { + DEBUG_ERROR("Failed to connect to the PulseAudio server: %s", + pa_strerror(pa_context_errno(pa.context))); + pulseaudio_context_close_nl(); + return false; + } + + struct PulseWait wait = {0}; + pa_time_event * timer = pa_context_rttime_new(pa.context, + pa_rtclock_now() + PULSEAUDIO_READY_TIMEOUT_US, + pulseaudio_timeout_cb, &wait); + if (!timer) + { + DEBUG_ERROR("Failed to create the PulseAudio connection timer"); + pulseaudio_context_close_nl(); + return false; + } + + pa_context_state_t state; + while (!wait.timedOut) + { + state = pa_context_get_state(pa.context); + if (state == PA_CONTEXT_READY || !PA_CONTEXT_IS_GOOD(state)) + break; + pa_threaded_mainloop_wait(pa.loop); + } + state = pa_context_get_state(pa.context); + pa.api->time_free(timer); + + if (state == PA_CONTEXT_READY) + return true; + + if (wait.timedOut) + DEBUG_ERROR("Timed out connecting to the PulseAudio server"); + else + DEBUG_ERROR("PulseAudio context did not become ready: %s", + pa_strerror(pa_context_errno(pa.context))); + pulseaudio_context_close_nl(); + return false; +} + static bool pulseaudio_init(void) { + pa.sinkIndex = PA_INVALID_INDEX; pa.loop = pa_threaded_mainloop_new(); if (!pa.loop) { @@ -258,74 +467,33 @@ static bool pulseaudio_init(void) } pa.api = pa_threaded_mainloop_get_api(pa.loop); - if (pa_signal_init(pa.api) != 0) - { - DEBUG_ERROR("Failed to init signals"); - goto err_loop; - } - if (pa_threaded_mainloop_start(pa.loop) < 0) { DEBUG_ERROR("Failed to start the main loop"); goto err_loop; } - - pa_proplist * propList = pa_proplist_new(); - if (!propList) - { - DEBUG_ERROR("Failed to create the proplist"); - goto err_thread; - } - pa_proplist_sets(propList, PA_PROP_MEDIA_ROLE, "video"); + pa.loopStarted = true; pa_threaded_mainloop_lock(pa.loop); - pa.context = pa_context_new_with_proplist( - pa.api, - "Looking Glass", - propList); - if (!pa.context) + if (!pulseaudio_contextConnect_nl()) { - DEBUG_ERROR("Failed to create the context"); - goto err_context; + pa_threaded_mainloop_unlock(pa.loop); + goto err_thread; } - - pa_context_set_state_callback(pa.context, - pulseaudio_ctx_state_change_cb, NULL); - - if (pa_context_connect(pa.context, NULL, PA_CONTEXT_NOAUTOSPAWN, NULL) < 0) - { - DEBUG_ERROR("Failed to connect to the context server"); - goto err_context; - } - - for(;;) - { - pa_context_state_t state = pa_context_get_state(pa.context); - if(!PA_CONTEXT_IS_GOOD(state)) - { - DEBUG_ERROR("Context is bad"); - goto err_context; - } - - if (state == PA_CONTEXT_READY) - break; - - pa_threaded_mainloop_wait(pa.loop); - } - pa_threaded_mainloop_unlock(pa.loop); - pa_proplist_free(propList); return true; -err_context: - pa_threaded_mainloop_unlock(pa.loop); - pa_proplist_free(propList); - err_thread: - pa_threaded_mainloop_stop(pa.loop); + if (pa.loopStarted) + { + pa_threaded_mainloop_stop(pa.loop); + pa.loopStarted = false; + } err_loop: pa_threaded_mainloop_free(pa.loop); + pa.loop = NULL; + pa.api = NULL; err: return false; @@ -333,6 +501,7 @@ err: static void pulseaudio_sink_close_nl(void) { + pulseaudio_sinkDisarm_nl(); if (!pa.sink) return; @@ -347,51 +516,92 @@ static void pulseaudio_sink_close_nl(void) pa_operation_cancel(operation); pa_operation_unref(operation); } - pulseaudio_unrefOperation(pa_stream_flush(pa.sink, NULL, NULL)); + pa_stream_disconnect(pa.sink); pa_stream_unref(pa.sink); - pa.sink = NULL; - pa.sinkResamplerEnabled = false; - pa.sinkRateFailed = false; - pa.sinkNominalRate = 0; - pa.sinkAppliedRate = 0; - pa.sinkPendingRate = 0; - pa.sinkRequestedRate = 0; + pa.sink = NULL; + pa.sinkIndex = PA_INVALID_INDEX; + pa.sinkResamplerEnabled = false; + pa.sinkRateFailed = false; + pa.sinkNominalRate = 0; + pa.sinkAppliedRate = 0; + pa.sinkPendingRate = 0; + pa.sinkRequestedRate = 0; + pa.sinkDeferredRate = 0; + pa.sinkNextRateUpdate = 0; + pa.sinkRateDeferredSince = 0; + pa.sinkRateUpdateArmed = false; atomic_store_explicit( - &pa.sinkPresentationDeadline, 0, memory_order_release); + &pa.sinkLatencyNs, 0, memory_order_release); + atomic_store_explicit( + &pa.sinkLatencyUpdateRequested, false, memory_order_relaxed); } static void pulseaudio_free(void) { + if (!pa.loop) + return; + pa_threaded_mainloop_lock(pa.loop); pulseaudio_sink_close_nl(); - - pa_context_set_state_callback(pa.context, NULL, NULL); - pa_context_set_subscribe_callback(pa.context, NULL, NULL); - pa_context_disconnect(pa.context); - pa_context_unref(pa.context); - - if (pa.contextSub) - { - pa_operation_unref(pa.contextSub); - pa.contextSub = NULL; - } - + pulseaudio_context_close_nl(); pa_threaded_mainloop_unlock(pa.loop); + pulseaudio_reportErrors(true); + + if (pa.loopStarted) + { + pa_threaded_mainloop_stop(pa.loop); + pa.loopStarted = false; + } + pa_threaded_mainloop_free(pa.loop); + pa.loop = NULL; + pa.api = NULL; } static void pulseaudio_state_cb(pa_stream * p, void * userdata) { - if (pa.sinkStarting && pa_stream_get_state(pa.sink) == PA_STREAM_READY) - { - pulseaudio_unrefOperation(pa_stream_cork(pa.sink, 0, NULL, NULL)); - pa.sinkCorked = false; - pa.sinkStarting = false; - } - + const pa_stream_state_t state = pa_stream_get_state(p); + if (p == pa.sink && + (state == PA_STREAM_FAILED || state == PA_STREAM_TERMINATED)) + pulseaudio_sinkFailed_nl(); pa_threaded_mainloop_signal(pa.loop, 0); } +static void pulseaudio_start_cb( + pa_stream * stream, int success, void * userdata) +{ + const uint32_t serial = (uint32_t)(uintptr_t)userdata; + if (stream != pa.sink || serial != pa.sinkStartSerial) + return; + + pa_operation * operation = pa.sinkStartOperation; + pa.sinkStartOperation = NULL; + if (operation) + pa_operation_unref(operation); + pulseaudio_cancelStartTimer_nl(); + if (!success || pa_stream_get_state(stream) != PA_STREAM_READY || + !pa.context || + pa_context_get_state(pa.context) != PA_CONTEXT_READY) + { + pa.sinkCorked = true; + pulseaudio_sinkFailed_nl(); + } +} + +static void pulseaudio_startTimeout_cb(pa_mainloop_api * api, + pa_time_event * event, const struct timeval * tv, void * userdata) +{ + (void)tv; + const uint32_t serial = (uint32_t)(uintptr_t)userdata; + if (event != pa.sinkStartTimer || serial != pa.sinkStartSerial) + return; + + pa.sinkStartTimer = NULL; + api->time_free(event); + pa.sinkCorked = true; + pulseaudio_sinkFailed_nl(); +} + static void pulseaudio_write_cb(pa_stream * p, size_t nbytes, void * userdata) { // PulseAudio tries to pull data from the stream as soon as it is created for @@ -403,11 +613,17 @@ static void pulseaudio_write_cb(pa_stream * p, size_t nbytes, void * userdata) if (pa_stream_begin_write(p, (void **)&dst, &nbytes) < 0) { - DEBUG_ERROR("pa_stream_begin_write failed: %s", - pa_strerror(pa_context_errno(pa.context))); + pulseaudio_noteError(&pa.sinkWriteErrors); return; } + pa_usec_t latency = 0; + int negative = 0; + bool latencyValid = false; + if (atomic_exchange_explicit( + &pa.sinkLatencyUpdateRequested, false, memory_order_acquire)) + latencyValid = pa_stream_get_latency(p, &latency, &negative) == 0; + int frames = nbytes / pa.sinkStride; frames = pa.sinkPullFn(dst, frames); if (frames <= 0) @@ -419,8 +635,7 @@ static void pulseaudio_write_cb(pa_stream * p, size_t nbytes, void * userdata) if (pa_stream_write( p, dst, frames * pa.sinkStride, NULL, 0, PA_SEEK_RELATIVE) < 0) { - DEBUG_ERROR("pa_stream_write failed: %s", - pa_strerror(pa_context_errno(pa.context))); + pulseaudio_noteError(&pa.sinkWriteErrors); pa_stream_cancel_write(p); return; } @@ -428,26 +643,24 @@ static void pulseaudio_write_cb(pa_stream * p, size_t nbytes, void * userdata) /* Queue rate changes after the current audio block. This keeps the rate * reported by the preceding pull aligned with the block PulseAudio has * already received. */ - pulseaudio_submitRateUpdate(); + pulseaudio_submitRateUpdate(false); - pa_usec_t latency; - int negative; - if (pa_stream_get_latency(p, &latency, &negative) == 0) + if (latencyValid) { const int64_t latencyNs = negative ? 0 : (int64_t)latency * 1000; - atomic_store_explicit(&pa.sinkPresentationDeadline, - (int64_t)nanotime() + latencyNs, memory_order_release); + atomic_store_explicit( + &pa.sinkLatencyNs, latencyNs, memory_order_release); } } static void pulseaudio_underflow_cb(pa_stream * p, void * userdata) { - DEBUG_WARN("Underflow"); + pulseaudio_noteError(&pa.sinkUnderflows); } static void pulseaudio_overflow_cb(pa_stream * p, void * userdata) { - DEBUG_WARN("Overflow"); + pulseaudio_noteError(&pa.sinkOverflows); } static bool pulseaudio_setup(const LG_AudioFormat * format, @@ -456,6 +669,7 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, LG_AudioPullFn pullFn) { *resamplerEnabled = false; + pulseaudio_reportErrors(false); const int channels = format->channelCount; const int sampleRate = format->sampleRate; @@ -488,10 +702,25 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, }; pa_threaded_mainloop_lock(pa.loop); + if (!pa.context || + pa_context_get_state(pa.context) != PA_CONTEXT_READY) + { + pulseaudio_sink_close_nl(); + pulseaudio_context_close_nl(); + if (!pulseaudio_contextConnect_nl()) + { + pa_threaded_mainloop_unlock(pa.loop); + return false; + } + } + /* pa_stream_update_sample_rate requires protocol version 12. */ const bool enableResampler = requestResampler && pa_context_get_server_protocol_version(pa.context) >= 12; - if (pa.sink && !pa.sinkRateFailed && + if (pa.sink && pa.context && + pa_stream_get_state(pa.sink) == PA_STREAM_READY && + pa_context_get_state(pa.context) == PA_CONTEXT_READY && + !pa.sinkRateFailed && pa.sinkResamplerEnabled == enableResampler && !pa.sinkRateOperation && pa.sinkAppliedRate == pa.sinkNominalRate && @@ -511,13 +740,16 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, pa.sinkStride = stride; pa.sinkPullFn = pullFn; pa.sinkCorked = true; - pa.sinkStarting = false; pa.sinkResamplerEnabled = enableResampler; pa.sinkRateFailed = false; pa.sinkNominalRate = sampleRate; pa.sinkAppliedRate = sampleRate; pa.sinkPendingRate = sampleRate; pa.sinkRequestedRate = sampleRate; + pa.sinkDeferredRate = sampleRate; + pa.sinkNextRateUpdate = 0; + pa.sinkRateDeferredSince = 0; + pa.sinkRateUpdateArmed = false; pa.sink = pa_stream_new( pa.context, "Looking Glass", &spec, &channelMap); @@ -550,18 +782,44 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, return false; } - while (pa_stream_get_state(pa.sink) == PA_STREAM_CREATING) - pa_threaded_mainloop_wait(pa.loop); - - if (pa_stream_get_state(pa.sink) != PA_STREAM_READY) + pa_stream * sink = pa.sink; + pa_context * context = pa.context; + struct PulseWait wait = {0}; + pa_time_event * timer = pa_context_rttime_new(context, + pa_rtclock_now() + PULSEAUDIO_READY_TIMEOUT_US, + pulseaudio_timeout_cb, &wait); + if (!timer) { - DEBUG_ERROR("PulseAudio stream did not become ready: %s", - pa_strerror(pa_context_errno(pa.context))); + DEBUG_ERROR("Failed to create the PulseAudio stream setup timer"); pulseaudio_sink_close_nl(); pa_threaded_mainloop_unlock(pa.loop); return false; } + while (!wait.timedOut && pa.sink == sink && + pa_stream_get_state(sink) == PA_STREAM_CREATING && + pa.context == context && + PA_CONTEXT_IS_GOOD(pa_context_get_state(context))) + pa_threaded_mainloop_wait(pa.loop); + + const bool ready = pa.sink == sink && pa.context == context && + pa_stream_get_state(sink) == PA_STREAM_READY && + pa_context_get_state(context) == PA_CONTEXT_READY; + pa.api->time_free(timer); + + if (!ready) + { + if (wait.timedOut) + DEBUG_ERROR("Timed out setting up the PulseAudio stream"); + else + DEBUG_ERROR("PulseAudio stream did not become ready: %s", + pa_strerror(pa_context_errno(context))); + pulseaudio_sink_close_nl(); + pa_threaded_mainloop_unlock(pa.loop); + return false; + } + pa.sinkIndex = pa_stream_get_index(pa.sink); + const pa_buffer_attr * actual = pa_stream_get_buffer_attr(pa.sink); const uint64_t minRequestFrames = actual && actual->minreq != UINT32_MAX ? @@ -584,79 +842,123 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, *resamplerEnabled = pa.sinkResamplerEnabled; atomic_store_explicit( - &pa.sinkPresentationDeadline, 0, memory_order_release); + &pa.sinkLatencyNs, 0, memory_order_release); + atomic_store_explicit( + &pa.sinkLatencyUpdateRequested, false, memory_order_relaxed); pa_threaded_mainloop_unlock(pa.loop); return true; } -static void pulseaudio_start(void) +static bool pulseaudio_start( + LG_AudioFailureFn failureFn, uint32_t failureCookie) { - if (!pa.sink) - return; + pulseaudio_reportErrors(false); + if (!pa.loop || pa_threaded_mainloop_in_thread(pa.loop)) + return false; pa_threaded_mainloop_lock(pa.loop); - pa_stream_state_t state = pa_stream_get_state(pa.sink); - if (state == PA_STREAM_CREATING) - pa.sinkStarting = true; - else + pa_stream * sink = pa.sink; + pa_context * context = pa.context; + if (!sink || !context || + pa_stream_get_state(sink) != PA_STREAM_READY || + pa_context_get_state(context) != PA_CONTEXT_READY) { - pulseaudio_unrefOperation(pa_stream_cork(pa.sink, 0, NULL, NULL)); - pa.sinkCorked = false; + pa_threaded_mainloop_unlock(pa.loop); + return false; } + pulseaudio_sinkDisarm_nl(); + pa.sinkFailureFn = failureFn; + pa.sinkFailureCookie = failureCookie; + const uint32_t serial = pa.sinkStartSerial; + pa.sinkStartTimer = pa_context_rttime_new(context, + pa_rtclock_now() + PULSEAUDIO_OPERATION_TIMEOUT_US, + pulseaudio_startTimeout_cb, (void *)(uintptr_t)serial); + if (!pa.sinkStartTimer) + { + pulseaudio_sinkDisarm_nl(); + pa_threaded_mainloop_unlock(pa.loop); + return false; + } + + pa.sinkStartOperation = pa_stream_cork(sink, 0, + pulseaudio_start_cb, (void *)(uintptr_t)serial); + if (!pa.sinkStartOperation) + { + pulseaudio_sinkDisarm_nl(); + pa_threaded_mainloop_unlock(pa.loop); + return false; + } + pa.sinkCorked = false; + pa_threaded_mainloop_unlock(pa.loop); + return true; } static void pulseaudio_stop(void) { - if (!pa.sink) + if (!pa.loop) return; bool needLock = !pa_threaded_mainloop_in_thread(pa.loop); if (needLock) pa_threaded_mainloop_lock(pa.loop); - pulseaudio_unrefOperation(pa_stream_cork(pa.sink, 1, NULL, NULL)); - pa.sinkCorked = true; - pa.sinkStarting = false; + pulseaudio_sinkDisarm_nl(); + if (!pa.sink) + { + if (needLock) + pa_threaded_mainloop_unlock(pa.loop); + return; + } + + pa.sinkCorked = true; + if (pa_stream_get_state(pa.sink) == PA_STREAM_READY) + { + pulseaudio_trackControlOperation( + pa_stream_cork(pa.sink, 1, pulseaudio_streamControl_cb, NULL)); + pulseaudio_trackControlOperation( + pa_stream_flush(pa.sink, pulseaudio_streamControl_cb, NULL)); + } if (pa.sinkResamplerEnabled) { pa.sinkRequestedRate = pa.sinkNominalRate; - pulseaudio_submitRateUpdate(); + pulseaudio_submitRateUpdate(true); } atomic_store_explicit( - &pa.sinkPresentationDeadline, 0, memory_order_release); + &pa.sinkLatencyNs, 0, memory_order_release); + atomic_store_explicit( + &pa.sinkLatencyUpdateRequested, false, memory_order_relaxed); if (needLock) + { pa_threaded_mainloop_unlock(pa.loop); + pulseaudio_reportErrors(false); + } } static void pulseaudio_volume(int channels, const uint16_t volume[]) { - if (!pa.sink || !pa.sinkIndex) - return; - struct pa_cvolume v = { .channels = channels }; for(int i = 0; i < channels; ++i) v.values[i] = pa_sw_volume_from_linear( - 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787); + max(0.0, + 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787)); pa_threaded_mainloop_lock(pa.loop); - pulseaudio_unrefOperation(pa_context_set_sink_input_volume( - pa.context, pa.sinkIndex, &v, NULL, NULL)); + if (pa.sink && pa.sinkIndex != PA_INVALID_INDEX) + pulseaudio_trackControlOperation(pa_context_set_sink_input_volume( + pa.context, pa.sinkIndex, &v, pulseaudio_contextControl_cb, NULL)); pa_threaded_mainloop_unlock(pa.loop); } static void pulseaudio_mute(bool mute) { - if (!pa.sink || !pa.sinkIndex || pa.sinkMuted == mute) - return; - - pa.sinkMuted = mute; pa_threaded_mainloop_lock(pa.loop); - pulseaudio_unrefOperation(pa_context_set_sink_input_mute( - pa.context, pa.sinkIndex, mute, NULL, NULL)); + if (pa.sink && pa.sinkIndex != PA_INVALID_INDEX) + pulseaudio_trackControlOperation(pa_context_set_sink_input_mute( + pa.context, pa.sinkIndex, mute, pulseaudio_contextControl_cb, NULL)); pa_threaded_mainloop_unlock(pa.loop); } @@ -671,8 +973,53 @@ static bool pulseaudio_setRate(double * ratio) return false; pa.sinkRequestedRate = (uint32_t)llround(requestedRate); - const uint32_t scheduledRate = pa.sinkRateOperation ? - pa.sinkPendingRate : pa.sinkRequestedRate; + + uint32_t scheduledRate; + if (pa.sinkRateOperation) + { + scheduledRate = pa.sinkPendingRate; + if (pa.sinkRequestedRate == scheduledRate) + pa.sinkRateDeferredSince = 0; + else if (!pa.sinkRateDeferredSince || + pa.sinkDeferredRate != pa.sinkRequestedRate) + { + pa.sinkDeferredRate = pa.sinkRequestedRate; + pa.sinkRateDeferredSince = (int64_t)nanotime(); + } + } + else + { + scheduledRate = pa.sinkAppliedRate; + pa.sinkRateUpdateArmed = false; + if (pa.sinkRequestedRate == scheduledRate) + pa.sinkRateDeferredSince = 0; + else + { + const int64_t now = (int64_t)nanotime(); + if (!pa.sinkRateDeferredSince || + pa.sinkDeferredRate != pa.sinkRequestedRate) + { + pa.sinkDeferredRate = pa.sinkRequestedRate; + pa.sinkRateDeferredSince = now; + } + + const uint32_t difference = pa.sinkRequestedRate > scheduledRate ? + pa.sinkRequestedRate - scheduledRate : + scheduledRate - pa.sinkRequestedRate; + /* Coalesce controller noise around an integer-Hz boundary while still + * applying a persistent one-Hz correction. Larger corrections only + * wait for the operation-rate limit. */ + if (now >= pa.sinkNextRateUpdate && + (difference > PULSEAUDIO_RATE_UPDATE_DEADBAND_HZ || + now - pa.sinkRateDeferredSince >= + PULSEAUDIO_RATE_UPDATE_MAX_HOLD_NS)) + { + pa.sinkRateUpdateArmed = true; + scheduledRate = pa.sinkRequestedRate; + } + } + } + if (!scheduledRate) return false; @@ -682,12 +1029,15 @@ static bool pulseaudio_setRate(double * ratio) static uint64_t pulseaudio_latency(void) { - const int64_t deadline = atomic_load_explicit( - &pa.sinkPresentationDeadline, memory_order_acquire); - if (deadline <= 0) + atomic_store_explicit( + &pa.sinkLatencyUpdateRequested, true, memory_order_release); + + const int64_t latencyNs = atomic_load_explicit( + &pa.sinkLatencyNs, memory_order_acquire); + if (latencyNs <= 0) return 0; - return max(INT64_C(0), deadline - (int64_t)nanotime()) / 1000; + return latencyNs / 1000; } struct LG_AudioDevOps LGAD_PulseAudio = diff --git a/client/include/interface/audio.h b/client/include/interface/audio.h index d1564736..2b80ac6b 100644 --- a/client/include/interface/audio.h +++ b/client/include/interface/audio.h @@ -106,7 +106,8 @@ typedef struct LG_AudioEventOps { /* Format and clock pointers are borrowed for the duration of each call. * A NULL source clock indicates that the provider has no usable clock. - * Providers must serialize event delivery for an attachment. Stream + * Providers must serialize event delivery within each stream direction. + * Playback and recording events may be delivered concurrently. Stream * generations are nonzero and uniquely identify each stream instance. */ void (*playbackStart)(void * opaque, uint32_t generation, const LG_AudioFormat * format, const LG_AudioClock * sourceClock); @@ -163,7 +164,9 @@ typedef struct LG_AudioOps * audio callback with the measured playback device clock. Its position uses * the device's independent output-frame timeline and its time includes the * backend's estimated presentation latency. targetRate is the source frame - * rate requested by the client's buffer controller. */ + * rate requested by the client's buffer controller. Return false while + * active rate feedback is unavailable; sustained failure makes the client + * restart playback with local rate control. */ bool (*clockFeedback)(void * opaque, uint32_t generation, const LG_AudioClock * playbackClock, double targetRate); } diff --git a/client/include/interface/audiodev.h b/client/include/interface/audiodev.h index 548dfec0..b08bf404 100644 --- a/client/include/interface/audiodev.h +++ b/client/include/interface/audiodev.h @@ -28,7 +28,9 @@ #include "interface/audio.h" typedef int (*LG_AudioPullFn)(uint8_t * dst, int frames); -typedef void (*LG_AudioPushFn)(uint8_t * src, int frames); +typedef bool (*LG_AudioPushFn)(uint8_t * src, int frames, + const LG_AudioClock * sourceClock); +typedef void (*LG_AudioFailureFn)(uint32_t cookie); struct LG_AudioDevOps { @@ -55,10 +57,13 @@ struct LG_AudioDevOps bool requestResampler, bool * resamplerEnabled, int * maxPeriodFrames, int * startFrames, LG_AudioPullFn pullFn); - /* called when there is data available to start playback */ - void (*start)(void); + /* Called when there is data available to start playback. failureFn may + * report an asynchronous failure using the supplied cookie. Returning + * false means the failure callback is already quiescent. */ + bool (*start)(LG_AudioFailureFn failureFn, uint32_t cookie); - /* called when the source reports the audio stream has stopped */ + /* Called when the source reports the audio stream has stopped. This must + * synchronously quiesce the failure callback before returning. */ void (*stop)(void); /* [optional] called to set the volume of the channels */ @@ -80,10 +85,17 @@ struct LG_AudioDevOps struct { - /* start the record stream using the requested interleaved format */ - void (*start)(const LG_AudioFormat * format, LG_AudioPushFn pushFn); + /* Start the record stream using the requested interleaved format. + * sourceClock is borrowed for the duration of pushFn and describes the + * first frame. It is NULL when the backend has no source clock. pushFn + * returns false when the active provider rejects the frames. + * failureFn may report an asynchronous failure using the supplied cookie. + * Returning false means both callbacks are already quiescent. */ + bool (*start)(const LG_AudioFormat * format, LG_AudioPushFn pushFn, + LG_AudioFailureFn failureFn, uint32_t cookie); - /* called when the source reports the audio stream has stopped */ + /* Called when the source reports the audio stream has stopped. This must + * synchronously quiesce both callbacks before returning. */ void (*stop)(void); /* [optional] called to set the volume of the channels */ diff --git a/client/src/audio.c b/client/src/audio.c index 26c578cb..2211d640 100644 --- a/client/src/audio.c +++ b/client/src/audio.c @@ -32,10 +32,12 @@ #include "dynamic/audiodev.h" +#include #include #include #include #include +#include #include #include #include @@ -49,6 +51,10 @@ #define PLAYBACK_MAX_RATE_CORRECTION 0.005 #define PLAYBACK_MAX_RATE_SLEW_PER_SEC 0.005 #define PLAYBACK_MAX_JITTER_SEC 0.1 +#define PLAYBACK_JITTER_DECAY_SEC 10.0 +#define AUDIO_START_RETRY_MIN_NS INT64_C(5000000) +#define AUDIO_START_RETRY_MAX_NS INT64_C(250000000) +#define AUDIO_RETRY_RESET_NS INT64_C(1000000000) #define PLAYBACK_PHASE_BASELINE_TIME_SEC 5.0 #define PLAYBACK_PHASE_RESERVE_DECAY_SEC 60.0 /* libsamplerate does not expose its buffered-frame delay. SRC_SINC_FASTEST @@ -65,7 +71,9 @@ #define PLAYBACK_DEVICE_RATE_STABLE_SEC 2.0 #define PLAYBACK_DEVICE_RATE_STABLE_DELTA_PPM 50.0 #define PLAYBACK_DEVICE_RATE_MAX_ACQUIRE_SEC 20.0 +#define PLAYBACK_MAX_SOURCE_PACKET_MS 100 #define PLAYBACK_FEEDBACK_INTERVAL_NS INT64_C(1000000) +#define PLAYBACK_FEEDBACK_FAILURE_NS INT64_C(100000000) #define PLAYBACK_GRAPH_INTERVAL_NS INT64_C(25000000) typedef enum @@ -80,6 +88,15 @@ typedef enum } StreamState; +typedef enum +{ + PLAYBACK_DATA_DROP, + PLAYBACK_DATA_PROCESSED, + PLAYBACK_DATA_RETRY, + PLAYBACK_DATA_RETRY_NOW +} +PlaybackDataResult; + typedef enum { PLAYBACK_RATE_SOFTWARE, @@ -88,6 +105,15 @@ typedef enum } PlaybackRateControl; +typedef enum +{ + PLAYBACK_DIAGNOSTIC_REGISTER_GRAPH = 1U << 0, + PLAYBACK_DIAGNOSTIC_INVALIDATE = 1U << 1, + PLAYBACK_DIAGNOSTIC_START_LOG = 1U << 2, + PLAYBACK_DIAGNOSTIC_SYNC_LOG = 1U << 3, +} +PlaybackDiagnosticFlags; + #define STREAM_ACTIVE(state) \ (state == STREAM_STATE_RUN || \ state == STREAM_STATE_KEEP_ALIVE || \ @@ -95,6 +121,8 @@ PlaybackRateControl; #define PLAYBACK_CALLBACK_DISABLED (UINT32_C(1) << 31) #define PLAYBACK_CALLBACK_COUNT_MASK (PLAYBACK_CALLBACK_DISABLED - 1) +#define ACTIVE_CALLBACK_WAITING (UINT32_C(1) << 31) +#define ACTIVE_CALLBACK_COUNT_MASK (ACTIVE_CALLBACK_WAITING - 1) typedef struct { @@ -129,6 +157,7 @@ typedef struct float * framesOut; int framesInSize; int framesOutSize; + int maxSourcePacketFrames; int64_t inputPosition; int64_t outputPosition; @@ -195,11 +224,43 @@ typedef struct } PlaybackDeviceTiming; +typedef struct +{ + double targetLatencyMs; + double queuedLatencyMs; +} +PlaybackStartDiagnostics; + +typedef struct +{ + double softwareLatencyMs; + double targetLatencyMs; + double sourcePpm; + double devicePpm; + double controlPpm; + double jitterMs; + unsigned int underruns; + unsigned int overruns; + PlaybackRateControl rateControl; +} +PlaybackSyncDiagnostics; + +typedef struct +{ + unsigned int epoch; + unsigned int pending; + float graphMax; + PlaybackStartDiagnostics start; + PlaybackSyncDiagnostics sync; +} +PlaybackDiagnostics; + typedef struct { const LG_AudioOps * ops; void * opaque; bool available; + uint32_t epoch; uint32_t generation; } AudioBinding; @@ -207,9 +268,14 @@ AudioBinding; typedef struct { struct LG_AudioDevOps * audioDev; + atomic_bool ready; - LG_Lock providerLock; - LG_RWLock activeLock; + LG_Lock bindingLock; + LG_Lock providerLock; + LG_RWLock activeLock; + uint32_t nextBindingEpoch; + atomic_uint activeCallbacks; + LGEvent * activeIdle; AudioBinding fallback; AudioBinding transport; AudioBinding active; @@ -217,25 +283,54 @@ typedef struct struct { LG_Lock sourceLock; + LG_Lock deviceLock; + struct + { + sem_t wake; + bool wakeInitialized; + atomic_bool wakePending; + LGThread * thread; + atomic_bool stop; + } + worker; _Atomic(StreamState) state; atomic_uint callbackState; + sem_t callbackIdle; + bool callbackIdleInitialized; atomic_uint streamGeneration; - int volumeChannels; - uint16_t volume[LG_AUDIO_MAX_CHANNELS]; - bool mute; - LG_AudioFormat format; - LG_AudioFormat lastFormat; - bool lastFormatValid; - int channels; - int sampleRate; - int stride; - bool convertToFloat; - int deviceMaxPeriodFrames; - int deviceStartFrames; - int targetStartFrames; - int startupLowWaterFrames; - int64_t startupPacketDeadline; - int64_t startupPacketPeriod; + uint64_t requestSerial; + uint32_t requestedGeneration; + LG_AudioFormat requestedFormat; + bool requestedFormatValid; + bool requestedProviderRateControl; + bool forceSoftwareResampler; + bool startPending; + bool startInProgress; + unsigned int startFailures; + int64_t nextStartRetry; + atomic_uint activeAttemptSerial; + atomic_uint failedAttemptSerial; + uint32_t nextAttemptSerial; + uint64_t backendRequestSerial; + int64_t backendStartTime; + uint64_t controlSerial; + bool controlsPending; + int volumeChannels; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool mute; + LG_AudioFormat format; + LG_AudioFormat lastFormat; + bool lastFormatValid; + int channels; + int sampleRate; + int stride; + bool convertToFloat; + int deviceMaxPeriodFrames; + int deviceStartFrames; + int targetStartFrames; + int startupLowWaterFrames; + int64_t startupPacketDeadline; + int64_t startupPacketPeriod; PlaybackRateControl rateControl; bool lastProviderRateControl; _Atomic(double) backendResampleRatio; @@ -244,8 +339,12 @@ typedef struct PlaybackDeviceTiming deviceTiming; atomic_uint underruns; - RingBuffer timings; - GraphHandle graph; + RingBuffer timings; + GraphHandle graph; + atomic_uint diagnosticsEpoch; + atomic_bool graphReady; + bool graphRegistrationAttempted; + PlaybackDiagnostics diagnostics; /* These two structs contain data specifically for use in the device and * source data threads respectively. Keep them on separate cache lines to @@ -257,33 +356,47 @@ typedef struct struct { - LG_Lock lock; - atomic_uint streamGeneration; - bool shuttingDown; - bool requested; - bool started; - int volumeChannels; - uint16_t volume[LG_AUDIO_MAX_CHANNELS]; - bool mute; - LG_AudioFormat format; - LG_AudioFormat lastFormat; - MsgBoxHandle confirmHandle; - uint64_t confirmGeneration; - bool confirmPending; - LG_AudioFormat confirmFormat; + LG_RWLock lock; + LGEvent * wake; + LGThread * thread; + atomic_bool workerAlive; + atomic_uint deliverySerial; + atomic_uint failedAttemptSerial; + uint32_t nextAttemptSerial; + AudioBinding deliveryBinding; + uint32_t deliveryGeneration; + bool shuttingDown; + bool requested; + bool enabled; + unsigned int startFailures; + int64_t nextStartRetry; + uint64_t requestSerial; + AudioBinding requestedBinding; + uint32_t requestedGeneration; + LG_AudioFormat requestedFormat; + int volumeChannels; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool mute; + uint64_t controlSerial; + MsgBoxHandle confirmHandle; + uint64_t confirmGeneration; + bool confirmPending; } record; struct { - LG_Lock lock; - LGEvent * event; - LGThread * thread; + LG_Lock lock; + sem_t wake; + bool wakeInitialized; + atomic_bool wakePending; + LGThread * thread; atomic_bool stop; bool pending; const LG_AudioOps * ops; void * opaque; + uint32_t bindingEpoch; uint32_t bindingGeneration; uint32_t generation; LG_AudioClock clock; @@ -703,7 +816,8 @@ static void playbackPublishDeviceTiming( double outputPosition) { PlaybackDeviceTiming * timing = &audio.playback.deviceTiming; - atomic_fetch_add_explicit(&timing->sequence, 1, memory_order_relaxed); + /* Publish the odd writer marker before any snapshot field. */ + atomic_fetch_add_explicit(&timing->sequence, 1, memory_order_seq_cst); atomic_store_explicit( &timing->periodFrames, periodFrames, memory_order_relaxed); atomic_store_explicit(&timing->time, time, memory_order_relaxed); @@ -824,10 +938,14 @@ static int64_t playbackMapMediaTime(PlaybackSourceData * sourceData, } static void playbackStop(void); +static bool playbackEnsureConversionBuffers( + PlaybackSourceData * sourceData, int frames); static MsgBoxHandle recordCancelConfirmLocked(void); -static void realRecordStartLocked(const LG_AudioFormat * format); -static void realRecordStopLocked(void); -static void recordStop(void); +static void recordStop( + const AudioBinding * binding, uint32_t generation); +static bool eventBegin( + const AudioBinding * binding, AudioBinding * active); +static void eventEnd(void); static StreamState playbackGetState(void) { @@ -861,8 +979,12 @@ static bool playbackCallbackEnter(void) static void playbackCallbackExit(void) { - atomic_fetch_sub_explicit( + const unsigned int previous = atomic_fetch_sub_explicit( &audio.playback.callbackState, 1, memory_order_release); + if ((previous & PLAYBACK_CALLBACK_DISABLED) && + (previous & PLAYBACK_CALLBACK_COUNT_MASK) == 1 && + audio.playback.callbackIdleInitialized) + sem_post(&audio.playback.callbackIdle); } static void playbackDisableCallbacks(void) @@ -877,12 +999,91 @@ static void playbackWaitForCallbacks(void) while((atomic_load_explicit( &audio.playback.callbackState, memory_order_acquire) & PLAYBACK_CALLBACK_COUNT_MASK) != 0) - ; + { + int result; + do + result = sem_wait(&audio.playback.callbackIdle); + while (result < 0 && errno == EINTR); + + if (result < 0) + break; + } +} + +static void playbackWorkerWake(void) +{ + if (!audio.playback.worker.wakeInitialized || + !audio.playback.worker.thread || + atomic_exchange_explicit( + &audio.playback.worker.wakePending, true, memory_order_acq_rel)) + return; + + if (sem_post(&audio.playback.worker.wake) < 0 && errno != EOVERFLOW) + atomic_store_explicit( + &audio.playback.worker.wakePending, false, memory_order_release); +} + +static void feedbackWorkerWake(void) +{ + if (!audio.feedback.wakeInitialized || !audio.feedback.thread || + atomic_exchange_explicit( + &audio.feedback.wakePending, true, memory_order_acq_rel)) + return; + + if (sem_post(&audio.feedback.wake) < 0 && errno != EOVERFLOW) + atomic_store_explicit( + &audio.feedback.wakePending, false, memory_order_release); +} + +static void playbackQueueStop(void) +{ + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + playbackDisableCallbacks(); + playbackSetState(STREAM_STATE_STOP_PENDING); + playbackWorkerWake(); +} + +static void playbackQueueSourceStop(void) +{ + atomic_store_explicit( + &audio.playback.activeAttemptSerial, 0, memory_order_release); + playbackQueueStop(); +} + +static void playbackBackendFailed(uint32_t attemptSerial) +{ + unsigned int expected = attemptSerial; + if (!attemptSerial || !atomic_compare_exchange_strong_explicit( + &audio.playback.activeAttemptSerial, &expected, 0, + memory_order_acq_rel, memory_order_acquire)) + return; + + atomic_store_explicit(&audio.playback.failedAttemptSerial, + attemptSerial, memory_order_release); + playbackQueueStop(); +} + +/* sourceLock must be held. */ +static uint32_t playbackArmBackend(void) +{ + if (!++audio.playback.nextAttemptSerial) + ++audio.playback.nextAttemptSerial; + + const uint32_t attemptSerial = audio.playback.nextAttemptSerial; + atomic_store_explicit( + &audio.playback.failedAttemptSerial, 0, memory_order_release); + audio.playback.backendRequestSerial = audio.playback.requestSerial; + audio.playback.backendStartTime = 0; + atomic_store_explicit(&audio.playback.activeAttemptSerial, + attemptSerial, memory_order_release); + return attemptSerial; } bool lgAudio_supportsPlayback(void) { - return audio.audioDev && audio.audioDev->playback.start; + return atomic_load_explicit(&audio.ready, memory_order_acquire) && + audio.audioDev && audio.audioDev->playback.start; } static const char * audioGraphFormatFn(const char * name, @@ -895,15 +1096,39 @@ static const char * audioGraphFormatFn(const char * name, return title; } +/* sourceLock must be held. */ +static PlaybackDiagnostics * playbackDiagnosticsLocked(void) +{ + PlaybackDiagnostics * diagnostics = &audio.playback.diagnostics; + const unsigned int epoch = atomic_load_explicit( + &audio.playback.diagnosticsEpoch, memory_order_acquire); + if (diagnostics->epoch != epoch) + *diagnostics = (PlaybackDiagnostics) { .epoch = epoch }; + + return diagnostics; +} + static void playbackStop(void) { - if (playbackGetState() == STREAM_STATE_STOP) + const bool alreadyStopped = playbackGetState() == STREAM_STATE_STOP; + playbackDisableCallbacks(); + atomic_store_explicit( + &audio.playback.activeAttemptSerial, 0, memory_order_release); + if (!alreadyStopped) + audio.audioDev->playback.stop(); + playbackWaitForCallbacks(); + atomic_store_explicit( + &audio.playback.failedAttemptSerial, 0, memory_order_release); + audio.playback.backendRequestSerial = 0; + audio.playback.backendStartTime = 0; + + if (alreadyStopped) return; - playbackDisableCallbacks(); - audio.audioDev->playback.stop(); - playbackWaitForCallbacks(); - + atomic_store_explicit( + &audio.playback.graphReady, false, memory_order_release); + atomic_fetch_add_explicit( + &audio.playback.diagnosticsEpoch, 1, memory_order_release); playbackSetState(STREAM_STATE_STOP); ringbuffer_free(&audio.playback.buffer); audio.playback.sourceData.src = src_delete(audio.playback.sourceData.src); @@ -917,14 +1142,16 @@ static void playbackStop(void) audio.playback.sourceData.framesInSize = 0; audio.playback.sourceData.framesOutSize = 0; } + audio.playback.sourceData.maxSourcePacketFrames = 0; if (audio.playback.timings) { if (audio.playback.graph) app_unregisterGraph(audio.playback.graph); - audio.playback.graph = NULL; ringbuffer_free(&audio.playback.timings); } + audio.playback.graph = NULL; + audio.playback.graphRegistrationAttempted = false; } static int playbackPullFrames(uint8_t * dst, int frames) @@ -998,7 +1225,10 @@ static int playbackPullFrames(uint8_t * dst, int frames) data->startupSilenceFrames = -offset; } - playbackSetState(STREAM_STATE_RUN); + StreamState expected = STREAM_STATE_SETUP_DEVICE; + atomic_compare_exchange_strong_explicit( + &audio.playback.state, &expected, STREAM_STATE_RUN, + memory_order_acq_rel, memory_order_acquire); } /* Timestamp the dequeue boundary before the current pull. The logical @@ -1035,6 +1265,8 @@ static int playbackPullFrames(uint8_t * dst, int frames) // Close the stream if nothing has played for a while if (audio.playback.buffer && + audio.playback.worker.wakeInitialized && + audio.playback.worker.thread && playbackGetState() == STREAM_STATE_KEEP_ALIVE) { int stopTimeSec = 30; @@ -1047,7 +1279,7 @@ static int playbackPullFrames(uint8_t * dst, int frames) memory_order_acq_rel, memory_order_acquire)) { playbackDisableCallbacks(); - audio.audioDev->playback.stop(); + playbackWorkerWake(); frames = 0; } } @@ -1074,25 +1306,26 @@ static bool playbackSetupDevice(const LG_AudioFormat * format, audio.playback.deviceStartFrames >= 0; } -static void playbackStart(const LG_AudioFormat * format, - const LG_AudioClock * sourceClock, bool providerRateControl) +static bool playbackStart(const LG_AudioFormat * format, + const LG_AudioClock * sourceClock, bool providerRateControl, + bool forceSoftwareResampler) { if (!audio.audioDev) - return; + return false; if (!audioFormatValid(format)) { DEBUG_ERROR("Invalid playback format"); if (playbackGetState() != STREAM_STATE_STOP) playbackStop(); - return; + return false; } const int channels = format->channelCount; const int sampleRate = format->sampleRate; StreamState state = playbackGetState(); - if (state == STREAM_STATE_KEEP_ALIVE && + if (!forceSoftwareResampler && state == STREAM_STATE_KEEP_ALIVE && audio.playback.lastFormatValid && audio.playback.lastProviderRateControl == providerRateControl && audioFormatEqual(format, &audio.playback.lastFormat)) @@ -1104,7 +1337,7 @@ static void playbackStart(const LG_AudioFormat * format, { playbackPrepareMediaClock( &audio.playback.sourceData, sourceClock); - return; + return true; } state = expected; @@ -1120,6 +1353,8 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.channels = channels; audio.playback.sampleRate = sampleRate; + audio.playback.sourceData.maxSourcePacketFrames = max( + (sampleRate * PLAYBACK_MAX_SOURCE_PACKET_MS + 999) / 1000, 1); playbackSetState(STREAM_STATE_SETUP_SOURCE); audio.playback.deviceData.nextPosition = 0; @@ -1180,6 +1415,7 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.startupPacketDeadline = 0; audio.playback.startupPacketPeriod = 0; const bool requestBackendResampler = !providerRateControl && + !forceSoftwareResampler && g_params.audioResampler != AUDIO_RESAMPLER_LIBSAMPLERATE; LG_AudioFormat deviceFormat = *format; @@ -1211,7 +1447,7 @@ static void playbackStart(const LG_AudioFormat * format, { DEBUG_ERROR("Failed to configure audio playback device"); playbackStop(); - return; + return false; } audio.playback.stride = channels * @@ -1223,12 +1459,21 @@ static void playbackStart(const LG_AudioFormat * format, PLAYBACK_RATE_PROVIDER : backendResampler ? PLAYBACK_RATE_BACKEND : PLAYBACK_RATE_SOFTWARE; + const int conversionBufferFrames = max(requestedPeriodFrames, + audio.playback.sourceData.maxSourcePacketFrames); + if (!playbackEnsureConversionBuffers( + &audio.playback.sourceData, conversionBufferFrames)) + { + playbackStop(); + return false; + } + audio.playback.buffer = ringbuffer_newUnbounded( sampleRate, audio.playback.stride); if (!audio.playback.buffer) { playbackStop(); - return; + return false; } if (g_params.audioResampler == AUDIO_RESAMPLER_BACKEND && @@ -1245,7 +1490,7 @@ static void playbackStart(const LG_AudioFormat * format, { DEBUG_ERROR("Failed to create resampler: %s", src_strerror(srcError)); playbackStop(); - return; + return false; } } else @@ -1266,22 +1511,383 @@ static void playbackStart(const LG_AudioFormat * format, break; } - // if a volume level was stored, set it before we return - if (audio.playback.volumeChannels) - audio.audioDev->playback.volume( - audio.playback.volumeChannels, - audio.playback.volume); - - // set the inital mute state - if (audio.audioDev->playback.mute) - audio.audioDev->playback.mute(audio.playback.mute); - // Set up synchronization instrumentation only when explicitly requested. if (g_params.audioDebug) audio.playback.timings = ringbuffer_new(1200, sizeof(float)); atomic_store_explicit( &audio.playback.callbackState, 0, memory_order_release); + return true; +} + +static int64_t audioStartRetryDelay(unsigned int failures) +{ + const unsigned int shift = min(failures, 6U); + return min(AUDIO_START_RETRY_MIN_NS << shift, + AUDIO_START_RETRY_MAX_NS); +} + +/* sourceLock must be held. */ +static bool playbackDelayStart(void) +{ + const unsigned int failures = audio.playback.startFailures; + audio.playback.nextStartRetry = nanotime() + + audioStartRetryDelay(failures); + if (audio.playback.startFailures < 7U) + ++audio.playback.startFailures; + + return failures == 0U || failures == 6U; +} + +/* sourceLock must be held. */ +static void playbackScheduleStart(void) +{ + if (!audio.playback.requestedGeneration || + !audio.playback.requestedFormatValid || + audio.playback.startPending || + audio.playback.startInProgress || + (atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == + audio.playback.requestedGeneration && + playbackGetState() != STREAM_STATE_STOP) || + nanotime() < audio.playback.nextStartRetry) + return; + + audio.playback.startPending = true; + playbackWorkerWake(); +} + +static void playbackProcessControls(void) +{ + uint64_t controlSerial; + uint32_t generation; + int volumeChannels; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool mute; + + LG_LOCK(audio.playback.sourceLock); + generation = atomic_load_explicit( + &audio.playback.streamGeneration, memory_order_acquire); + if (!audio.playback.controlsPending || !generation || + generation != audio.playback.requestedGeneration) + { + LG_UNLOCK(audio.playback.sourceLock); + return; + } + + controlSerial = audio.playback.controlSerial; + volumeChannels = audio.playback.volumeChannels; + if (volumeChannels) + memcpy(volume, audio.playback.volume, + sizeof(*volume) * volumeChannels); + mute = audio.playback.mute; + LG_UNLOCK(audio.playback.sourceLock); + + bool applied = false; + LG_LOCK(audio.playback.deviceLock); + const StreamState state = playbackGetState(); + if (audio.audioDev && state != STREAM_STATE_STOP && + state != STREAM_STATE_STOP_PENDING && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + { + if (volumeChannels && audio.audioDev->playback.volume) + audio.audioDev->playback.volume(volumeChannels, volume); + + if (audio.audioDev->playback.mute) + audio.audioDev->playback.mute(mute); + applied = true; + } + LG_UNLOCK(audio.playback.deviceLock); + + if (!applied) + return; + + LG_LOCK(audio.playback.sourceLock); + if (audio.playback.requestedGeneration == generation && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation && + audio.playback.controlSerial == controlSerial) + audio.playback.controlsPending = false; + LG_UNLOCK(audio.playback.sourceLock); +} + +static void playbackProcessStopPending(void) +{ + bool pending = false; + bool reportFailure = false; + uint32_t failedAttempt = 0; + uint64_t backendRequestSerial = 0; + int64_t backendStartTime = 0; + + LG_LOCK(audio.playback.sourceLock); + if (audio.audioDev && + playbackGetState() == STREAM_STATE_STOP_PENDING) + { + failedAttempt = atomic_load_explicit( + &audio.playback.failedAttemptSerial, memory_order_acquire); + backendRequestSerial = audio.playback.backendRequestSerial; + backendStartTime = audio.playback.backendStartTime; + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + pending = true; + } + LG_UNLOCK(audio.playback.sourceLock); + + if (!pending) + return; + + LG_LOCK(audio.playback.deviceLock); + playbackStop(); + LG_UNLOCK(audio.playback.deviceLock); + + LG_LOCK(audio.playback.sourceLock); + if (failedAttempt && audio.playback.requestedGeneration && + audio.playback.requestedFormatValid && + backendRequestSerial == audio.playback.requestSerial) + { + if (backendStartTime && + nanotime() - backendStartTime >= AUDIO_RETRY_RESET_NS) + { + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + } + + audio.playback.startPending = false; + reportFailure = playbackDelayStart(); + } + LG_UNLOCK(audio.playback.sourceLock); + + if (reportFailure) + DEBUG_ERROR("Audio playback device failed; retrying"); +} + +static void playbackProcessStart(void) +{ + LG_AudioFormat format; + uint64_t requestSerial; + uint32_t generation; + bool providerRateControl; + bool forceSoftwareResampler; + + LG_LOCK(audio.playback.sourceLock); + if (!audio.playback.startPending || + audio.playback.startInProgress || + !audio.playback.requestedGeneration || + !audio.playback.requestedFormatValid || + nanotime() < audio.playback.nextStartRetry) + { + LG_UNLOCK(audio.playback.sourceLock); + return; + } + + audio.playback.startPending = false; + audio.playback.startInProgress = true; + format = audio.playback.requestedFormat; + requestSerial = audio.playback.requestSerial; + generation = audio.playback.requestedGeneration; + providerRateControl = + audio.playback.requestedProviderRateControl; + forceSoftwareResampler = + audio.playback.forceSoftwareResampler; + LG_UNLOCK(audio.playback.sourceLock); + + LG_LOCK(audio.playback.deviceLock); + const bool started = playbackStart(&format, NULL, + providerRateControl, forceSoftwareResampler); + LG_UNLOCK(audio.playback.deviceLock); + + bool stopStale = false; + bool wake = false; + LG_LOCK(audio.playback.sourceLock); + const bool current = + requestSerial == audio.playback.requestSerial && + generation == audio.playback.requestedGeneration && + audio.playback.requestedFormatValid && + audioFormatEqual(&format, &audio.playback.requestedFormat) && + providerRateControl == + audio.playback.requestedProviderRateControl && + forceSoftwareResampler == + audio.playback.forceSoftwareResampler; + + audio.playback.startInProgress = false; + if (started && current) + { + audio.playback.startPending = false; + LG_LOCK(audio.playback.deviceLock); + if (playbackGetState() != STREAM_STATE_STOP_PENDING) + { + if (atomic_load_explicit(&audio.playback.activeAttemptSerial, + memory_order_acquire)) + audio.playback.backendRequestSerial = requestSerial; + atomic_store_explicit(&audio.playback.streamGeneration, + generation, memory_order_release); + audio.playback.controlsPending = true; + audio.playback.nextStartRetry = 0; + } + LG_UNLOCK(audio.playback.deviceLock); + } + else + { + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + if (started) + stopStale = true; + else if (current) + playbackDelayStart(); + } + + wake = audio.playback.startPending; + LG_UNLOCK(audio.playback.sourceLock); + + if (stopStale) + { + LG_LOCK(audio.playback.deviceLock); + playbackStop(); + LG_UNLOCK(audio.playback.deviceLock); + } + + if (wake) + playbackWorkerWake(); +} + +static void playbackProcessDeviceStart(void) +{ + uint64_t requestSerial = 0; + uint32_t generation = 0; + uint32_t attemptSerial = 0; + + LG_LOCK(audio.playback.sourceLock); + const StreamState state = playbackGetState(); + attemptSerial = atomic_load_explicit( + &audio.playback.activeAttemptSerial, memory_order_acquire); + generation = atomic_load_explicit( + &audio.playback.streamGeneration, memory_order_acquire); + if (audio.audioDev && state == STREAM_STATE_SETUP_DEVICE && + attemptSerial && !audio.playback.backendStartTime && generation && + generation == audio.playback.requestedGeneration) + requestSerial = audio.playback.requestSerial; + else + attemptSerial = 0; + LG_UNLOCK(audio.playback.sourceLock); + + if (!attemptSerial) + return; + + bool started = false; + LG_LOCK(audio.playback.deviceLock); + if (audio.audioDev && + playbackGetState() == STREAM_STATE_SETUP_DEVICE && + atomic_load_explicit(&audio.playback.activeAttemptSerial, + memory_order_acquire) == attemptSerial && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + started = audio.audioDev->playback.start( + playbackBackendFailed, attemptSerial); + LG_UNLOCK(audio.playback.deviceLock); + + LG_LOCK(audio.playback.sourceLock); + const StreamState completedState = playbackGetState(); + const bool attemptCurrent = attemptSerial == atomic_load_explicit( + &audio.playback.activeAttemptSerial, memory_order_acquire); + const bool current = started && + attemptCurrent && + requestSerial == audio.playback.requestSerial && + requestSerial == audio.playback.backendRequestSerial && + generation == audio.playback.requestedGeneration && + generation == atomic_load_explicit( + &audio.playback.streamGeneration, memory_order_acquire) && + (completedState == STREAM_STATE_SETUP_DEVICE || + completedState == STREAM_STATE_RUN); + const bool keepAlive = started && attemptCurrent && + completedState == STREAM_STATE_KEEP_ALIVE; + if (current || keepAlive) + audio.playback.backendStartTime = nanotime(); + LG_UNLOCK(audio.playback.sourceLock); + + if (!current && !keepAlive) + playbackBackendFailed(attemptSerial); +} + +static void playbackProcessDiagnostics(void) +{ + PlaybackDiagnostics diagnostics; + RingBuffer timings; + + LG_LOCK(audio.playback.sourceLock); + if (!audio.playback.diagnostics.pending) + { + LG_UNLOCK(audio.playback.sourceLock); + return; + } + + diagnostics = audio.playback.diagnostics; + timings = audio.playback.timings; + audio.playback.diagnostics.pending = 0; + LG_UNLOCK(audio.playback.sourceLock); + + bool current = false; + double backendLatencyMs = 0.0; + LG_LOCK(audio.playback.deviceLock); + const StreamState state = playbackGetState(); + current = diagnostics.epoch == atomic_load_explicit( + &audio.playback.diagnosticsEpoch, memory_order_acquire) && + state != STREAM_STATE_STOP && state != STREAM_STATE_STOP_PENDING; + if (current) + { + const bool timingsCurrent = timings && + timings == audio.playback.timings; + if ((diagnostics.pending & PLAYBACK_DIAGNOSTIC_REGISTER_GRAPH) && + timingsCurrent && !audio.playback.graph && + !audio.playback.graphRegistrationAttempted) + { + audio.playback.graphRegistrationAttempted = true; + audio.playback.graph = app_registerGraph("PLAYBACK RING", + timings, 0.0f, diagnostics.graphMax, audioGraphFormatFn); + atomic_store_explicit(&audio.playback.graphReady, + audio.playback.graph != NULL, memory_order_release); + } + + if ((diagnostics.pending & PLAYBACK_DIAGNOSTIC_INVALIDATE) && + timingsCurrent && audio.playback.graph) + app_invalidateGraph(audio.playback.graph); + + if ((diagnostics.pending & PLAYBACK_DIAGNOSTIC_SYNC_LOG) && + audio.audioDev && audio.audioDev->playback.latency) + backendLatencyMs = + audio.audioDev->playback.latency() / 1000.0; + } + LG_UNLOCK(audio.playback.deviceLock); + + if (!current) + return; + + if (diagnostics.pending & PLAYBACK_DIAGNOSTIC_START_LOG) + DEBUG_INFO( + "Audio start: %.2f/%.2f ms target/queued", + diagnostics.start.targetLatencyMs, + diagnostics.start.queuedLatencyMs); + + if (diagnostics.pending & PLAYBACK_DIAGNOSTIC_SYNC_LOG) + { + const bool providerControl = + diagnostics.sync.rateControl == PLAYBACK_RATE_PROVIDER; + const char * controlName = providerControl ? "feedback" : + diagnostics.sync.rateControl == PLAYBACK_RATE_BACKEND ? + "backend" : "software"; + const char * sourceRateName = providerControl ? "arrival" : "source"; + + DEBUG_INFO( + "Audio sync: ring %.2f/%.2f ms, backend %.2f ms, " + "%s %+.1f ppm, rates %s/device %+.1f/%+.1f ppm, " + "jitter %.2f ms, xruns %u/%u", + diagnostics.sync.softwareLatencyMs, + diagnostics.sync.targetLatencyMs, + backendLatencyMs, controlName, diagnostics.sync.controlPpm, + sourceRateName, diagnostics.sync.sourcePpm, + diagnostics.sync.devicePpm, diagnostics.sync.jitterMs, + diagnostics.sync.underruns, diagnostics.sync.overruns); + } } static void playbackSourceStop(void) @@ -1289,14 +1895,18 @@ static void playbackSourceStop(void) if (!audio.audioDev) return; - switch (playbackGetState()) + StreamState state = playbackGetState(); + switch (state) { case STREAM_STATE_RUN: case STREAM_STATE_RESUMING: { // Keep the audio device open for a while to reduce startup latency if // playback starts again - playbackSetState(STREAM_STATE_KEEP_ALIVE); + if (!atomic_compare_exchange_strong_explicit( + &audio.playback.state, &state, STREAM_STATE_KEEP_ALIVE, + memory_order_acq_rel, memory_order_acquire)) + break; // Reset the software resampler so it is safe for the next playback if (audio.playback.sourceData.src) @@ -1326,34 +1936,27 @@ static void playbackSourceStop(void) } } -static void playbackVolume(int channels, const uint16_t volume[]) +static int playbackStoreVolume(int channels, const uint16_t volume[]) { if (!audio.audioDev || !audio.audioDev->playback.volume || !g_params.audioSyncVolume) - return; + return 0; // store the values so we can restore the state if the stream is restarted channels = min(ARRAY_LENGTH(audio.playback.volume), channels); memcpy(audio.playback.volume, volume, sizeof(uint16_t) * channels); audio.playback.volumeChannels = channels; - - if (!STREAM_ACTIVE(playbackGetState())) - return; - - audio.audioDev->playback.volume(channels, volume); + return channels; } -static void playbackMute(bool mute) +static bool playbackStoreMute(bool mute) { if (!audio.audioDev || !audio.audioDev->playback.mute) - return; + return false; // store the value so we can restore it if the stream is restarted audio.playback.mute = mute; - if (!STREAM_ACTIVE(playbackGetState())) - return; - - audio.audioDev->playback.mute(mute); + return true; } static double computeDevicePosition(int64_t curTime) @@ -1384,8 +1987,10 @@ static bool playbackEnsureConversionBuffers( if (audio.playback.convertToFloat && frames > sourceData->framesInSize) { + const int capacity = max(frames, + max(sourceData->framesInSize * 2, 64)); float * framesIn = realloc(sourceData->framesIn, - (size_t)frames * audio.playback.channels * sizeof(float)); + (size_t)capacity * audio.playback.channels * sizeof(float)); if (!framesIn) { DEBUG_ERROR("Failed to grow playback input buffer"); @@ -1393,7 +1998,7 @@ static bool playbackEnsureConversionBuffers( } sourceData->framesIn = framesIn; - sourceData->framesInSize = frames; + sourceData->framesInSize = capacity; } if (audio.playback.rateControl == PLAYBACK_RATE_SOFTWARE) @@ -1402,8 +2007,10 @@ static bool playbackEnsureConversionBuffers( (int)ceil(frames * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) + 64; if (framesOut > sourceData->framesOutSize) { + const int capacity = max(framesOut, + max(sourceData->framesOutSize * 2, 64)); float * output = realloc(sourceData->framesOut, - (size_t)framesOut * audio.playback.channels * sizeof(float)); + (size_t)capacity * audio.playback.channels * sizeof(float)); if (!output) { DEBUG_ERROR("Failed to grow playback output buffer"); @@ -1411,7 +2018,7 @@ static bool playbackEnsureConversionBuffers( } sourceData->framesOut = output; - sourceData->framesOutSize = framesOut; + sourceData->framesOutSize = capacity; } } @@ -1463,30 +2070,37 @@ static int playbackSlewBuffer( return advanced; } -static void playbackData(const void * data, size_t frameCount, - const LG_AudioClock * sourceClock) +static PlaybackDataResult playbackData(const void * data, size_t frameCount, + const LG_AudioClock * sourceClock, int64_t arrivalTime) { StreamState state = playbackGetState(); if (state == STREAM_STATE_STOP_PENDING) - { - playbackStop(); - return; - } + return PLAYBACK_DATA_DROP; - if (state == STREAM_STATE_STOP || !audio.audioDev || frameCount == 0) - return; + if (frameCount == 0) + return PLAYBACK_DATA_DROP; + if (state == STREAM_STATE_STOP || !audio.audioDev) + return PLAYBACK_DATA_RETRY_NOW; + + PlaybackSourceData * sourceData = &audio.playback.sourceData; + if (!data || frameCount > (size_t)sourceData->maxSourcePacketFrames) + { + DEBUG_ERROR("Invalid playback packet length: %zu frames", frameCount); + return PLAYBACK_DATA_DROP; + } + const int frames = frameCount; if (audio.playback.rateControl == PLAYBACK_RATE_BACKEND && atomic_exchange_explicit( &audio.playback.backendResamplerFailed, false, memory_order_acq_rel)) { - DEBUG_ERROR("Audio backend resampler failed"); - playbackStop(); - return; + DEBUG_WARN("Audio backend resampler failed; using libsamplerate"); + audio.playback.forceSoftwareResampler = true; + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY_NOW; } - PlaybackSourceData * sourceData = &audio.playback.sourceData; /* Backend resampling changes how many source frames PipeWire requests per * device period. Use the command-normalized output clock for rate matching, * while deviceClock remains in the ring's source-frame domain for latency. */ @@ -1494,23 +2108,15 @@ static void playbackData(const void * data, size_t frameCount, audio.playback.rateControl == PLAYBACK_RATE_BACKEND ? &sourceData->outputClock : &sourceData->deviceClock; const int64_t now = nanotime(); + if (audio.playback.startFailures && + audio.playback.backendStartTime && + now - audio.playback.backendStartTime >= AUDIO_RETRY_RESET_NS) + { + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + } const double nominalFrameSec = 1.0 / audio.playback.sampleRate; - if (!data || frameCount > INT_MAX || - frameCount > (size_t)audio.playback.sampleRate * 2) - { - DEBUG_ERROR("Invalid playback packet length: %zu frames", frameCount); - playbackStop(); - return; - } - const int frames = frameCount; - - if (!playbackEnsureConversionBuffers(sourceData, frames)) - { - playbackStop(); - return; - } - const void * inputFrames = data; if (audio.playback.convertToFloat) { @@ -1519,54 +2125,57 @@ static void playbackData(const void * data, size_t frameCount, audio.playback.format.sampleFormat)) { DEBUG_ERROR("Failed to convert playback samples"); - playbackStop(); - return; + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY; } inputFrames = sourceData->framesIn; } const bool providerRateControl = audio.playback.rateControl == PLAYBACK_RATE_PROVIDER; - /* Do not carry an underrun as buffer debt. Provider feedback is bounded, - * so repaying a guest stall by rate correction would discard resumed audio - * for many seconds. Rebase at the first packet instead. */ - const bool providerUnderrun = - providerRateControl && playbackGetState() == STREAM_STATE_RUN && + /* An unbounded ring represents an underrun by advancing the reader beyond + * the writer. Do not carry that logical debt into resumed playback: bounded + * rate correction would otherwise discard fresh audio for many seconds. */ + const bool bufferUnderrun = + STREAM_ACTIVE(playbackGetState()) && ringbuffer_getCount(audio.playback.buffer) < 0; bool discontinuity = - providerUnderrun || (sourceClock && sourceClock->discontinuity); + bufferUnderrun || (sourceClock && sourceClock->discontinuity); const int64_t packetTime = playbackMapMediaTime(sourceData, sourceClock, frames, - audio.playback.sampleRate, now, &discontinuity); + audio.playback.sampleRate, arrivalTime, &discontinuity); if (sourceData->bufferOverrunPending) { discontinuity = true; sourceData->bufferOverrunPending = false; } - if (sourceData->lastPacketTime != INT64_MIN && + if (!discontinuity && + sourceData->lastPacketTime != INT64_MIN && sourceData->lastArrivalTime != INT64_MIN) { const double mediaDelta = (packetTime - sourceData->lastPacketTime) * 1.0e-9; const double arrivalDelta = - (now - sourceData->lastArrivalTime) * 1.0e-9; + (arrivalTime - sourceData->lastArrivalTime) * 1.0e-9; const double jitter = fabs(arrivalDelta - mediaDelta); /* Keep a slowly decaying peak rather than feeding arrival jitter into the * virtual clock. This lets the buffer absorb real delivery jitter while * the rate controller follows only the source media clock. */ + const double decaySec = max(arrivalDelta, 0.0); sourceData->arrivalJitterSec = min(PLAYBACK_MAX_JITTER_SEC, - max(jitter, sourceData->arrivalJitterSec * 0.999)); + max(jitter, sourceData->arrivalJitterSec * + exp(-decaySec / PLAYBACK_JITTER_DECAY_SEC))); } sourceData->lastPacketTime = packetTime; - sourceData->lastArrivalTime = now; + sourceData->lastArrivalTime = arrivalTime; const bool sourceRateWasValid = sourceData->sourceRateValid; const int64_t sourceRateTimeMs = providerRateControl ? - (now - sourceData->mediaLocalOrigin) / INT64_C(1000000) : + (arrivalTime - sourceData->mediaLocalOrigin) / INT64_C(1000000) : sourceData->mediaTimeMs; playbackSourceRateAdd( sourceData, sourceRateTimeMs, nominalFrameSec); @@ -1578,6 +2187,9 @@ static void playbackData(const void * data, size_t frameCount, frameSec <= nominalFrameSec * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) { + if (!sourceData->sourceRateValid || + sourceData->rateFilterTimeMs == INT64_MIN) + sourceData->rateFilterTimeMs = sourceRateTimeMs; sourceData->sourceRateFrameSec = frameSec; sourceData->sourceRateValid = true; } @@ -1679,6 +2291,17 @@ static void playbackData(const void * data, size_t frameCount, (providerRateControl ? 0.0 : sourceReserveFrames); } + if (discontinuity && sourceData->src) + { + const int error = src_reset(sourceData->src); + if (error) + { + DEBUG_ERROR("Failed to reset resampler: %s", src_strerror(error)); + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY; + } + } + const int maxPeriodFrames = max(audio.playback.deviceMaxPeriodFrames, sourceData->devPeriodFrames); /* The device period, delivery jitter, packet phase, and resampler delay @@ -1710,7 +2333,7 @@ static void playbackData(const void * data, size_t frameCount, double devPosition = DBL_MIN; state = playbackGetState(); - if (providerRateControl && + if ((providerRateControl || bufferUnderrun) && (discontinuity || state == STREAM_STATE_KEEP_ALIVE || state == STREAM_STATE_RESUMING)) @@ -1726,12 +2349,16 @@ static void playbackData(const void * data, size_t frameCount, sourceData->offsetError = 0.0; sourceData->offsetErrorIntegral = 0.0; sourceData->ratioIntegral = 0.0; - if (providerUnderrun) + if (providerRateControl && bufferUnderrun) { sourceData->lastRatio = 1.0; sourceData->nextFeedbackTime = 0; } - playbackSetState(STREAM_STATE_RUN); + if (state == STREAM_STATE_KEEP_ALIVE || + state == STREAM_STATE_RESUMING) + atomic_compare_exchange_strong_explicit( + &audio.playback.state, &state, STREAM_STATE_RUN, + memory_order_acq_rel, memory_order_acquire); } else if ((discontinuity || state == STREAM_STATE_KEEP_ALIVE || @@ -1750,7 +2377,11 @@ static void playbackData(const void * data, size_t frameCount, sourceData->offsetError = 0.0; sourceData->offsetErrorIntegral = 0.0; sourceData->ratioIntegral = 0.0; - playbackSetState(STREAM_STATE_RUN); + if (state == STREAM_STATE_KEEP_ALIVE || + state == STREAM_STATE_RESUMING) + atomic_compare_exchange_strong_explicit( + &audio.playback.state, &state, STREAM_STATE_RUN, + memory_order_acq_rel, memory_order_acquire); } double actualLatencyFrames = 0.0; @@ -1955,15 +2586,15 @@ static void playbackData(const void * data, size_t frameCount, if (error) { DEBUG_ERROR("Resampling failed: %s", src_strerror(error)); - playbackStop(); - return; + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY; } if (srcData.input_frames_used == 0 && srcData.output_frames_gen == 0) { DEBUG_ERROR("Resampler made no progress"); - playbackStop(); - return; + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY; } const int outputFrames = playbackAppendFrames( @@ -1995,16 +2626,25 @@ static void playbackData(const void * data, size_t frameCount, if (ringbuffer_getCount(audio.playback.buffer) >= audio.playback.targetStartFrames) { - if (audio.playback.timings && !audio.playback.graph) + if (g_params.audioDebug) { - const float graphMax = + PlaybackDiagnostics * diagnostics = playbackDiagnosticsLocked(); + if (audio.playback.timings) + { + diagnostics->graphMax = + targetLatencyFrames * 1000.0 / + audio.playback.sampleRate * 2; + diagnostics->pending |= + PLAYBACK_DIAGNOSTIC_REGISTER_GRAPH; + } + + diagnostics->start.targetLatencyMs = targetLatencyFrames * 1000.0 / - audio.playback.sampleRate * 2; - audio.playback.graph = app_registerGraph("PLAYBACK RING", - audio.playback.timings, 0.0f, graphMax, - audioGraphFormatFn); - if (!audio.playback.graph) - ringbuffer_free(&audio.playback.timings); + audio.playback.sampleRate; + diagnostics->start.queuedLatencyMs = + audio.playback.targetStartFrames * 1000.0 / + audio.playback.sampleRate; + diagnostics->pending |= PLAYBACK_DIAGNOSTIC_START_LOG; } audio.playback.startupLowWaterFrames = @@ -2012,40 +2652,36 @@ static void playbackData(const void * data, size_t frameCount, audio.playback.startupPacketPeriod = max(llrint(packetSec * 1.0e9), INT64_C(1)); audio.playback.startupPacketDeadline = - now + audio.playback.startupPacketPeriod; - - if (g_params.audioDebug) - DEBUG_INFO( - "Audio start: %.2f/%.2f ms target/queued", - targetLatencyFrames * 1000.0 / - audio.playback.sampleRate, - audio.playback.targetStartFrames * 1000.0 / - audio.playback.sampleRate); + arrivalTime + audio.playback.startupPacketPeriod; playbackSetState(STREAM_STATE_SETUP_DEVICE); - audio.audioDev->playback.start(); + playbackArmBackend(); + playbackWorkerWake(); } } if (!g_params.audioDebug) - return; + return PLAYBACK_DATA_PROCESSED; const double softwareLatencyMs = actualLatencyFrames * 1000.0 / audio.playback.sampleRate; + bool wakeDiagnostics = false; - if (audio.playback.graph && now >= sourceData->nextGraphTime) + if (audio.playback.timings && + atomic_load_explicit( + &audio.playback.graphReady, memory_order_acquire) && + now >= sourceData->nextGraphTime) { sourceData->nextGraphTime = now + PLAYBACK_GRAPH_INTERVAL_NS; const float latency = softwareLatencyMs; ringbuffer_push(audio.playback.timings, &latency); - app_invalidateGraph(audio.playback.graph); + playbackDiagnosticsLocked()->pending |= + PLAYBACK_DIAGNOSTIC_INVALIDATE; + wakeDiagnostics = true; } if (now >= sourceData->nextLogTime) { - const double backendLatencyMs = - audio.audioDev->playback.latency ? - audio.audioDev->playback.latency() / 1000.0 : 0.0; const double sourcePpm = sourceData->sourceRateValid ? (nominalFrameSec / sourceData->sourceRateFrameSec - 1.0) * 1.0e6 : 0.0; @@ -2058,27 +2694,39 @@ static void playbackData(const void * data, size_t frameCount, (playbackProviderRate(sourceData) / audio.playback.sampleRate - 1.0) * 1.0e6 : (ratio - 1.0) * 1.0e6; - const char * controlName = providerControl ? "feedback" : - audio.playback.rateControl == PLAYBACK_RATE_BACKEND ? - "backend" : "software"; - const char * sourceRateName = providerControl ? "arrival" : "source"; const unsigned int underruns = atomic_exchange_explicit( &audio.playback.underruns, 0, memory_order_relaxed); - - DEBUG_INFO( - "Audio sync: ring %.2f/%.2f ms, backend %.2f ms, " - "%s %+.1f ppm, rates %s/device %+.1f/%+.1f ppm, " - "jitter %.2f ms, xruns %u/%u", - softwareLatencyMs, + PlaybackDiagnostics * diagnostics = playbackDiagnosticsLocked(); + const unsigned int pendingUnderruns = + diagnostics->pending & PLAYBACK_DIAGNOSTIC_SYNC_LOG ? + diagnostics->sync.underruns : 0; + const unsigned int pendingOverruns = + diagnostics->pending & PLAYBACK_DIAGNOSTIC_SYNC_LOG ? + diagnostics->sync.overruns : 0; + diagnostics->sync = (PlaybackSyncDiagnostics) + { + .softwareLatencyMs = softwareLatencyMs, + .targetLatencyMs = targetLatencyFrames * 1000.0 / audio.playback.sampleRate, - backendLatencyMs, controlName, controlPpm, - sourceRateName, sourcePpm, devicePpm, - sourceData->arrivalJitterSec * 1000.0, - underruns, sourceData->bufferOverruns); + .sourcePpm = sourcePpm, + .devicePpm = devicePpm, + .controlPpm = controlPpm, + .jitterMs = sourceData->arrivalJitterSec * 1000.0, + .underruns = pendingUnderruns + underruns, + .overruns = pendingOverruns + sourceData->bufferOverruns, + .rateControl = audio.playback.rateControl, + }; + diagnostics->pending |= PLAYBACK_DIAGNOSTIC_SYNC_LOG; sourceData->bufferOverruns = 0; sourceData->nextLogTime = now + INT64_C(5000000000); + wakeDiagnostics = true; } + + if (wakeDiagnostics) + playbackWorkerWake(); + + return PLAYBACK_DATA_PROCESSED; } static bool playbackGetFeedback( @@ -2115,28 +2763,148 @@ static bool playbackGetFeedback( return true; } -bool lgAudio_supportsRecord(void) +static bool audioBindingEqual( + const AudioBinding * a, const AudioBinding * b) { - return audio.audioDev && audio.audioDev->record.start; + return + a->ops == b->ops && + a->opaque == b->opaque && + a->epoch == b->epoch && + a->generation == b->generation; } -static void recordPushFrames(uint8_t * data, int frames) +static void recordWorkerWake(void) +{ + LG_LOCK_SHARED(audio.record.lock); + if (audio.record.wake && audio.record.thread) + lgSignalEvent(audio.record.wake); + LG_UNLOCK_SHARED(audio.record.lock); +} + +static void recordDisarmLocked(void) +{ + atomic_store_explicit( + &audio.record.deliverySerial, 0, memory_order_release); +} + +static void recordBackendFailed(uint32_t attemptSerial) +{ + unsigned int expected = attemptSerial; + if (!attemptSerial || !atomic_compare_exchange_strong_explicit( + &audio.record.deliverySerial, &expected, 0, + memory_order_acq_rel, memory_order_acquire)) + return; + + atomic_store_explicit(&audio.record.failedAttemptSerial, + attemptSerial, memory_order_release); + lgSignalEvent(audio.record.wake); +} + +static void recordAdvanceRequestLocked(void) +{ + if (!++audio.record.requestSerial) + ++audio.record.requestSerial; +} + +static void recordResetStartRetryLocked(void) +{ + audio.record.startFailures = 0; + audio.record.nextStartRetry = 0; +} + +/* record.lock must be held exclusively. */ +static bool recordDelayStartLocked(void) +{ + const unsigned int failures = audio.record.startFailures; + audio.record.nextStartRetry = nanotime() + + audioStartRetryDelay(failures); + if (audio.record.startFailures < 7U) + ++audio.record.startFailures; + + return failures == 0U || failures == 6U; +} + +/* record.lock must be held exclusively. */ +static bool recordDelayRequestLocked( + uint64_t requestSerial, bool resetRetry) +{ + if (audio.record.shuttingDown || !audio.record.requested || + !audio.record.enabled || !audio.record.requestedGeneration || + requestSerial != audio.record.requestSerial) + return false; + + if (resetRetry) + recordResetStartRetryLocked(); + return recordDelayStartLocked(); +} + +static unsigned int recordWorkerWaitTimeout(void) +{ + unsigned int timeout = TIMEOUT_INFINITE; + + LG_LOCK_SHARED(audio.record.lock); + if (!audio.record.shuttingDown && audio.record.requested && + audio.record.enabled && audio.record.requestedGeneration && + audio.record.nextStartRetry) + { + const int64_t remaining = + audio.record.nextStartRetry - nanotime(); + timeout = remaining <= 0 ? 0 : + (unsigned int)((remaining + INT64_C(999999)) / + INT64_C(1000000)); + } + LG_UNLOCK_SHARED(audio.record.lock); + + return timeout; +} + +bool lgAudio_supportsRecord(void) +{ + return atomic_load_explicit(&audio.ready, memory_order_acquire) && + audio.audioDev && audio.audioDev->record.start && + audio.audioDev->record.stop && + audio.record.wake && audio.record.thread && + atomic_load_explicit( + &audio.record.workerAlive, memory_order_acquire); +} + +static bool recordPushFrames(uint8_t * data, int frames, + const LG_AudioClock * sourceClock) { if (frames <= 0) - return; + return true; - const uint32_t generation = atomic_load_explicit( - &audio.record.streamGeneration, memory_order_acquire); - if (!generation) - return; + const uint32_t serial = atomic_load_explicit( + &audio.record.deliverySerial, memory_order_acquire); + if (!serial) + return true; - LG_LOCK_SHARED(audio.activeLock); - if (generation == atomic_load_explicit( - &audio.record.streamGeneration, memory_order_acquire) && - audio.active.ops && audio.active.ops->recordData) - audio.active.ops->recordData(audio.active.opaque, - generation, data, frames, NULL); - LG_UNLOCK_SHARED(audio.activeLock); + AudioBinding binding; + uint32_t generation; + + LG_LOCK_SHARED(audio.record.lock); + if (serial != atomic_load_explicit( + &audio.record.deliverySerial, memory_order_acquire)) + { + LG_UNLOCK_SHARED(audio.record.lock); + return true; + } + binding = audio.record.deliveryBinding; + generation = audio.record.deliveryGeneration; + LG_UNLOCK_SHARED(audio.record.lock); + + AudioBinding active; + if (!eventBegin(&binding, &active)) + return true; + + bool accepted = true; + if (serial == atomic_load_explicit( + &audio.record.deliverySerial, memory_order_acquire) && + active.ops->recordData) + accepted = active.ops->recordData(active.opaque, + generation, data, frames, sourceClock); + eventEnd(); + return accepted; } static MsgBoxHandle recordCancelConfirmLocked(void) @@ -2148,36 +2916,16 @@ static MsgBoxHandle recordCancelConfirmLocked(void) return handle; } -static void realRecordStartLocked(const LG_AudioFormat * format) -{ - audio.record.started = true; - audio.record.format = *format; - - audio.audioDev->record.start(format, recordPushFrames); - - // if a volume level was stored, set it before we return - if (audio.record.volumeChannels && audio.audioDev->record.volume) - audio.audioDev->record.volume( - audio.record.volumeChannels, - audio.record.volume); - - // set the inital mute state - if (audio.audioDev->record.mute) - audio.audioDev->record.mute(audio.record.mute); - - if (g_params.micShowIndicator) - app_showRecord(true); -} - static void recordConfirm(bool yes, void * opaque) { const uint64_t generation = (uint64_t)(uintptr_t)opaque; + bool wake = false; - LG_LOCK(audio.record.lock); + LG_LOCK_EXCLUSIVE(audio.record.lock); if (!audio.record.confirmPending || generation != audio.record.confirmGeneration) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } @@ -2185,419 +2933,818 @@ static void recordConfirm(bool yes, void * opaque) audio.record.confirmHandle = NULL; if (yes && audio.record.requested && - !audio.record.shuttingDown && audio.audioDev) + audio.record.requestedGeneration && + !audio.record.shuttingDown && audio.audioDev && + audio.record.thread && atomic_load_explicit( + &audio.record.workerAlive, memory_order_acquire)) { DEBUG_INFO("Microphone access granted"); - realRecordStartLocked(&audio.record.confirmFormat); + audio.record.enabled = true; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + wake = true; } else if (yes) DEBUG_INFO("Ignoring stale microphone access confirmation"); else DEBUG_INFO("Microphone access denied"); - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (wake) + recordWorkerWake(); } -static void recordStart(const LG_AudioFormat * format) +static void recordCreateConfirm( + MsgBoxHandle oldConfirm, uint64_t generation) { - LG_LOCK(audio.record.lock); - if (!audio.audioDev || !audio.audioDev->record.start || - audio.record.shuttingDown || - !audioFormatValid(format)) + app_msgBoxClose(oldConfirm); + + MsgBoxHandle handle = app_confirmMsgBox( + "Microphone", recordConfirm, (void *)(uintptr_t)generation, + "An application just opened the microphone!\n" + "Do you want it to access your microphone?"); + MsgBoxHandle stale = NULL; + + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (handle && audio.record.confirmPending && + generation == audio.record.confirmGeneration) + audio.record.confirmHandle = handle; + else { - if (format && !audioFormatValid(format)) - DEBUG_ERROR("Invalid recording format"); - LG_UNLOCK(audio.record.lock); + stale = handle; + if (!handle && audio.record.confirmPending && + generation == audio.record.confirmGeneration) + { + audio.record.confirmPending = false; + ++audio.record.confirmGeneration; + } + } + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + + app_msgBoxClose(stale); +} + +static void recordStart(const AudioBinding * binding, + uint32_t generation, const LG_AudioFormat * format) +{ + MsgBoxHandle oldConfirm = NULL; + uint64_t confirmGeneration = 0; + bool wake = false; + + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (!audio.audioDev || !audio.audioDev->record.start || + !audio.audioDev->record.stop || + !audio.record.thread || audio.record.shuttingDown || + !atomic_load_explicit( + &audio.record.workerAlive, memory_order_acquire) || + !binding || !generation) + { + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } - const bool restart = audio.record.started; - if (audio.record.started) + if (!audioFormatValid(format)) { - if (audioFormatEqual(format, &audio.record.lastFormat)) - { - LG_UNLOCK(audio.record.lock); - return; - } - - realRecordStopLocked(); + oldConfirm = recordCancelConfirmLocked(); + audio.record.requested = false; + audio.record.enabled = false; + audio.record.requestedBinding = (AudioBinding) { 0 }; + audio.record.requestedGeneration = 0; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + app_msgBoxClose(oldConfirm); + recordWorkerWake(); + DEBUG_ERROR("Invalid recording format"); + return; } - MsgBoxHandle oldConfirm = recordCancelConfirmLocked(); - audio.record.requested = true; - audio.record.lastFormat = *format; + const bool sameRequest = audio.record.requested && + generation == audio.record.requestedGeneration && + audioBindingEqual(binding, &audio.record.requestedBinding); + if (sameRequest) + { + const bool formatChanged = !audioFormatEqual( + format, &audio.record.requestedFormat); + if (formatChanged) + { + audio.record.requestedFormat = *format; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + wake = true; + } - if (restart) - realRecordStartLocked(format); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (wake) + recordWorkerWake(); + return; + } + + const bool keepEnabled = audio.record.requested && + audio.record.enabled && + audioBindingEqual(binding, &audio.record.requestedBinding); + wake = true; + oldConfirm = recordCancelConfirmLocked(); + audio.record.requested = true; + audio.record.enabled = false; + audio.record.requestedBinding = *binding; + audio.record.requestedGeneration = generation; + audio.record.requestedFormat = *format; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + + if (keepEnabled) + { + audio.record.enabled = true; + wake = true; + } else if (g_state.micDefaultState == MIC_DEFAULT_DENY) DEBUG_INFO("Microphone access denied by default"); else if (g_state.micDefaultState == MIC_DEFAULT_ALLOW) { DEBUG_INFO("Microphone access granted by default"); - realRecordStartLocked(format); + audio.record.enabled = true; + wake = true; } else { - audio.record.confirmFormat = *format; - audio.record.confirmPending = true; - const uint64_t generation = ++audio.record.confirmGeneration; - LG_UNLOCK(audio.record.lock); + audio.record.confirmPending = true; + confirmGeneration = ++audio.record.confirmGeneration; + } + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (wake) + recordWorkerWake(); + if (confirmGeneration) + recordCreateConfirm(oldConfirm, confirmGeneration); + else app_msgBoxClose(oldConfirm); +} - LG_LOCK(audio.record.lock); - const bool current = - audio.record.confirmPending && - generation == audio.record.confirmGeneration && - audio.record.requested && - !audio.record.shuttingDown && - audio.audioDev; - if (current) - { - audio.record.confirmHandle = app_confirmMsgBox( - "Microphone", recordConfirm, (void *)(uintptr_t)generation, - "An application just opened the microphone!\n" - "Do you want it to access your microphone?"); - if (!audio.record.confirmHandle) - { - audio.record.confirmPending = false; - ++audio.record.confirmGeneration; - } - } - LG_UNLOCK(audio.record.lock); +static void recordStop( + const AudioBinding * binding, uint32_t generation) +{ + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (binding && + (!generation || !audio.record.requested || + generation != audio.record.requestedGeneration || + !audioBindingEqual(binding, &audio.record.requestedBinding))) + { + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } - LG_UNLOCK(audio.record.lock); - app_msgBoxClose(oldConfirm); -} - -static void recordReconfigure(const LG_AudioFormat * format) -{ - LG_LOCK(audio.record.lock); - if (!audio.audioDev || !audio.audioDev->record.start || - audio.record.shuttingDown || !audioFormatValid(format)) - { - LG_UNLOCK(audio.record.lock); - return; - } - - audio.record.lastFormat = *format; - if (audio.record.started && - !audioFormatEqual(format, &audio.record.format)) - { - realRecordStopLocked(); - realRecordStartLocked(format); - } - else if (audio.record.confirmPending) - audio.record.confirmFormat = *format; - - LG_UNLOCK(audio.record.lock); -} - -static void realRecordStopLocked(void) -{ - audio.audioDev->record.stop(); - audio.record.started = false; - - if (g_params.micShowIndicator) - app_showRecord(false); -} - -static void recordStop(void) -{ - LG_LOCK(audio.record.lock); - audio.record.requested = false; + const bool wasRequested = audio.record.requested; + audio.record.requested = false; + audio.record.enabled = false; + audio.record.requestedBinding = (AudioBinding) { 0 }; + audio.record.requestedGeneration = 0; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); MsgBoxHandle confirm = recordCancelConfirmLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); - if (audio.audioDev && audio.record.started) - { + if (wasRequested) DEBUG_INFO("Microphone recording stopped"); - realRecordStopLocked(); - } - LG_UNLOCK(audio.record.lock); - app_msgBoxClose(confirm); + recordWorkerWake(); } void lgAudio_recordToggleKeybind(int sc, void * opaque) { - LG_LOCK(audio.record.lock); - if (!audio.audioDev || audio.record.shuttingDown) + if (!atomic_load_explicit(&audio.ready, memory_order_acquire)) + return; + + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (!audio.audioDev || !audio.record.thread || + !atomic_load_explicit( + &audio.record.workerAlive, memory_order_acquire) || + audio.record.shuttingDown) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } if (!audio.record.requested) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); app_alert(LG_ALERT_WARNING, "No application is requesting microphone access."); return; } MsgBoxHandle confirm = recordCancelConfirmLocked(); - bool started; - if (audio.record.started) - { - DEBUG_INFO("Microphone recording stopped by user"); - realRecordStopLocked(); - started = false; - } - else - { - DEBUG_INFO("Microphone recording started by user"); - realRecordStartLocked(&audio.record.lastFormat); - started = true; - } - LG_UNLOCK(audio.record.lock); + audio.record.enabled = !audio.record.enabled; + const bool enabled = audio.record.enabled; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + DEBUG_INFO("Microphone recording %s by user", + enabled ? "started" : "stopped"); app_msgBoxClose(confirm); + recordWorkerWake(); app_alert(LG_ALERT_INFO, - started ? "Microphone enabled" : "Microphone disabled"); + enabled ? "Microphone enabled" : "Microphone disabled"); } -static void recordVolume(int channels, const uint16_t volume[]) +static void recordVolume(const AudioBinding * binding, + uint32_t generation, int channels, const uint16_t volume[]) { - LG_LOCK(audio.record.lock); + LG_LOCK_EXCLUSIVE(audio.record.lock); if (!audio.audioDev || !audio.audioDev->record.volume || - !g_params.audioSyncVolume || audio.record.shuttingDown) + !g_params.audioSyncVolume || audio.record.shuttingDown || + !audio.record.requested || + generation != audio.record.requestedGeneration || + !audioBindingEqual(binding, &audio.record.requestedBinding)) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } - // store the values so we can restore the state if the stream is restarted channels = min(ARRAY_LENGTH(audio.record.volume), channels); memcpy(audio.record.volume, volume, sizeof(uint16_t) * channels); audio.record.volumeChannels = channels; + if (!++audio.record.controlSerial) + ++audio.record.controlSerial; + LG_UNLOCK_EXCLUSIVE(audio.record.lock); - if (!audio.record.started) - { - LG_UNLOCK(audio.record.lock); - return; - } - - audio.audioDev->record.volume(channels, volume); - LG_UNLOCK(audio.record.lock); + recordWorkerWake(); } -static void recordMute(bool mute) +static void recordMute( + const AudioBinding * binding, uint32_t generation, bool mute) { - LG_LOCK(audio.record.lock); + LG_LOCK_EXCLUSIVE(audio.record.lock); if (!audio.audioDev || !audio.audioDev->record.mute || - audio.record.shuttingDown) + audio.record.shuttingDown || !audio.record.requested || + generation != audio.record.requestedGeneration || + !audioBindingEqual(binding, &audio.record.requestedBinding)) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } - // store the value so we can restore it if the stream is restarted audio.record.mute = mute; - if (!audio.record.started) + if (!++audio.record.controlSerial) + ++audio.record.controlSerial; + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + + recordWorkerWake(); +} + +static bool recordRequestCurrentLocked(uint64_t requestSerial, + const AudioBinding * binding, uint32_t generation, + const LG_AudioFormat * format) +{ + return !audio.record.shuttingDown && audio.record.requested && + audio.record.enabled && + requestSerial == audio.record.requestSerial && + generation == audio.record.requestedGeneration && + audioBindingEqual(binding, &audio.record.requestedBinding) && + audioFormatEqual(format, &audio.record.requestedFormat); +} + +static uint32_t recordArmLocked( + const AudioBinding * binding, uint32_t generation) +{ + if (!++audio.record.nextAttemptSerial) + ++audio.record.nextAttemptSerial; + + atomic_store_explicit( + &audio.record.failedAttemptSerial, 0, memory_order_release); + audio.record.deliveryBinding = *binding; + audio.record.deliveryGeneration = generation; + atomic_store_explicit(&audio.record.deliverySerial, + audio.record.nextAttemptSerial, memory_order_release); + return audio.record.nextAttemptSerial; +} + +static int recordThread(void * opaque) +{ + bool backendStarted = false; + bool controlsApplied = false; + uint64_t backendRequestSerial = 0; + uint32_t backendAttemptSerial = 0; + int64_t backendStartTime = 0; + uint64_t appliedControlSerial = 0; + + for (;;) { - LG_UNLOCK(audio.record.lock); - return; + const unsigned int timeout = recordWorkerWaitTimeout(); + if (!lgWaitEvent(audio.record.wake, timeout) && + timeout == TIMEOUT_INFINITE) + { + DEBUG_ERROR("Failed to wait for audio recording work"); + break; + } + + for (;;) + { + enum + { + RECORD_ACTION_WAIT, + RECORD_ACTION_START, + RECORD_ACTION_STOP, + RECORD_ACTION_CONTROLS, + RECORD_ACTION_EXIT, + } + action = RECORD_ACTION_WAIT; + + uint64_t requestSerial = 0; + uint32_t attemptSerial = 0; + uint64_t controlSerial = 0; + AudioBinding binding = { 0 }; + uint32_t generation = 0; + LG_AudioFormat format = { 0 }; + int volumeChannels = 0; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool mute = false; + bool reportStartFailure = false; + bool runtimeFailure = false; + bool resetStartRetry = false; + + LG_LOCK_EXCLUSIVE(audio.record.lock); + const bool desired = !audio.record.shuttingDown && + audio.record.requested && audio.record.enabled && + audio.record.requestedGeneration && audio.audioDev && + audio.audioDev->record.start && audio.audioDev->record.stop; + + const bool backendFailed = backendStarted && + backendAttemptSerial == atomic_load_explicit( + &audio.record.failedAttemptSerial, memory_order_acquire); + if (backendStarted && + (backendFailed || !desired || + backendRequestSerial != audio.record.requestSerial)) + { + recordDisarmLocked(); + if (backendFailed && desired && + backendRequestSerial == audio.record.requestSerial) + { + requestSerial = backendRequestSerial; + runtimeFailure = true; + resetStartRetry = + nanotime() - backendStartTime >= AUDIO_RETRY_RESET_NS; + } + action = RECORD_ACTION_STOP; + } + else if (!backendStarted && desired && + nanotime() >= audio.record.nextStartRetry) + { + requestSerial = audio.record.requestSerial; + binding = audio.record.requestedBinding; + generation = audio.record.requestedGeneration; + format = audio.record.requestedFormat; + attemptSerial = recordArmLocked(&binding, generation); + action = RECORD_ACTION_START; + } + else if (backendStarted && + (!controlsApplied || + appliedControlSerial != audio.record.controlSerial)) + { + requestSerial = backendRequestSerial; + controlSerial = audio.record.controlSerial; + volumeChannels = audio.record.volumeChannels; + if (volumeChannels) + memcpy(volume, audio.record.volume, + sizeof(*volume) * volumeChannels); + mute = audio.record.mute; + action = RECORD_ACTION_CONTROLS; + } + else if (audio.record.shuttingDown) + action = RECORD_ACTION_EXIT; + + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + + if (action == RECORD_ACTION_WAIT) + break; + + if (action == RECORD_ACTION_EXIT) + goto exit; + + if (action == RECORD_ACTION_STOP) + { + if (audio.audioDev && audio.audioDev->record.stop) + audio.audioDev->record.stop(); + + if (runtimeFailure) + { + LG_LOCK_EXCLUSIVE(audio.record.lock); + reportStartFailure = recordDelayRequestLocked( + requestSerial, resetStartRetry); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + } + + backendStarted = false; + controlsApplied = false; + backendRequestSerial = 0; + backendAttemptSerial = 0; + backendStartTime = 0; + + if (g_params.micShowIndicator) + app_showRecord(false); + if (runtimeFailure && reportStartFailure) + DEBUG_ERROR("Audio recording device failed; retrying"); + continue; + } + + if (action == RECORD_ACTION_START) + { + const bool started = audio.audioDev->record.start( + &format, recordPushFrames, + recordBackendFailed, attemptSerial); + + LG_LOCK_EXCLUSIVE(audio.record.lock); + const bool current = recordRequestCurrentLocked( + requestSerial, &binding, generation, &format); + const bool failed = attemptSerial == atomic_load_explicit( + &audio.record.failedAttemptSerial, memory_order_acquire); + const bool delayAfterStop = started && current && failed; + if (started && current && !failed) + { + backendStarted = true; + backendRequestSerial = requestSerial; + backendAttemptSerial = attemptSerial; + backendStartTime = nanotime(); + controlsApplied = false; + audio.record.nextStartRetry = 0; + } + else + { + if (attemptSerial == atomic_load_explicit( + &audio.record.deliverySerial, memory_order_relaxed)) + recordDisarmLocked(); + if (current && !started) + reportStartFailure = recordDelayStartLocked(); + } + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + + if (started && (!current || failed)) + audio.audioDev->record.stop(); + + if (delayAfterStop) + { + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (recordRequestCurrentLocked( + requestSerial, &binding, generation, &format)) + reportStartFailure = recordDelayStartLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + } + else if (started && current && g_params.micShowIndicator) + app_showRecord(true); + if (current && (!started || failed) && reportStartFailure) + DEBUG_ERROR("Failed to start audio recording device; retrying"); + continue; + } + + if (volumeChannels && audio.audioDev->record.volume) + audio.audioDev->record.volume(volumeChannels, volume); + if (audio.audioDev->record.mute) + audio.audioDev->record.mute(mute); + + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (backendStarted && + requestSerial == backendRequestSerial && + requestSerial == audio.record.requestSerial) + { + controlsApplied = true; + appliedControlSerial = controlSerial; + } + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + } } - audio.audioDev->record.mute(mute); - LG_UNLOCK(audio.record.lock); +exit: + if (backendStarted && audio.audioDev && audio.audioDev->record.stop) + audio.audioDev->record.stop(); + + LG_LOCK_EXCLUSIVE(audio.record.lock); + recordDisarmLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (backendStarted && g_params.micShowIndicator) + app_showRecord(false); + atomic_store_explicit( + &audio.record.workerAlive, false, memory_order_release); + return 0; } static bool bindingActiveNL(const AudioBinding * binding) { - return binding->ops && - audio.active.ops == binding->ops && - audio.active.opaque == binding->opaque && - audio.active.generation == binding->generation; + return binding->ops && audioBindingEqual(&audio.active, binding); +} + +static bool eventBegin( + const AudioBinding * binding, AudioBinding * active) +{ + LG_LOCK_SHARED(audio.activeLock); + const bool current = bindingActiveNL(binding); + if (current) + { + *active = audio.active; + atomic_fetch_add_explicit( + &audio.activeCallbacks, 1, memory_order_acq_rel); + } + LG_UNLOCK_SHARED(audio.activeLock); + return current; +} + +static void eventEnd(void) +{ + const unsigned int previous = atomic_fetch_sub_explicit( + &audio.activeCallbacks, 1, memory_order_release); + if ((previous & ACTIVE_CALLBACK_WAITING) && + (previous & ACTIVE_CALLBACK_COUNT_MASK) == 1) + lgSignalEvent(audio.activeIdle); +} + +static void waitForActiveCallbacks(void) +{ + atomic_fetch_or_explicit( + &audio.activeCallbacks, ACTIVE_CALLBACK_WAITING, + memory_order_acq_rel); + while ((atomic_load_explicit( + &audio.activeCallbacks, memory_order_acquire) & + ACTIVE_CALLBACK_COUNT_MASK) != 0) + lgWaitEvent(audio.activeIdle, TIMEOUT_INFINITE); + atomic_fetch_and_explicit( + &audio.activeCallbacks, ~ACTIVE_CALLBACK_WAITING, + memory_order_release); } static void queueFeedback(const LG_AudioOps * ops, void * opaque, - uint32_t bindingGeneration, uint32_t generation, + uint32_t bindingEpoch, uint32_t bindingGeneration, + uint32_t generation, const LG_AudioClock * clock, double targetRate) { - if (!audio.feedback.event || !audio.feedback.thread || !clock) + if (!audio.feedback.wakeInitialized || !audio.feedback.thread || !clock) return; LG_LOCK(audio.feedback.lock); audio.feedback.ops = ops; audio.feedback.opaque = opaque; + audio.feedback.bindingEpoch = bindingEpoch; audio.feedback.bindingGeneration = bindingGeneration; audio.feedback.generation = generation; audio.feedback.clock = *clock; audio.feedback.targetRate = targetRate; audio.feedback.pending = true; LG_UNLOCK(audio.feedback.lock); - lgSignalEvent(audio.feedback.event); + feedbackWorkerWake(); } static void eventPlaybackStart(void * opaque, uint32_t generation, const LG_AudioFormat * format, const LG_AudioClock * sourceClock) { AudioBinding * binding = opaque; + AudioBinding active; + (void)sourceClock; - LG_LOCK_SHARED(audio.activeLock); - if (bindingActiveNL(binding)) + if (!eventBegin(binding, &active)) + return; + + const bool providerRateControl = + active.ops->clockFeedback && + audio.feedback.wakeInitialized && audio.feedback.thread; + + LG_LOCK(audio.playback.sourceLock); + ++audio.playback.requestSerial; + audio.playback.requestedGeneration = generation; + audio.playback.requestedFormatValid = + generation && audioFormatValid(format); + audio.playback.requestedProviderRateControl = providerRateControl; + audio.playback.forceSoftwareResampler = false; + audio.playback.startPending = false; + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + + if (audio.playback.requestedFormatValid) { - LG_LOCK(audio.playback.sourceLock); - atomic_store_explicit(&audio.playback.streamGeneration, - generation, memory_order_release); - playbackStart(format, sourceClock, - binding->ops->clockFeedback && - audio.feedback.event && audio.feedback.thread); - LG_UNLOCK(audio.playback.sourceLock); + audio.playback.requestedFormat = *format; + if (audio.playback.startInProgress) + { + audio.playback.startPending = true; + playbackWorkerWake(); + } + else + playbackScheduleStart(); } - LG_UNLOCK_SHARED(audio.activeLock); + else + { + if (generation) + DEBUG_ERROR("Invalid playback format"); + if (playbackGetState() != STREAM_STATE_STOP) + { + LG_LOCK(audio.playback.deviceLock); + playbackStop(); + LG_UNLOCK(audio.playback.deviceLock); + } + audio.playback.requestedGeneration = 0; + } + LG_UNLOCK(audio.playback.sourceLock); + eventEnd(); } static void eventPlaybackStop(void * opaque, uint32_t generation) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - if (bindingActiveNL(binding)) + if (!eventBegin(binding, &active)) + return; + + LG_LOCK(audio.playback.sourceLock); + if (audio.playback.requestedGeneration == generation) { - LG_LOCK(audio.playback.sourceLock); + ++audio.playback.requestSerial; if (atomic_load_explicit(&audio.playback.streamGeneration, memory_order_acquire) == generation) { + LG_LOCK(audio.playback.deviceLock); playbackSourceStop(); - atomic_store_explicit( - &audio.playback.streamGeneration, 0, memory_order_release); + LG_UNLOCK(audio.playback.deviceLock); } - LG_UNLOCK(audio.playback.sourceLock); + audio.playback.requestedGeneration = 0; + audio.playback.requestedFormatValid = false; + audio.playback.requestedProviderRateControl = false; + audio.playback.forceSoftwareResampler = false; + audio.playback.startPending = false; + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); } - LG_UNLOCK_SHARED(audio.activeLock); + LG_UNLOCK(audio.playback.sourceLock); + eventEnd(); } static void eventPlaybackVolume(void * opaque, uint32_t generation, uint8_t channels, const uint16_t volume[]) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - if (bindingActiveNL(binding)) + if (!eventBegin(binding, &active)) + return; + + bool wake = false; + LG_LOCK(audio.playback.sourceLock); + if (volume && + audio.playback.requestedGeneration == generation) { - LG_LOCK(audio.playback.sourceLock); - if (volume && - atomic_load_explicit(&audio.playback.streamGeneration, - memory_order_acquire) == generation) - playbackVolume(channels, volume); - LG_UNLOCK(audio.playback.sourceLock); + const int storedChannels = playbackStoreVolume(channels, volume); + if (storedChannels) + { + if (!++audio.playback.controlSerial) + ++audio.playback.controlSerial; + audio.playback.controlsPending = true; + wake = atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation; + } } - LG_UNLOCK_SHARED(audio.activeLock); + LG_UNLOCK(audio.playback.sourceLock); + if (wake) + playbackWorkerWake(); + eventEnd(); } static void eventPlaybackMute(void * opaque, uint32_t generation, bool mute) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - if (bindingActiveNL(binding)) + if (!eventBegin(binding, &active)) + return; + + bool wake = false; + LG_LOCK(audio.playback.sourceLock); + if (audio.playback.requestedGeneration == generation) { - LG_LOCK(audio.playback.sourceLock); - if (atomic_load_explicit(&audio.playback.streamGeneration, - memory_order_acquire) == generation) - playbackMute(mute); - LG_UNLOCK(audio.playback.sourceLock); + const bool stored = playbackStoreMute(mute); + if (stored) + { + if (!++audio.playback.controlSerial) + ++audio.playback.controlSerial; + audio.playback.controlsPending = true; + wake = atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation; + } } - LG_UNLOCK_SHARED(audio.activeLock); + LG_UNLOCK(audio.playback.sourceLock); + if (wake) + playbackWorkerWake(); + eventEnd(); } static void eventPlaybackData(void * opaque, uint32_t generation, const void * data, size_t frames, const LG_AudioClock * sourceClock) { + const int64_t arrivalTime = nanotime(); AudioBinding * binding = opaque; - const LG_AudioOps * ops; - void * providerOpaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding); - ops = active ? binding->ops : NULL; - providerOpaque = active ? binding->opaque : NULL; - if (active) + if (!eventBegin(binding, &active)) + return; + + LG_LOCK(audio.playback.sourceLock); + if (audio.playback.requestedGeneration == generation) { - LG_LOCK(audio.playback.sourceLock); - if (atomic_load_explicit(&audio.playback.streamGeneration, - memory_order_acquire) == generation) + const uint32_t liveGeneration = atomic_load_explicit( + &audio.playback.streamGeneration, memory_order_acquire); + PlaybackDataResult result = PLAYBACK_DATA_DROP; + if (liveGeneration == generation) { - playbackData(data, frames, sourceClock); + result = playbackData(data, frames, sourceClock, arrivalTime); - LG_AudioClock feedback; - double targetRate; - if (ops->clockFeedback && - playbackGetFeedback(&feedback, &targetRate)) - queueFeedback(ops, providerOpaque, binding->generation, - generation, &feedback, targetRate); + if (result == PLAYBACK_DATA_PROCESSED && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) != generation) + result = PLAYBACK_DATA_DROP; + + if (result == PLAYBACK_DATA_RETRY || + result == PLAYBACK_DATA_RETRY_NOW) + { + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + if (result == PLAYBACK_DATA_RETRY_NOW) + audio.playback.nextStartRetry = 0; + else + playbackDelayStart(); + } } - LG_UNLOCK(audio.playback.sourceLock); + + if (result != PLAYBACK_DATA_PROCESSED) + playbackScheduleStart(); + + LG_AudioClock feedback; + double targetRate; + if (result == PLAYBACK_DATA_PROCESSED && + active.ops->clockFeedback && + playbackGetFeedback(&feedback, &targetRate)) + queueFeedback(active.ops, active.opaque, active.epoch, + active.generation, generation, &feedback, targetRate); } - LG_UNLOCK_SHARED(audio.activeLock); + LG_UNLOCK(audio.playback.sourceLock); + eventEnd(); } static void eventRecordStart(void * opaque, uint32_t generation, const LG_AudioFormat * format) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding) && - binding->ops->recordData; - if (active) - { - const uint32_t previous = atomic_exchange_explicit( - &audio.record.streamGeneration, generation, memory_order_acq_rel); - if (previous == generation) - recordReconfigure(format); - else - recordStart(format); - } - LG_UNLOCK_SHARED(audio.activeLock); + if (!eventBegin(binding, &active)) + return; + + if (active.ops->recordData) + recordStart(&active, generation, format); + eventEnd(); } static void eventRecordStop(void * opaque, uint32_t generation) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding); - if (active && - atomic_load_explicit(&audio.record.streamGeneration, - memory_order_acquire) == generation) - { - atomic_store_explicit( - &audio.record.streamGeneration, 0, memory_order_release); - recordStop(); - } - LG_UNLOCK_SHARED(audio.activeLock); + if (!eventBegin(binding, &active)) + return; + + recordStop(&active, generation); + eventEnd(); } static void eventRecordVolume(void * opaque, uint32_t generation, uint8_t channels, const uint16_t volume[]) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding); - if (active && volume && - atomic_load_explicit(&audio.record.streamGeneration, - memory_order_acquire) == generation) - recordVolume(channels, volume); - LG_UNLOCK_SHARED(audio.activeLock); + if (!eventBegin(binding, &active)) + return; + + if (volume) + recordVolume(&active, generation, channels, volume); + eventEnd(); } static void eventRecordMute(void * opaque, uint32_t generation, bool mute) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding); - if (active && - atomic_load_explicit(&audio.record.streamGeneration, - memory_order_acquire) == generation) - recordMute(mute); - LG_UNLOCK_SHARED(audio.activeLock); + if (!eventBegin(binding, &active)) + return; + + recordMute(&active, generation, mute); + eventEnd(); } static const LG_AudioEventOps eventOps = @@ -2620,11 +3767,20 @@ static bool validOps(const LG_AudioOps * ops) static AudioBinding makeBinding(const LG_AudioOps * ops, void * opaque) { + uint32_t epoch = 0; + if (ops) + { + epoch = ++audio.nextBindingEpoch; + if (!epoch) + epoch = ++audio.nextBindingEpoch; + } + return (AudioBinding) { .ops = ops, .opaque = opaque, .available = ops && !ops->setStatusListener, + .epoch = epoch, .generation = 0, }; } @@ -2642,14 +3798,24 @@ static void stopStreams(void) { LG_LOCK(audio.playback.sourceLock); if (audio.audioDev) + { + LG_LOCK(audio.playback.deviceLock); playbackStop(); + LG_UNLOCK(audio.playback.deviceLock); + } + ++audio.playback.requestSerial; + audio.playback.requestedGeneration = 0; + audio.playback.requestedFormatValid = false; + audio.playback.requestedProviderRateControl = false; + audio.playback.forceSoftwareResampler = false; + audio.playback.startPending = false; + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; atomic_store_explicit( &audio.playback.streamGeneration, 0, memory_order_release); LG_UNLOCK(audio.playback.sourceLock); - atomic_store_explicit( - &audio.record.streamGeneration, 0, memory_order_release); - recordStop(); + recordStop(NULL, 0); } /* providerLock must be held. dropActive suppresses calls into an endpoint @@ -2663,6 +3829,7 @@ static void updateActive(bool dropActive) AudioBinding next = slot ? *slot : (AudioBinding) { 0 }; const AudioBinding old = audio.active; if (old.ops == next.ops && old.opaque == next.opaque && + old.epoch == next.epoch && old.generation == next.generation) { audio.active = next; @@ -2672,6 +3839,8 @@ static void updateActive(bool dropActive) audio.active = (AudioBinding) { 0 }; LG_UNLOCK_EXCLUSIVE(audio.activeLock); + waitForActiveCallbacks(); + if (old.ops && !dropActive) old.ops->detach(old.opaque); dropActive = false; @@ -2695,17 +3864,20 @@ static void updateActive(bool dropActive) return; } - next.ops->detach(next.opaque); - stopStreams(); - LG_LOCK_EXCLUSIVE(audio.activeLock); if (audio.active.ops == next.ops && - audio.active.opaque == next.opaque) + audio.active.opaque == next.opaque && + audio.active.epoch == next.epoch && + audio.active.generation == next.generation) audio.active = (AudioBinding) { 0 }; if (slot->ops == next.ops && slot->opaque == next.opaque) slot->available = false; LG_UNLOCK_EXCLUSIVE(audio.activeLock); + waitForActiveCallbacks(); + next.ops->detach(next.opaque); + stopStreams(); + DEBUG_WARN("Failed to attach Audio provider: %s", next.ops->name); } } @@ -2752,16 +3924,10 @@ static void transportStatusChanged(void * opaque, LG_UNLOCK(audio.providerLock); } -static void setBinding(AudioBinding * target, const LG_AudioOps * ops, +/* bindingLock must be held. */ +static void setBindingLocked(AudioBinding * target, const LG_AudioOps * ops, void * opaque, LG_AudioStatusFn statusFn) { - if (ops && !validOps(ops)) - { - DEBUG_ERROR("Invalid audio operations"); - ops = NULL; - opaque = NULL; - } - LG_LOCK(audio.providerLock); const AudioBinding old = *target; LG_UNLOCK(audio.providerLock); @@ -2781,16 +3947,100 @@ static void setBinding(AudioBinding * target, const LG_AudioOps * ops, next.ops->setStatusListener(next.opaque, statusFn, next.opaque); } +static void setBinding(AudioBinding * target, const LG_AudioOps * ops, + void * opaque, LG_AudioStatusFn statusFn) +{ + if (ops && !validOps(ops)) + { + DEBUG_ERROR("Invalid audio operations"); + ops = NULL; + opaque = NULL; + } + + LG_LOCK(audio.bindingLock); + if (atomic_load_explicit(&audio.ready, memory_order_acquire)) + setBindingLocked(target, ops, opaque, statusFn); + LG_UNLOCK(audio.bindingLock); +} + +static int playbackThread(void * opaque) +{ + for (;;) + { + int result; + do + result = sem_wait(&audio.playback.worker.wake); + while (result < 0 && errno == EINTR); + if (result < 0) + break; + + atomic_store_explicit( + &audio.playback.worker.wakePending, false, memory_order_release); + if (atomic_load_explicit( + &audio.playback.worker.stop, memory_order_acquire)) + break; + + playbackProcessStopPending(); + playbackProcessStart(); + playbackProcessControls(); + playbackProcessDeviceStart(); + playbackProcessDiagnostics(); + } + + return 0; +} + +static void playbackFeedbackFailed(uint32_t generation) +{ + bool restart = false; + + LG_LOCK(audio.playback.sourceLock); + if (generation && + audio.playback.requestedGeneration == generation && + audio.playback.requestedProviderRateControl && + audio.playback.rateControl == PLAYBACK_RATE_PROVIDER && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + { + ++audio.playback.requestSerial; + audio.playback.requestedProviderRateControl = false; + audio.playback.forceSoftwareResampler = false; + audio.playback.startPending = true; + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + playbackQueueSourceStop(); + restart = true; + } + LG_UNLOCK(audio.playback.sourceLock); + + if (restart) + DEBUG_WARN("Audio feedback stopped; using local rate control"); +} + static int feedbackThread(void * opaque) { - while (lgWaitEvent(audio.feedback.event, TIMEOUT_INFINITE)) + AudioBinding rejectedBinding = { 0 }; + uint32_t rejectedGeneration = 0; + int64_t rejectedSince = 0; + + for (;;) { + int result; + do + result = sem_wait(&audio.feedback.wake); + while (result < 0 && errno == EINTR); + if (result < 0) + break; + + atomic_store_explicit( + &audio.feedback.wakePending, false, memory_order_release); if (atomic_load_explicit( &audio.feedback.stop, memory_order_acquire)) break; const LG_AudioOps * ops; void * providerOpaque; + uint32_t bindingEpoch; uint32_t bindingGeneration; uint32_t generation; LG_AudioClock clock; @@ -2800,6 +4050,7 @@ static int feedbackThread(void * opaque) const bool pending = audio.feedback.pending; ops = audio.feedback.ops; providerOpaque = audio.feedback.opaque; + bindingEpoch = audio.feedback.bindingEpoch; bindingGeneration = audio.feedback.bindingGeneration; generation = audio.feedback.generation; clock = audio.feedback.clock; @@ -2807,18 +4058,59 @@ static int feedbackThread(void * opaque) audio.feedback.pending = false; LG_UNLOCK(audio.feedback.lock); - if (!pending || !ops || !ops->clockFeedback) + if (!pending) continue; - LG_LOCK_SHARED(audio.activeLock); - if (audio.active.ops == ops && - audio.active.opaque == providerOpaque && - audio.active.generation == bindingGeneration && + const AudioBinding binding = + { + .ops = ops, + .opaque = providerOpaque, + .epoch = bindingEpoch, + .generation = bindingGeneration, + }; + AudioBinding active; + if (!eventBegin(&binding, &active)) + { + rejectedSince = 0; + continue; + } + + bool attempted = false; + bool accepted = false; + if (active.ops->clockFeedback && atomic_load_explicit(&audio.playback.streamGeneration, memory_order_acquire) == generation) - ops->clockFeedback( - providerOpaque, generation, &clock, targetRate); - LG_UNLOCK_SHARED(audio.activeLock); + { + attempted = true; + accepted = active.ops->clockFeedback( + active.opaque, generation, &clock, targetRate); + } + + if (!attempted || accepted) + { + rejectedSince = 0; + eventEnd(); + continue; + } + + const int64_t now = nanotime(); + if (!rejectedSince || + !audioBindingEqual(&rejectedBinding, &binding) || + rejectedGeneration != generation) + { + rejectedBinding = binding; + rejectedGeneration = generation; + rejectedSince = now; + eventEnd(); + continue; + } + + if (now - rejectedSince >= PLAYBACK_FEEDBACK_FAILURE_NS) + { + rejectedSince = 0; + playbackFeedbackFailed(generation); + } + eventEnd(); } return 0; @@ -2826,75 +4118,215 @@ static int feedbackThread(void * opaque) void lgAudio_init(void) { + atomic_init(&audio.ready, false); + LG_LOCK_INIT(audio.bindingLock); LG_LOCK_INIT(audio.providerLock); LG_RWLOCK_INIT(audio.activeLock); + atomic_init(&audio.activeCallbacks, 0); LG_LOCK_INIT(audio.playback.sourceLock); - LG_LOCK_INIT(audio.record.lock); + LG_LOCK_INIT(audio.playback.deviceLock); + LG_RWLOCK_INIT(audio.record.lock); LG_LOCK_INIT(audio.feedback.lock); audio.record.shuttingDown = false; atomic_init(&audio.playback.streamGeneration, 0); - atomic_init(&audio.record.streamGeneration, 0); + atomic_init(&audio.playback.activeAttemptSerial, 0); + atomic_init(&audio.playback.failedAttemptSerial, 0); + atomic_init(&audio.playback.diagnosticsEpoch, 1); + atomic_init(&audio.playback.graphReady, false); + atomic_init(&audio.playback.worker.stop, false); + atomic_init(&audio.playback.worker.wakePending, false); + atomic_init(&audio.record.deliverySerial, 0); + atomic_init(&audio.record.failedAttemptSerial, 0); + atomic_init(&audio.record.workerAlive, false); atomic_init(&audio.feedback.stop, false); + atomic_init(&audio.feedback.wakePending, false); atomic_store_explicit( &audio.playback.callbackState, PLAYBACK_CALLBACK_DISABLED, memory_order_release); - audio.feedback.event = lgCreateEvent(true, 0); - if (audio.feedback.event && - !lgCreateThread("audioFeedback", feedbackThread, + if (sem_init(&audio.playback.callbackIdle, 0, 0) != 0) + { + DEBUG_ERROR("Failed to create the audio callback semaphore"); + return; + } + audio.playback.callbackIdleInitialized = true; + + audio.activeIdle = lgCreateEvent(true, 0); + if (!audio.activeIdle) + { + DEBUG_ERROR("Failed to create the audio provider event"); + goto err_callback; + } + + if (sem_init(&audio.playback.worker.wake, 0, 0) != 0) + { + DEBUG_ERROR("Failed to create the audio playback semaphore"); + goto err_active; + } + audio.playback.worker.wakeInitialized = true; + if (!lgCreateThread("audioPlayback", playbackThread, + NULL, &audio.playback.worker.thread)) + { + DEBUG_ERROR("Failed to create the audio playback thread"); + goto err_playbackWake; + } + + if (sem_init(&audio.feedback.wake, 0, 0) != 0) + { + DEBUG_ERROR("Failed to create the audio feedback semaphore"); + goto err_playbackThread; + } + audio.feedback.wakeInitialized = true; + if (!lgCreateThread("audioFeedback", feedbackThread, NULL, &audio.feedback.thread)) { - lgFreeEvent(audio.feedback.event); - audio.feedback.event = NULL; + DEBUG_ERROR("Failed to create the audio feedback thread"); + goto err_feedbackWake; + } + + audio.record.wake = lgCreateEvent(true, 0); + if (!audio.record.wake) + DEBUG_ERROR("Failed to create the audio recording event"); + else + { + atomic_store_explicit( + &audio.record.workerAlive, true, memory_order_release); + if (!lgCreateThread("audioRecord", recordThread, + NULL, &audio.record.thread)) + { + atomic_store_explicit( + &audio.record.workerAlive, false, memory_order_release); + DEBUG_ERROR("Failed to create the audio recording thread"); + lgFreeEvent(audio.record.wake); + audio.record.wake = NULL; + } } for (int i = 0; i < LG_AUDIODEV_COUNT; ++i) if (LG_AudioDevs[i]->init()) { audio.audioDev = LG_AudioDevs[i]; + atomic_store_explicit(&audio.ready, true, memory_order_release); DEBUG_INFO("Using AudioDev: %s", audio.audioDev->name); return; } DEBUG_WARN("Failed to initialize an audio backend"); + return; + +err_feedbackWake: + sem_destroy(&audio.feedback.wake); + audio.feedback.wakeInitialized = false; + +err_playbackThread: + atomic_store_explicit( + &audio.playback.worker.stop, true, memory_order_release); + playbackWorkerWake(); + lgJoinThread(audio.playback.worker.thread, NULL); + audio.playback.worker.thread = NULL; + +err_playbackWake: + sem_destroy(&audio.playback.worker.wake); + audio.playback.worker.wakeInitialized = false; + +err_active: + lgFreeEvent(audio.activeIdle); + audio.activeIdle = NULL; + +err_callback: + sem_destroy(&audio.playback.callbackIdle); + audio.playback.callbackIdleInitialized = false; } void lgAudio_free(void) { - lgAudio_setTransport(NULL, NULL); - lgAudio_setFallback(NULL, NULL); + LG_LOCK(audio.bindingLock); + const bool ready = atomic_exchange_explicit( + &audio.ready, false, memory_order_acq_rel); + if (ready) + { + setBindingLocked( + &audio.fallback, NULL, NULL, fallbackStatusChanged); + setBindingLocked( + &audio.transport, NULL, NULL, transportStatusChanged); + } + LG_UNLOCK(audio.bindingLock); stopStreams(); + if (audio.playback.worker.thread) + { + atomic_store_explicit( + &audio.playback.worker.stop, true, memory_order_release); + playbackWorkerWake(); + lgJoinThread(audio.playback.worker.thread, NULL); + audio.playback.worker.thread = NULL; + } + if (audio.playback.worker.wakeInitialized) + { + sem_destroy(&audio.playback.worker.wake); + audio.playback.worker.wakeInitialized = false; + } + if (audio.feedback.thread) { atomic_store_explicit( &audio.feedback.stop, true, memory_order_release); - lgSignalEvent(audio.feedback.event); + feedbackWorkerWake(); lgJoinThread(audio.feedback.thread, NULL); audio.feedback.thread = NULL; } - if (audio.feedback.event) + if (audio.feedback.wakeInitialized) { - lgFreeEvent(audio.feedback.event); - audio.feedback.event = NULL; + sem_destroy(&audio.feedback.wake); + audio.feedback.wakeInitialized = false; } - LG_LOCK(audio.record.lock); + LG_LOCK_EXCLUSIVE(audio.record.lock); audio.record.shuttingDown = true; - audio.record.requested = false; + audio.record.requested = false; + audio.record.enabled = false; + recordAdvanceRequestLocked(); + recordDisarmLocked(); MsgBoxHandle confirm = recordCancelConfirmLocked(); - struct LG_AudioDevOps * audioDev = audio.audioDev; - audio.audioDev = NULL; - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); app_msgBoxClose(confirm); + if (audio.record.thread) + { + recordWorkerWake(); + lgJoinThread(audio.record.thread, NULL); + } + + LG_LOCK_EXCLUSIVE(audio.record.lock); + LGEvent * recordWake = audio.record.wake; + audio.record.thread = NULL; + audio.record.wake = NULL; + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (recordWake) + lgFreeEvent(recordWake); + + if (audio.playback.callbackIdleInitialized) + { + sem_destroy(&audio.playback.callbackIdle); + audio.playback.callbackIdleInitialized = false; + } + if (audio.activeIdle) + { + lgFreeEvent(audio.activeIdle); + audio.activeIdle = NULL; + } + + struct LG_AudioDevOps * audioDev = audio.audioDev; + audio.audioDev = NULL; if (audioDev) audioDev->free(); LG_RWLOCK_FREE(audio.activeLock); LG_LOCK_FREE(audio.playback.sourceLock); + LG_LOCK_FREE(audio.playback.deviceLock); + LG_LOCK_FREE(audio.bindingLock); LG_LOCK_FREE(audio.providerLock); - LG_LOCK_FREE(audio.record.lock); + LG_RWLOCK_FREE(audio.record.lock); LG_LOCK_FREE(audio.feedback.lock); } @@ -2910,6 +4342,13 @@ void lgAudio_setTransport(const LG_AudioOps * ops, void * opaque) void lgAudio_dropTransport(void) { + LG_LOCK(audio.bindingLock); + if (!atomic_load_explicit(&audio.ready, memory_order_acquire)) + { + LG_UNLOCK(audio.bindingLock); + return; + } + LG_LOCK(audio.providerLock); LG_LOCK_EXCLUSIVE(audio.activeLock); const AudioBinding old = audio.transport; @@ -2921,6 +4360,7 @@ void lgAudio_dropTransport(void) if (old.ops && old.ops->setStatusListener) old.ops->setStatusListener(old.opaque, NULL, NULL); + LG_UNLOCK(audio.bindingLock); } #endif diff --git a/client/src/audio_spice.c b/client/src/audio_spice.c index 310bb88e..1a359e99 100644 --- a/client/src/audio_spice.c +++ b/client/src/audio_spice.c @@ -21,6 +21,7 @@ #include "audio_spice.h" #include "common/debug.h" +#include "common/event.h" #include "common/locking.h" #include @@ -50,6 +51,22 @@ typedef struct SpiceAudioEventTarget } SpiceAudioEventTarget; +typedef struct SpiceAudioCallbackWaiter +{ + struct SpiceAudioCallbackWaiter * next; + LGEvent * event; + unsigned int depth; +} +SpiceAudioCallbackWaiter; + +typedef struct SpiceAudioCallbackWaitQueue +{ + LG_Lock lock; + atomic_uint count; + SpiceAudioCallbackWaiter * waiters; +} +SpiceAudioCallbackWaitQueue; + static struct { LG_RWLock lock; @@ -59,11 +76,13 @@ static struct LG_AudioStatusFn statusCallback; void * statusOpaque; atomic_uint statusInFlight; + SpiceAudioCallbackWaitQueue statusWait; const LG_AudioEventOps * events; void * eventOpaque; uint32_t eventGeneration; atomic_uint inFlight; + SpiceAudioCallbackWaitQueue eventWait; SpiceAudioStream playback; SpiceAudioStream record; @@ -83,7 +102,17 @@ l_spice = .writer = ATOMIC_FLAG_INIT, }, .statusInFlight = ATOMIC_VAR_INIT(0), + .statusWait = + { + .lock = ATOMIC_FLAG_INIT, + .count = ATOMIC_VAR_INIT(0), + }, .inFlight = ATOMIC_VAR_INIT(0), + .eventWait = + { + .lock = ATOMIC_FLAG_INIT, + .count = ATOMIC_VAR_INIT(0), + }, }; static _Thread_local unsigned int l_eventDepth; @@ -96,6 +125,96 @@ static uint32_t nextGeneration(uint32_t generation) return generation; } +static LGEvent * createWaitEvent(void) +{ + LGEvent * event = lgCreateEvent(true, 0); + if (!event) + DEBUG_FATAL("Failed to create SPICE audio wait event"); + return event; +} + +static void waitEvent(LGEvent * event) +{ + if (!lgWaitEvent(event, TIMEOUT_INFINITE)) + DEBUG_FATAL("Failed to wait for SPICE audio event"); +} + +static void signalWaitEvent(LGEvent * event) +{ + if (!lgSignalEvent(event)) + DEBUG_FATAL("Failed to signal SPICE audio event"); +} + +static void signalCallbackWaiters( + SpiceAudioCallbackWaitQueue * queue, unsigned int remaining) +{ + if (!atomic_load_explicit(&queue->count, memory_order_acquire)) + return; + + LG_LOCK(queue->lock); + SpiceAudioCallbackWaiter ** link = &queue->waiters; + while (*link) + { + SpiceAudioCallbackWaiter * waiter = *link; + if (remaining > waiter->depth) + { + link = &waiter->next; + continue; + } + + *link = waiter->next; + atomic_fetch_sub_explicit( + &queue->count, 1, memory_order_release); + signalWaitEvent(waiter->event); + } + LG_UNLOCK(queue->lock); +} + +static void endCallback(atomic_uint * inFlight, + SpiceAudioCallbackWaitQueue * waitQueue) +{ + const unsigned int previous = atomic_fetch_sub_explicit( + inFlight, 1, memory_order_release); + DEBUG_ASSERT(previous > 0); + signalCallbackWaiters(waitQueue, previous - 1); +} + +static void waitCallbacks(atomic_uint * inFlight, + SpiceAudioCallbackWaitQueue * waitQueue, unsigned int depth) +{ + if (atomic_load_explicit(inFlight, memory_order_acquire) <= depth) + return; + + SpiceAudioCallbackWaiter waiter = + { + .event = createWaitEvent(), + .depth = depth, + }; + + bool queued = false; + LG_LOCK(waitQueue->lock); + if (atomic_load_explicit(inFlight, memory_order_acquire) > depth) + { + waiter.next = waitQueue->waiters; + waitQueue->waiters = &waiter; + atomic_fetch_add_explicit( + &waitQueue->count, 1, memory_order_release); + queued = true; + } + LG_UNLOCK(waitQueue->lock); + + if (queued) + { + waitEvent(waiter.event); + + /* The signaler owns the waiter until it releases the queue lock. */ + LG_LOCK(waitQueue->lock); + LG_UNLOCK(waitQueue->lock); + } + + lgFreeEvent(waiter.event); +} + /* l_spice.lock must be held exclusively while admitting a callback so detach * cannot invalidate the target between the snapshot and in-flight increment. */ static bool beginEventNL(SpiceAudioEventTarget * target) @@ -142,7 +261,7 @@ static bool recordEventCurrent(const SpiceAudioEventTarget * target, static void endEvent(void) { --l_eventDepth; - atomic_fetch_sub_explicit(&l_spice.inFlight, 1, memory_order_release); + endCallback(&l_spice.inFlight, &l_spice.eventWait); } static bool sampleFormat(PSAudioFormat source, @@ -318,13 +437,11 @@ static void spiceSetStatusListener(void * opaque, { callback(callbackOpaque, &status); --l_statusDepth; - atomic_fetch_sub_explicit( - &l_spice.statusInFlight, 1, memory_order_release); + endCallback(&l_spice.statusInFlight, &l_spice.statusWait); } else - while (atomic_load_explicit( - &l_spice.statusInFlight, memory_order_acquire) > l_statusDepth) - ; + waitCallbacks(&l_spice.statusInFlight, + &l_spice.statusWait, l_statusDepth); } static bool spiceAttach(void * opaque, const LG_AudioEventOps * events, @@ -413,9 +530,7 @@ static void spiceDetach(void * opaque) nextGeneration(l_spice.eventGeneration); LG_UNLOCK_EXCLUSIVE(l_spice.lock); - while (atomic_load_explicit( - &l_spice.inFlight, memory_order_acquire) > l_eventDepth) - ; + waitCallbacks(&l_spice.inFlight, &l_spice.eventWait, l_eventDepth); } static bool spiceRecordData(void * opaque, uint32_t generation, @@ -501,8 +616,7 @@ void lgaSpice_setAvailable(bool available) { callback(callbackOpaque, &status); --l_statusDepth; - atomic_fetch_sub_explicit( - &l_spice.statusInFlight, 1, memory_order_release); + endCallback(&l_spice.statusInFlight, &l_spice.statusWait); } } diff --git a/client/src/audio_usb.c b/client/src/audio_usb.c index bb4855ff..52d94096 100644 --- a/client/src/audio_usb.c +++ b/client/src/audio_usb.c @@ -22,11 +22,15 @@ #include "usb_audio.h" +#include "common/debug.h" +#include "common/event.h" #include "common/locking.h" #include "common/time.h" +#include #include #include +#include #define USB_AUDIO_NS_PER_SECOND INT64_C(1000000000) @@ -37,49 +41,94 @@ typedef struct USBAudioCallbackFrame } USBAudioCallbackFrame; +typedef struct USBAudioCallbackWaiter +{ + struct USBAudioCallbackWaiter * next; + LGEvent * event; + unsigned int depth; +} +USBAudioCallbackWaiter; + +typedef struct USBAudioCallbackWaitQueue +{ + LG_Lock lock; + atomic_uint count; + USBAudioCallbackWaiter * waiters; +} +USBAudioCallbackWaitQueue; + +typedef struct USBAudioCallbackGate +{ + atomic_uint inFlight; + USBAudioCallbackWaitQueue wait; +} +USBAudioCallbackGate; + typedef struct USBAudioEventTarget { - const LG_AudioEventOps * events; - void * opaque; - uint32_t attachmentGeneration; - USBAudioCallbackFrame frame; + const LG_AudioEventOps * events; + void * opaque; + uint32_t attachmentGeneration; + USBAudioCallbackFrame frame; + USBAudioCallbackFrame ** frames; + USBAudioCallbackGate * gate; } USBAudioEventTarget; +typedef struct USBAudioOperationWaiter +{ + struct USBAudioOperationWaiter * next; + LGEvent * event; +} +USBAudioOperationWaiter; + +typedef struct USBAudioOperationQueue +{ + LG_Lock lock; + bool active; + USBAudioOperationWaiter * head; + USBAudioOperationWaiter * tail; +} +USBAudioOperationQueue; + struct LGA_USBState { LG_USBAudio * device; LG_USBRedir * redir; - LG_Lock statusLock; - atomic_bool available; - uint32_t statusGeneration; - LG_AudioStatusFn statusCallback; - void * statusOpaque; - atomic_uint statusInFlight; - atomic_uint_fast64_t statusNextTicket; - atomic_uint_fast64_t statusServingTicket; + LG_Lock statusLock; + atomic_bool available; + uint32_t statusGeneration; + LG_AudioStatusFn statusCallback; + void * statusOpaque; + atomic_uint statusInFlight; + USBAudioCallbackWaitQueue statusWait; + USBAudioOperationQueue statusOperation; - LG_Lock stateLock; - bool attached; - bool detaching; - uint32_t attachmentGeneration; - const LG_AudioEventOps * events; - void * eventOpaque; + LG_Lock stateLock; + LGEvent * detachEvent; + bool attached; + bool detaching; + uint32_t attachmentGeneration; + const LG_AudioEventOps * events; + void * eventOpaque; - LG_AudioFormat playbackFormat; - uint32_t playbackGeneration; - LG_AudioFormat recordFormat; - uint32_t recordGeneration; - uint32_t generationSerial; - int64_t playbackClockOrigin; - atomic_uint_fast64_t playbackPosition; - atomic_uint playbackDeliveryGeneration; - atomic_uint recordDeliveryGeneration; - atomic_uint inFlight; + LG_AudioFormat playbackFormat; + uint32_t playbackGeneration; + LG_AudioFormat recordFormat; + uint32_t recordGeneration; + uint32_t generationSerial; + int64_t playbackClockOrigin; + atomic_uint_fast64_t playbackPosition; + atomic_uint playbackDeliveryGeneration; + atomic_uint recordDeliveryGeneration; + alignas(64) USBAudioCallbackGate playbackGate; + alignas(64) USBAudioCallbackGate recordGate; + USBAudioOperationQueue recordOperation; }; -static _Thread_local USBAudioCallbackFrame * l_eventFrames; +static _Thread_local USBAudioCallbackFrame * l_playbackFrames; +static _Thread_local USBAudioCallbackFrame * l_recordFrames; static _Thread_local USBAudioCallbackFrame * l_statusFrames; static const LG_AudioFormat l_formatTemplate = @@ -139,6 +188,51 @@ static unsigned int callbackDepth( return depth; } +static LGEvent * createWaitEvent(void) +{ + LGEvent * event = lgCreateEvent(true, 0); + if (!event) + DEBUG_FATAL("Failed to create USB audio wait event"); + return event; +} + +static void waitEvent(LGEvent * event) +{ + if (!lgWaitEvent(event, TIMEOUT_INFINITE)) + DEBUG_FATAL("Failed to wait for USB audio event"); +} + +static void signalWaitEvent(LGEvent * event) +{ + if (!lgSignalEvent(event)) + DEBUG_FATAL("Failed to signal USB audio event"); +} + +static void signalCallbackWaiters( + USBAudioCallbackWaitQueue * queue, unsigned int remaining) +{ + if (!atomic_load_explicit(&queue->count, memory_order_seq_cst)) + return; + + LG_LOCK(queue->lock); + USBAudioCallbackWaiter ** link = &queue->waiters; + while (*link) + { + USBAudioCallbackWaiter * waiter = *link; + if (remaining > waiter->depth) + { + link = &waiter->next; + continue; + } + + *link = waiter->next; + atomic_fetch_sub_explicit( + &queue->count, 1, memory_order_release); + signalWaitEvent(waiter->event); + } + LG_UNLOCK(queue->lock); +} + static void beginCallback(LGA_USBState * state, USBAudioCallbackFrame * frame, USBAudioCallbackFrame ** frames, atomic_uint * inFlight) @@ -150,10 +244,59 @@ static void beginCallback(LGA_USBState * state, } static void endCallback(USBAudioCallbackFrame * frame, - USBAudioCallbackFrame ** frames, atomic_uint * inFlight) + USBAudioCallbackFrame ** frames, atomic_uint * inFlight, + USBAudioCallbackWaitQueue * waitQueue) { *frames = frame->previous; - atomic_fetch_sub_explicit(inFlight, 1, memory_order_seq_cst); + const unsigned int previous = atomic_fetch_sub_explicit( + inFlight, 1, memory_order_seq_cst); + DEBUG_ASSERT(previous > 0); + signalCallbackWaiters(waitQueue, previous - 1); +} + +static void waitCallbacks(LGA_USBState * state, + USBAudioCallbackFrame * frames, atomic_uint * inFlight, + USBAudioCallbackWaitQueue * waitQueue) +{ + const unsigned int depth = callbackDepth(frames, state); + if (atomic_load_explicit(inFlight, memory_order_seq_cst) <= depth) + return; + + USBAudioCallbackWaiter waiter = + { + .event = createWaitEvent(), + .depth = depth, + }; + + bool queued = false; + LG_LOCK(waitQueue->lock); + + /* Publish the waiter before rechecking inFlight. This prevents the final + * callback from taking the lock-free path while a waiter is being queued. */ + atomic_fetch_add_explicit( + &waitQueue->count, 1, memory_order_seq_cst); + if (atomic_load_explicit(inFlight, memory_order_seq_cst) > depth) + { + waiter.next = waitQueue->waiters; + waitQueue->waiters = &waiter; + queued = true; + } + else + atomic_fetch_sub_explicit( + &waitQueue->count, 1, memory_order_seq_cst); + + LG_UNLOCK(waitQueue->lock); + + if (queued) + { + waitEvent(waiter.event); + + /* The signaler owns the waiter until it releases the queue lock. */ + LG_LOCK(waitQueue->lock); + LG_UNLOCK(waitQueue->lock); + } + + lgFreeEvent(waiter.event); } static LG_AudioClock makePlaybackClock( @@ -178,7 +321,8 @@ static LG_AudioClock makePlaybackClock( /* stateLock must be held while admitting a control event. The data event * path performs the equivalent admission using deliveryGeneration. */ static bool beginEventNL( - LGA_USBState * state, USBAudioEventTarget * target) + LGA_USBState * state, USBAudioEventTarget * target, + USBAudioCallbackFrame ** frames, USBAudioCallbackGate * gate) { if (!state->attached || !state->events) return false; @@ -186,11 +330,27 @@ static bool beginEventNL( target->events = state->events; target->opaque = state->eventOpaque; target->attachmentGeneration = state->attachmentGeneration; + target->frames = frames; + target->gate = gate; beginCallback( - state, &target->frame, &l_eventFrames, &state->inFlight); + state, &target->frame, frames, &gate->inFlight); return true; } +static bool beginPlaybackEventNL( + LGA_USBState * state, USBAudioEventTarget * target) +{ + return beginEventNL( + state, target, &l_playbackFrames, &state->playbackGate); +} + +static bool beginRecordEventNL( + LGA_USBState * state, USBAudioEventTarget * target) +{ + return beginEventNL( + state, target, &l_recordFrames, &state->recordGate); +} + static bool attachmentCurrentNL(const LGA_USBState * state, const USBAudioEventTarget * target) { @@ -214,37 +374,84 @@ static bool recordEventCurrentNL(const LGA_USBState * state, state->recordGeneration == streamGeneration; } -static void endEvent(LGA_USBState * state, USBAudioEventTarget * target) +static void endEvent(USBAudioEventTarget * target) { - endCallback(&target->frame, &l_eventFrames, &state->inFlight); + endCallback(&target->frame, target->frames, + &target->gate->inFlight, &target->gate->wait); } -static void waitEvents(const LGA_USBState * state) +static void waitPlaybackEvents(LGA_USBState * state) { - const unsigned int depth = callbackDepth(l_eventFrames, state); - while (atomic_load_explicit( - &state->inFlight, memory_order_seq_cst) > depth) - ; + waitCallbacks(state, l_playbackFrames, + &state->playbackGate.inFlight, &state->playbackGate.wait); } -static bool beginStatusOperation(LGA_USBState * state) +static void waitRecordEvents(LGA_USBState * state) { - if (callbackDepth(l_statusFrames, state)) + waitCallbacks(state, l_recordFrames, + &state->recordGate.inFlight, &state->recordGate.wait); +} + +static bool beginOperation(LGA_USBState * state, + USBAudioOperationQueue * operation, + const USBAudioCallbackFrame * callbackFrames) +{ + /* Reentrant control must run inline: the current owner may be waiting for + * this callback to finish before it can release the operation. */ + if (callbackDepth(callbackFrames, state)) return false; - const uint_fast64_t ticket = atomic_fetch_add_explicit( - &state->statusNextTicket, 1, memory_order_relaxed); - while (atomic_load_explicit( - &state->statusServingTicket, memory_order_acquire) != ticket) - ; + USBAudioOperationWaiter waiter = + { + .event = createWaitEvent(), + }; + + bool queued = false; + LG_LOCK(operation->lock); + if (operation->active) + { + if (operation->tail) + operation->tail->next = &waiter; + else + operation->head = &waiter; + operation->tail = &waiter; + queued = true; + } + else + operation->active = true; + LG_UNLOCK(operation->lock); + + if (queued) + { + waitEvent(waiter.event); + + /* The previous owner owns the waiter until it releases the queue lock. */ + LG_LOCK(operation->lock); + LG_UNLOCK(operation->lock); + } + + lgFreeEvent(waiter.event); return true; } -static void endStatusOperation(LGA_USBState * state, bool owner) +static void endOperation( + USBAudioOperationQueue * operation, bool owner) { - if (owner) - atomic_fetch_add_explicit( - &state->statusServingTicket, 1, memory_order_release); + if (!owner) + return; + + LG_LOCK(operation->lock); + USBAudioOperationWaiter * waiter = operation->head; + if (waiter) + { + operation->head = waiter->next; + if (!operation->head) + operation->tail = NULL; + signalWaitEvent(waiter->event); + } + else + operation->active = false; + LG_UNLOCK(operation->lock); } static void beginStatusCallback( @@ -257,15 +464,32 @@ static void beginStatusCallback( static void endStatusCallback( LGA_USBState * state, USBAudioCallbackFrame * frame) { - endCallback(frame, &l_statusFrames, &state->statusInFlight); + endCallback(frame, &l_statusFrames, + &state->statusInFlight, &state->statusWait); } -static void waitStatusCallbacks(const LGA_USBState * state) +static void waitStatusCallbacks(LGA_USBState * state) { - const unsigned int depth = callbackDepth(l_statusFrames, state); - while (atomic_load_explicit( - &state->statusInFlight, memory_order_acquire) > depth) - ; + waitCallbacks(state, l_statusFrames, + &state->statusInFlight, &state->statusWait); +} + +/* Returns with stateLock held. A callback must not wait for a detach which + * may itself be waiting for that callback to return. */ +static bool lockStateAfterDetach(LGA_USBState * state) +{ + for (;;) + { + LG_LOCK(state->stateLock); + if (!state->detaching) + return true; + LG_UNLOCK(state->stateLock); + + if (callbackDepth(l_playbackFrames, state) || + callbackDepth(l_recordFrames, state)) + return false; + waitEvent(state->detachEvent); + } } static void usbPlaybackStart( @@ -294,10 +518,10 @@ static void usbPlaybackStart( atomic_store_explicit( &state->playbackDeliveryGeneration, 0, memory_order_seq_cst); clock = makePlaybackClock(state, 0); - const bool admitted = beginEventNL(state, &target); + const bool admitted = beginPlaybackEventNL(state, &target); dispatch = admitted && target.events->playbackStart; if (admitted && !dispatch) - endEvent(state, &target); + endEvent(&target); LG_UNLOCK(state->stateLock); if (!dispatch) @@ -311,7 +535,7 @@ static void usbPlaybackStart( atomic_store_explicit(&state->playbackDeliveryGeneration, generation, memory_order_seq_cst); LG_UNLOCK(state->stateLock); - endEvent(state, &target); + endEvent(&target); } static void usbPlaybackStop(void * opaque) @@ -330,10 +554,10 @@ static void usbPlaybackStop(void * opaque) state->playbackGeneration = 0; atomic_store_explicit( &state->playbackDeliveryGeneration, 0, memory_order_seq_cst); - const bool admitted = beginEventNL(state, &target); + const bool admitted = beginPlaybackEventNL(state, &target); LG_UNLOCK(state->stateLock); - waitEvents(state); + waitPlaybackEvents(state); if (!admitted) return; @@ -345,7 +569,7 @@ static void usbPlaybackStop(void * opaque) if (dispatch) target.events->playbackStop(target.opaque, generation); - endEvent(state, &target); + endEvent(&target); } static void usbPlaybackData( @@ -353,7 +577,8 @@ static void usbPlaybackData( { LGA_USBState * state = opaque; USBAudioCallbackFrame frame; - beginCallback(state, &frame, &l_eventFrames, &state->inFlight); + beginCallback(state, &frame, &l_playbackFrames, + &state->playbackGate.inFlight); const uint64_t position = atomic_fetch_add_explicit( &state->playbackPosition, frames, memory_order_relaxed); @@ -370,7 +595,8 @@ static void usbPlaybackData( target, generation, data, frames, &clock); } } - endCallback(&frame, &l_eventFrames, &state->inFlight); + endCallback(&frame, &l_playbackFrames, + &state->playbackGate.inFlight, &state->playbackGate.wait); } static void usbRecordStart( @@ -378,11 +604,15 @@ static void usbRecordStart( { LGA_USBState * state = opaque; USBAudioEventTarget target; + LG_AudioFormat format; bool dispatch; uint32_t generation; + const bool recordOwner = beginOperation( + state, &state->recordOperation, l_recordFrames); LG_LOCK(state->stateLock); setStreamFormat(&state->recordFormat, sampleRate, channelMask); + format = state->recordFormat; generation = state->recordGeneration; if (!generation) { @@ -392,49 +622,58 @@ static void usbRecordStart( } atomic_store_explicit( &state->recordDeliveryGeneration, 0, memory_order_seq_cst); - const bool admitted = beginEventNL(state, &target); + const bool admitted = beginRecordEventNL(state, &target); dispatch = admitted && target.events->recordStart; + if (dispatch) + atomic_store_explicit(&state->recordDeliveryGeneration, + generation, memory_order_seq_cst); if (admitted && !dispatch) - endEvent(state, &target); + endEvent(&target); LG_UNLOCK(state->stateLock); if (!dispatch) + { + endOperation(&state->recordOperation, recordOwner); return; + } target.events->recordStart( - target.opaque, generation, &state->recordFormat); - - LG_LOCK(state->stateLock); - if (recordEventCurrentNL(state, &target, generation)) - atomic_store_explicit(&state->recordDeliveryGeneration, - generation, memory_order_seq_cst); - LG_UNLOCK(state->stateLock); - endEvent(state, &target); + target.opaque, generation, &format); + endEvent(&target); + endOperation(&state->recordOperation, recordOwner); } static void usbRecordStop(void * opaque) { LGA_USBState * state = opaque; USBAudioEventTarget target; + const bool recordOwner = beginOperation( + state, &state->recordOperation, l_recordFrames); LG_LOCK(state->stateLock); const uint32_t generation = state->recordGeneration; if (!generation) { LG_UNLOCK(state->stateLock); + endOperation(&state->recordOperation, recordOwner); return; } state->recordGeneration = 0; atomic_store_explicit( &state->recordDeliveryGeneration, 0, memory_order_seq_cst); - const bool admitted = beginEventNL(state, &target); + const bool admitted = beginRecordEventNL(state, &target); LG_UNLOCK(state->stateLock); - waitEvents(state); + /* A reentrant stop cannot quiesce callbacks for the same reason that it + * cannot wait for ownership above. */ + waitRecordEvents(state); if (!admitted) + { + endOperation(&state->recordOperation, recordOwner); return; + } LG_LOCK(state->stateLock); const bool dispatch = attachmentCurrentNL(state, &target) && @@ -443,7 +682,8 @@ static void usbRecordStop(void * opaque) if (dispatch) target.events->recordStop(target.opaque, generation); - endEvent(state, &target); + endEvent(&target); + endOperation(&state->recordOperation, recordOwner); } static const LG_USBAudioEventOps l_usbAudioEvents = @@ -458,7 +698,8 @@ static const LG_USBAudioEventOps l_usbAudioEvents = static void usbSetAvailable(void * opaque, bool available) { LGA_USBState * state = opaque; - const bool statusOwner = beginStatusOperation(state); + const bool statusOwner = beginOperation( + state, &state->statusOperation, l_statusFrames); LG_LOCK(state->statusLock); const bool changed = atomic_load_explicit( @@ -488,14 +729,15 @@ static void usbSetAvailable(void * opaque, bool available) callback(callbackOpaque, &status); endStatusCallback(state, &frame); } - endStatusOperation(state, statusOwner); + endOperation(&state->statusOperation, statusOwner); } static void usbSetStatusListener(void * opaque, LG_AudioStatusFn callback, void * callbackOpaque) { LGA_USBState * state = opaque; - const bool statusOwner = beginStatusOperation(state); + const bool statusOwner = beginOperation( + state, &state->statusOperation, l_statusFrames); LG_LOCK(state->statusLock); state->statusCallback = callback; @@ -518,7 +760,7 @@ static void usbSetStatusListener(void * opaque, } else waitStatusCallbacks(state); - endStatusOperation(state, statusOwner); + endOperation(&state->statusOperation, statusOwner); } static bool usbAttach(void * opaque, const LG_AudioEventOps * events, @@ -529,25 +771,21 @@ static bool usbAttach(void * opaque, const LG_AudioEventOps * events, &state->available, memory_order_acquire)) return false; - USBAudioEventTarget target; + USBAudioEventTarget playbackTarget; + USBAudioEventTarget recordTarget; LG_AudioFormat playbackFormat; LG_AudioFormat recordFormat; LG_AudioClock playbackClock; + uint32_t attachmentGeneration; uint32_t playbackGeneration; uint32_t recordGeneration; bool playbackDispatch; bool recordDispatch; - bool admitted; + bool playbackAdmitted; + bool recordAdmitted; - for (;;) - { - LG_LOCK(state->stateLock); - if (!state->detaching) - break; - LG_UNLOCK(state->stateLock); - if (callbackDepth(l_eventFrames, state)) - return false; - } + if (!lockStateAfterDetach(state)) + return false; if (state->attached || !atomic_load_explicit( &state->available, memory_order_acquire)) @@ -561,61 +799,72 @@ static bool usbAttach(void * opaque, const LG_AudioEventOps * events, nextGeneration(state->attachmentGeneration); state->events = events; state->eventOpaque = eventOpaque; + attachmentGeneration = state->attachmentGeneration; playbackGeneration = state->playbackGeneration; recordGeneration = state->recordGeneration; playbackDispatch = playbackGeneration && events->playbackStart; recordDispatch = recordGeneration && events->recordStart; - admitted = (playbackDispatch || recordDispatch) && - beginEventNL(state, &target); - playbackDispatch = playbackDispatch && admitted; - recordDispatch = recordDispatch && admitted; + playbackAdmitted = playbackDispatch && + beginPlaybackEventNL(state, &playbackTarget); + playbackDispatch = playbackDispatch && playbackAdmitted; + recordAdmitted = false; if (playbackDispatch) { playbackFormat = state->playbackFormat; playbackClock = makePlaybackClock(state, atomic_load_explicit( &state->playbackPosition, memory_order_relaxed)); } - if (recordDispatch) - recordFormat = state->recordFormat; lgUsbRedir_setPlugged(state->redir, true); LG_UNLOCK(state->stateLock); if (playbackDispatch) { - target.events->playbackStart( - target.opaque, playbackGeneration, + playbackTarget.events->playbackStart( + playbackTarget.opaque, playbackGeneration, &playbackFormat, &playbackClock); LG_LOCK(state->stateLock); if (playbackEventCurrentNL( - state, &target, playbackGeneration)) + state, &playbackTarget, playbackGeneration)) atomic_store_explicit(&state->playbackDeliveryGeneration, playbackGeneration, memory_order_seq_cst); LG_UNLOCK(state->stateLock); } + if (playbackAdmitted) + endEvent(&playbackTarget); if (recordDispatch) { + const bool recordOwner = beginOperation( + state, &state->recordOperation, l_recordFrames); + LG_LOCK(state->stateLock); - recordDispatch = recordEventCurrentNL( - state, &target, recordGeneration); + recordDispatch = state->attached && + state->attachmentGeneration == attachmentGeneration && + state->events == events && + state->eventOpaque == eventOpaque && + state->recordGeneration == recordGeneration; + recordAdmitted = recordDispatch && + beginRecordEventNL(state, &recordTarget); + recordDispatch = recordAdmitted && + recordEventCurrentNL(state, &recordTarget, recordGeneration); + if (recordDispatch) + { + recordFormat = state->recordFormat; + atomic_store_explicit(&state->recordDeliveryGeneration, + recordGeneration, memory_order_seq_cst); + } LG_UNLOCK(state->stateLock); if (recordDispatch) - target.events->recordStart( - target.opaque, recordGeneration, &recordFormat); + recordTarget.events->recordStart( + recordTarget.opaque, recordGeneration, &recordFormat); - LG_LOCK(state->stateLock); - if (recordDispatch && - recordEventCurrentNL(state, &target, recordGeneration)) - atomic_store_explicit(&state->recordDeliveryGeneration, - recordGeneration, memory_order_seq_cst); - LG_UNLOCK(state->stateLock); + if (recordAdmitted) + endEvent(&recordTarget); + endOperation(&state->recordOperation, recordOwner); } - if (admitted) - endEvent(state, &target); - return true; } @@ -623,17 +872,11 @@ static void usbDetach(void * opaque) { LGA_USBState * state = opaque; - for (;;) - { - LG_LOCK(state->stateLock); - if (!state->detaching) - break; - LG_UNLOCK(state->stateLock); - if (callbackDepth(l_eventFrames, state)) - return; - } + if (!lockStateAfterDetach(state)) + return; state->detaching = true; + lgResetEvent(state->detachEvent); state->attached = false; state->attachmentGeneration = nextGeneration(state->attachmentGeneration); @@ -646,7 +889,8 @@ static void usbDetach(void * opaque) lgUsbRedir_setPlugged(state->redir, false); LG_UNLOCK(state->stateLock); - waitEvents(state); + waitPlaybackEvents(state); + waitRecordEvents(state); LG_LOCK(state->stateLock); if (!state->attached && @@ -655,6 +899,7 @@ static void usbDetach(void * opaque) state->events = NULL; state->eventOpaque = NULL; state->detaching = false; + signalWaitEvent(state->detachEvent); } LG_UNLOCK(state->stateLock); } @@ -666,7 +911,8 @@ static bool usbClockFeedback(void * opaque, uint32_t generation, LG_LOCK(state->stateLock); if (!state->attached || - state->playbackGeneration != generation) + state->playbackGeneration != generation || + !lgUsbAudio_feedbackActive(state->device)) { LG_UNLOCK(state->stateLock); return false; @@ -686,17 +932,19 @@ static bool usbRecordData(void * opaque, uint32_t generation, const void * data, size_t frames, const LG_AudioClock * sourceClock) { - (void)sourceClock; LGA_USBState * state = opaque; USBAudioCallbackFrame frame; - beginCallback(state, &frame, &l_eventFrames, &state->inFlight); + beginCallback(state, &frame, &l_recordFrames, + &state->recordGate.inFlight); const bool valid = generation && generation == atomic_load_explicit( &state->recordDeliveryGeneration, memory_order_seq_cst); const bool result = valid && - lgUsbAudio_recordData(state->device, data, frames); + lgUsbAudio_recordData( + state->device, data, frames, sourceClock); - endCallback(&frame, &l_eventFrames, &state->inFlight); + endCallback(&frame, &l_recordFrames, + &state->recordGate.inFlight, &state->recordGate.wait); return result; } @@ -710,28 +958,47 @@ const LG_AudioOps LGA_USB = .clockFeedback = usbClockFeedback, }; -LGA_USBState * lgaUsb_create(void) +LGA_USBState * lgaUsb_create(bool debug) { - LGA_USBState * state = calloc(1, sizeof(*state)); + LGA_USBState * state = aligned_alloc( + alignof(LGA_USBState), sizeof(*state)); if (!state) return NULL; + memset(state, 0, sizeof(*state)); LG_LOCK_INIT(state->statusLock); + LG_LOCK_INIT(state->statusWait.lock); + LG_LOCK_INIT(state->statusOperation.lock); LG_LOCK_INIT(state->stateLock); + LG_LOCK_INIT(state->playbackGate.wait.lock); + LG_LOCK_INIT(state->recordGate.wait.lock); + LG_LOCK_INIT(state->recordOperation.lock); atomic_init(&state->available, false); atomic_init(&state->playbackPosition, 0); atomic_init(&state->playbackDeliveryGeneration, 0); atomic_init(&state->recordDeliveryGeneration, 0); - atomic_init(&state->inFlight, 0); + atomic_init(&state->playbackGate.inFlight, 0); + atomic_init(&state->recordGate.inFlight, 0); atomic_init(&state->statusInFlight, 0); - atomic_init(&state->statusNextTicket, 0); - atomic_init(&state->statusServingTicket, 0); + atomic_init(&state->playbackGate.wait.count, 0); + atomic_init(&state->recordGate.wait.count, 0); + atomic_init(&state->statusWait.count, 0); state->playbackFormat = l_formatTemplate; state->recordFormat = l_formatTemplate; - state->device = lgUsbAudio_create(&l_usbAudioEvents, state); + state->detachEvent = lgCreateEvent(false, 0); + if (!state->detachEvent || !lgSignalEvent(state->detachEvent)) + { + if (state->detachEvent) + lgFreeEvent(state->detachEvent); + free(state); + return NULL; + } + + state->device = lgUsbAudio_create(&l_usbAudioEvents, state, debug); if (!state->device) { + lgFreeEvent(state->detachEvent); free(state); return NULL; } @@ -741,6 +1008,7 @@ LGA_USBState * lgaUsb_create(void) if (!state->redir) { lgUsbAudio_destroy(state->device); + lgFreeEvent(state->detachEvent); free(state); return NULL; } @@ -757,7 +1025,13 @@ void lgaUsb_destroy(LGA_USBState * state) usbDetach(state); lgUsbRedir_destroy(state->redir); lgUsbAudio_destroy(state->device); + lgFreeEvent(state->detachEvent); + LG_LOCK_FREE(state->recordOperation.lock); + LG_LOCK_FREE(state->recordGate.wait.lock); + LG_LOCK_FREE(state->playbackGate.wait.lock); LG_LOCK_FREE(state->stateLock); + LG_LOCK_FREE(state->statusOperation.lock); + LG_LOCK_FREE(state->statusWait.lock); LG_LOCK_FREE(state->statusLock); free(state); } @@ -767,7 +1041,7 @@ LG_USBRedir * lgaUsb_redir(LGA_USBState * state) return state ? state->redir : NULL; } -bool lgaUsb_recording(const LGA_USBState * state) +uint64_t lgaUsb_processDelayNs(const LGA_USBState * state) { - return state && lgUsbAudio_recording(state->device); + return state ? lgUsbAudio_processDelayNs(state->device) : UINT64_MAX; } diff --git a/client/src/audio_usb.h b/client/src/audio_usb.h index b704a491..5f26ce87 100644 --- a/client/src/audio_usb.h +++ b/client/src/audio_usb.h @@ -24,14 +24,16 @@ #include "interface/audio.h" #include "usbredir.h" +#include + typedef struct LGA_USBState LGA_USBState; -LGA_USBState * lgaUsb_create(void); +LGA_USBState * lgaUsb_create(bool debug); /* Detach this provider and stop PureSpice before destroying its state. */ void lgaUsb_destroy(LGA_USBState * state); LG_USBRedir * lgaUsb_redir(LGA_USBState * state); -bool lgaUsb_recording(const LGA_USBState * state); +uint64_t lgaUsb_processDelayNs(const LGA_USBState * state); extern const LG_AudioOps LGA_USB; diff --git a/client/src/main.c b/client/src/main.c index 6aec5bc6..cf3515b7 100644 --- a/client/src/main.c +++ b/client/src/main.c @@ -1728,7 +1728,7 @@ int spiceThread(void * arg) DEBUG_WARN("USB audio requires a playback backend, using SPICE audio"); g_params.useSpiceUSBAudio = false; } - else if (!(usbAudio = lgaUsb_create())) + else if (!(usbAudio = lgaUsb_create(g_params.audioDebug))) { DEBUG_WARN("Failed to initialize USB audio, using SPICE audio"); g_params.useSpiceUSBAudio = false; @@ -1839,7 +1839,17 @@ int spiceThread(void * arg) if (usbRedir && !lgUsbRedir_process(usbRedir)) DEBUG_WARN("Failed to process USB audio redirection"); if (usbAudio) - processTimeout = lgaUsb_recording(usbAudio) ? 1 : 10; + { + const uint64_t delay = lgaUsb_processDelayNs(usbAudio); + if (delay == UINT64_MAX) + processTimeout = 10; + else + { + const uint64_t timeout = delay / UINT64_C(1000000) + + (delay % UINT64_C(1000000) != 0); + processTimeout = (int)min(timeout, UINT64_C(10)); + } + } #endif if ((status = purespice_process(processTimeout)) != PS_STATUS_RUN) diff --git a/client/src/overlay/status.c b/client/src/overlay/status.c index c9f83507..e35876a1 100644 --- a/client/src/overlay/status.c +++ b/client/src/overlay/status.c @@ -21,6 +21,7 @@ #include "interface/overlay.h" #include "math.h" #include "cimgui.h" +#include #include "../overlays.h" #include "../main.h" @@ -32,7 +33,7 @@ //TODO: Make this user configurable? #define ICON_SIZE 32 -static bool l_state[LG_USER_STATUS_MAX] = { 0 }; +static atomic_bool l_state[LG_USER_STATUS_MAX] = { 0 }; static OverlayImage l_image[LG_USER_STATUS_MAX] = { 0 }; static bool l_recordToggle; static double l_scale = 1.0; @@ -92,7 +93,8 @@ static int status_render(void * udata, bool interactive, struct Rect * windowRec for(int i = 0; i < LG_USER_STATUS_MAX; ++i) { OverlayImage * img = &l_image[i]; - if (!l_state[i] || !img->tex) + if (!atomic_load_explicit(&l_state[i], memory_order_relaxed) || + !img->tex) continue; // if the recording indicator is off, don't draw but reserve space @@ -147,9 +149,9 @@ struct LG_OverlayOps LGOverlayStatus = void overlayStatus_set(LGUserStatus status, bool value) { - if (l_state[status] == value) + if (atomic_exchange_explicit( + &l_state[status], value, memory_order_relaxed) == value) return; - l_state[status] = value; app_invalidateOverlay(true); }; diff --git a/client/src/usb_audio.c b/client/src/usb_audio.c index 2c65d8b3..a9bcde97 100644 --- a/client/src/usb_audio.c +++ b/client/src/usb_audio.c @@ -21,12 +21,13 @@ #include "usb_audio.h" #include "common/debug.h" -#include "common/locking.h" +#include "common/event.h" #include "common/ringbuffer.h" #include "common/time.h" #include +#include #include #include #include @@ -60,7 +61,15 @@ enum USB_AUDIO_FEEDBACK_SIZE = 4, USB_AUDIO_FEEDBACK_MAX_QUEUE = 512, USB_AUDIO_SAMPLE_SIZE = 3, + USB_AUDIO_MAX_SAMPLE_RATE = 192000, + USB_AUDIO_PLAYBACK_MAX_CHANNELS = 8, USB_AUDIO_PLAYBACK_HS_PACKET_FRAMES = 25, + USB_AUDIO_PLAYBACK_BATCH_PACKETS = USB_AUDIO_FEEDBACK_INTERVAL, + USB_AUDIO_PLAYBACK_BATCH_FRAMES = + USB_AUDIO_PLAYBACK_HS_PACKET_FRAMES * USB_AUDIO_PLAYBACK_BATCH_PACKETS, + USB_AUDIO_PLAYBACK_BATCH_SIZE = + USB_AUDIO_PLAYBACK_BATCH_FRAMES * + USB_AUDIO_PLAYBACK_MAX_CHANNELS * USB_AUDIO_SAMPLE_SIZE, USB_AUDIO_FS_PACKET_FRAMES = 97, USB_AUDIO_RECORD_PACKET_FRAMES = 97, USB_AUDIO_RECORD_PACKETS_PER_SECOND = 2000, @@ -70,8 +79,15 @@ enum USB_AUDIO_RECORD_RATE_GAP_NS = 100000000, USB_AUDIO_RECORD_RATE_DENOMINATOR = USB_AUDIO_RECORD_PACKETS_PER_SECOND * USB_AUDIO_RECORD_RATE_SCALE, - USB_AUDIO_RECORD_MAX_BATCH = 64, - USB_AUDIO_RECORD_QUEUE_FRAMES = 3840, + USB_AUDIO_RECORD_MAX_BATCH = 128, + USB_AUDIO_RECORD_MAX_SAFETY_PACKETS = + USB_AUDIO_RECORD_PACKETS_PER_SECOND / 10, + /* Hold the complete QEMU ISO-IN safety window in addition to the normal + * 20 ms queue target at the maximum supported sample rate. */ + USB_AUDIO_RECORD_QUEUE_FRAMES = + USB_AUDIO_RECORD_MAX_SAFETY_PACKETS * + USB_AUDIO_RECORD_PACKET_FRAMES + + USB_AUDIO_MAX_SAMPLE_RATE / 50, USB_AUDIO_RECORD_FRAME_SIZE = 2 * USB_AUDIO_SAMPLE_SIZE, USB_AUDIO_RECORD_PACKET_SIZE = @@ -82,6 +98,8 @@ enum USB_AUDIO_RECORD_STREAM_DESC_SIZE = 46, }; +#define USB_AUDIO_RECORD_DEBUG_INTERVAL_NS UINT64_C(5000000000) + enum { USB_AUDIO_LAYOUT_STEREO = 0x00000003, @@ -299,7 +317,7 @@ static const uint32_t l_sampleRates[] = 88200, 96000, 176400, - 192000, + USB_AUDIO_MAX_SAMPLE_RATE, }; static const uint8_t l_clockValid[] = { 0x01 }; @@ -327,33 +345,67 @@ _Static_assert(USB_AUDIO_FS_PACKET_FRAMES * 2 * USB_AUDIO_SAMPLE_SIZE <= _Static_assert(USB_AUDIO_RECORD_PACKET_SIZE <= 1024, "high-speed recording packet is too large"); +typedef enum +{ + RECORD_RATE_NONE, + RECORD_RATE_ARRIVAL, + RECORD_RATE_SOURCE +} +RecordRateMode; + struct LG_USBAudio { const LG_USBAudioEventOps * events; void * eventOpaque; struct usbredirparser * parser; + bool debug; - uint8_t configuration; - uint8_t playbackAlt; - uint8_t recordAlt; - uint32_t playbackSampleRate; - uint32_t recordSampleRate; - bool playbackStreaming; - bool feedbackStreaming; - uint64_t feedbackPacketId; - uint8_t feedbackDataPackets; - atomic_uint feedbackValue; + uint8_t configuration; + uint8_t playbackAlt; + uint8_t recordAlt; + uint32_t playbackSampleRate; + atomic_uint recordSampleRate; + bool playbackStreaming; + uint8_t playbackBatchPackets; + uint8_t playbackBatchFrameSize; + uint16_t playbackBatchFrames; + atomic_bool feedbackStreaming; + uint64_t feedbackPacketId; + uint8_t feedbackDataPackets; + atomic_uint feedbackValue; - atomic_bool recordStreaming; - uint64_t recordPacketId; - uint64_t recordPacketPhase; - uint64_t recordNextPacketTime; - LG_Lock recordLock; - RingBuffer recordBuffer; - uint64_t recordRateQ16; - uint64_t recordRateStartTime; - uint64_t recordRateLastTime; - uint64_t recordRateFrames; + atomic_bool recordStreaming; + atomic_bool recordAccepting; + atomic_bool recordOverflowPending; + atomic_uint recordGeneration; + atomic_uint recordInFlight; + LGEvent * recordIdle; + uint64_t recordPacketId; + uint64_t recordPacketPhase; + uint64_t recordNextPacketTime; + uint32_t recordBatchPackets; + uint32_t recordSafetyPackets; + uint32_t recordRefillPackets; + RingBuffer recordBuffer; + atomic_uint_fast64_t recordRateQ16; + RecordRateMode recordRateMode; + uint64_t recordArrivalStartTime; + uint64_t recordArrivalLastTime; + uint64_t recordArrivalFrames; + uint64_t recordSourceStartPosition; + uint64_t recordSourceLastPosition; + int64_t recordSourceStartTime; + int64_t recordSourceLastTime; + + atomic_uint_fast64_t recordOverflowFrames; + atomic_uint_fast64_t recordUnderflowFrames; + atomic_uint_fast64_t recordTrimmedFrames; + atomic_uint_fast64_t recordLatePackets; + atomic_uint_fast64_t recordMaxLateness; + atomic_uint_fast64_t recordDiscontinuities; + uint64_t recordNextDebugTime; + + uint8_t playbackBatch[USB_AUDIO_PLAYBACK_BATCH_SIZE]; }; static LG_USBAudio * getAudio(void * opaque) @@ -368,6 +420,43 @@ static uint64_t recordTime(void) return microtime() * UINT64_C(1000); } +static void recordCounterMax( + atomic_uint_fast64_t * counter, uint64_t value) +{ + uint_fast64_t current = atomic_load_explicit( + counter, memory_order_relaxed); + while (current < value && !atomic_compare_exchange_weak_explicit( + counter, ¤t, value, + memory_order_relaxed, memory_order_relaxed)) + ; +} + +static void recordDisableInput(LG_USBAudio * audio) +{ + /* A producer takes the generation before entering recordInFlight. Changing + * both values prevents a callback which raced the gate from entering after + * the consumer has reset the SPSC queue. */ + atomic_store_explicit( + &audio->recordAccepting, false, memory_order_seq_cst); + atomic_fetch_add_explicit( + &audio->recordGeneration, 1, memory_order_seq_cst); +} + +static void recordWaitForInput(LG_USBAudio * audio) +{ + while (atomic_load_explicit( + &audio->recordInFlight, memory_order_seq_cst)) + if (!lgWaitEvent(audio->recordIdle, TIMEOUT_INFINITE)) + DEBUG_FATAL("Failed to wait for USB capture callbacks"); +} + +static int recordDiscardQueued(LG_USBAudio * audio) +{ + const int queued = ringbuffer_getCount(audio->recordBuffer); + return queued > 0 ? + ringbuffer_consume(audio->recordBuffer, NULL, queued) : 0; +} + static uint32_t readLE32(const uint8_t * data) { return @@ -454,8 +543,43 @@ static size_t writeSampleRateRange(uint8_t * buffer) return 2 + count * 12; } +static void flushPlayback(LG_USBAudio * audio) +{ + const size_t frames = audio->playbackBatchFrames; + audio->playbackBatchPackets = 0; + audio->playbackBatchFrameSize = 0; + audio->playbackBatchFrames = 0; + + if (frames && audio->playbackStreaming && audio->events && + audio->events->playbackData) + audio->events->playbackData( + audio->eventOpaque, audio->playbackBatch, frames); +} + +static void queuePlaybackPacket(LG_USBAudio * audio, + const uint8_t * data, size_t frames, uint8_t frameSize) +{ + if (audio->playbackBatchPackets == USB_AUDIO_PLAYBACK_BATCH_PACKETS || + audio->playbackBatchFrames + frames > + USB_AUDIO_PLAYBACK_BATCH_FRAMES || + (audio->playbackBatchFrames && + audio->playbackBatchFrameSize != frameSize)) + flushPlayback(audio); + + if (frames) + { + memcpy(audio->playbackBatch + + (size_t)audio->playbackBatchFrames * frameSize, + data, frames * frameSize); + audio->playbackBatchFrameSize = frameSize; + audio->playbackBatchFrames += (uint16_t)frames; + } + ++audio->playbackBatchPackets; +} + static void stopPlayback(LG_USBAudio * audio) { + flushPlayback(audio); if (!audio->playbackStreaming) return; @@ -467,7 +591,8 @@ static void stopPlayback(LG_USBAudio * audio) static void stopFeedback(LG_USBAudio * audio) { - audio->feedbackStreaming = false; + atomic_store_explicit( + &audio->feedbackStreaming, false, memory_order_release); audio->feedbackDataPackets = 0; } @@ -493,29 +618,184 @@ static void startPlayback(LG_USBAudio * audio) static void stopRecord(LG_USBAudio * audio) { - if (!atomic_exchange_explicit( - &audio->recordStreaming, false, memory_order_acq_rel)) + if (!atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire)) return; + recordDisableInput(audio); + atomic_store_explicit( + &audio->recordStreaming, false, memory_order_release); + if (audio->events && audio->events->recordStop) audio->events->recordStop(audio->eventOpaque); - LG_LOCK(audio->recordLock); + recordWaitForInput(audio); ringbuffer_reset(audio->recordBuffer); - LG_UNLOCK(audio->recordLock); + atomic_store_explicit( + &audio->recordOverflowPending, false, memory_order_relaxed); + audio->recordRefillPackets = 0; +} + +static void clearRecordRateWindows(LG_USBAudio * audio) +{ + audio->recordRateMode = RECORD_RATE_NONE; + audio->recordArrivalStartTime = 0; + audio->recordArrivalLastTime = 0; + audio->recordArrivalFrames = 0; + audio->recordSourceStartPosition = 0; + audio->recordSourceLastPosition = 0; + audio->recordSourceStartTime = 0; + audio->recordSourceLastTime = 0; +} + +static void resetRecordRate( + LG_USBAudio * audio, uint32_t sampleRate) +{ + atomic_store_explicit(&audio->recordRateQ16, + (uint64_t)sampleRate * USB_AUDIO_RECORD_RATE_SCALE, + memory_order_release); + clearRecordRateWindows(audio); +} + +static bool recordRateInRange( + uint32_t sampleRate, double measured) +{ + return measured >= sampleRate * 0.995 && + measured <= sampleRate * 1.005; +} + +static void updateMeasuredRecordRate( + LG_USBAudio * audio, double measured) +{ + const uint64_t measuredQ16 = (uint64_t)( + measured * USB_AUDIO_RECORD_RATE_SCALE + 0.5); + const uint64_t currentQ16 = atomic_load_explicit( + &audio->recordRateQ16, memory_order_relaxed); + atomic_store_explicit(&audio->recordRateQ16, + (currentQ16 * 3 + measuredQ16) / 4, + memory_order_release); +} + +static void startRecordArrivalWindow( + LG_USBAudio * audio, uint64_t now) +{ + clearRecordRateWindows(audio); + audio->recordRateMode = RECORD_RATE_ARRIVAL; + audio->recordArrivalStartTime = now; + audio->recordArrivalLastTime = now; +} + +static void updateRecordArrivalRate( + LG_USBAudio * audio, uint32_t sampleRate, size_t frames) +{ + const uint64_t now = recordTime(); + if (audio->recordRateMode != RECORD_RATE_ARRIVAL) + { + /* Preserve the current rate while a temporarily absent source clock + * starts a fresh fallback window. */ + startRecordArrivalWindow(audio, now); + return; + } + + if (now < audio->recordArrivalLastTime || + now - audio->recordArrivalLastTime > USB_AUDIO_RECORD_RATE_GAP_NS) + { + atomic_store_explicit(&audio->recordRateQ16, + (uint64_t)sampleRate * USB_AUDIO_RECORD_RATE_SCALE, + memory_order_release); + startRecordArrivalWindow(audio, now); + return; + } + + audio->recordArrivalFrames += frames; + audio->recordArrivalLastTime = now; + const uint64_t elapsed = now - audio->recordArrivalStartTime; + if (elapsed < USB_AUDIO_RECORD_RATE_SETTLE_NS) + return; + + const double measured = + (double)audio->recordArrivalFrames * 1000000000.0 / elapsed; + if (recordRateInRange(sampleRate, measured)) + updateMeasuredRecordRate(audio, measured); + + audio->recordArrivalStartTime = now; + audio->recordArrivalFrames = 0; +} + +static void startRecordSourceWindow( + LG_USBAudio * audio, const LG_AudioClock * sourceClock) +{ + clearRecordRateWindows(audio); + audio->recordRateMode = RECORD_RATE_SOURCE; + audio->recordSourceStartPosition = sourceClock->position; + audio->recordSourceLastPosition = sourceClock->position; + audio->recordSourceStartTime = sourceClock->time; + audio->recordSourceLastTime = sourceClock->time; +} + +static bool updateRecordSourceRate(LG_USBAudio * audio, + uint32_t sampleRate, const LG_AudioClock * sourceClock) +{ + if (sourceClock->discontinuity) + { + resetRecordRate(audio, sampleRate); + return false; + } + + if (audio->recordRateMode != RECORD_RATE_SOURCE) + { + /* Clock stability is not needed to establish the first position/time + * baseline. */ + startRecordSourceWindow(audio, sourceClock); + return true; + } + + if (sourceClock->position <= audio->recordSourceLastPosition || + sourceClock->time <= audio->recordSourceLastTime) + { + resetRecordRate(audio, sampleRate); + return false; + } + + if (!sourceClock->stable) + { + /* Keep an unstable clock as the next baseline, but do not let it + * contribute to the measured device rate. */ + startRecordSourceWindow(audio, sourceClock); + return true; + } + + audio->recordSourceLastPosition = sourceClock->position; + audio->recordSourceLastTime = sourceClock->time; + const int64_t elapsed = + sourceClock->time - audio->recordSourceStartTime; + if (elapsed < USB_AUDIO_RECORD_RATE_SETTLE_NS) + return true; + + const double measured = + (double)(sourceClock->position - + audio->recordSourceStartPosition) * 1000000000.0 / elapsed; + if (!recordRateInRange(sampleRate, measured)) + { + resetRecordRate(audio, sampleRate); + return false; + } + + updateMeasuredRecordRate(audio, measured); + audio->recordSourceStartPosition = sourceClock->position; + audio->recordSourceStartTime = sourceClock->time; + return true; } static void resetRecordData(LG_USBAudio * audio) { - LG_LOCK(audio->recordLock); ringbuffer_reset(audio->recordBuffer); - audio->recordRateQ16 = - (uint64_t)audio->recordSampleRate * USB_AUDIO_RECORD_RATE_SCALE; - audio->recordRateStartTime = 0; - audio->recordRateLastTime = 0; - audio->recordRateFrames = 0; - LG_UNLOCK(audio->recordLock); - audio->recordPacketPhase = 0; + resetRecordRate(audio, atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed)); + audio->recordPacketPhase = 0; + audio->recordRefillPackets = 0; + atomic_store_explicit( + &audio->recordOverflowPending, false, memory_order_relaxed); } static void startRecord(LG_USBAudio * audio) @@ -524,6 +804,8 @@ static void startRecord(LG_USBAudio * audio) &audio->recordStreaming, memory_order_acquire)) return; + recordDisableInput(audio); + recordWaitForInput(audio); resetRecordData(audio); audio->recordPacketId = 0; atomic_store_explicit( @@ -531,27 +813,39 @@ static void startRecord(LG_USBAudio * audio) if (audio->events && audio->events->recordStart) audio->events->recordStart(audio->eventOpaque, - audio->recordSampleRate, USB_AUDIO_LAYOUT_STEREO); + atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed), + USB_AUDIO_LAYOUT_STEREO); /* Backend startup may block while the capture graph is configured. Do not * turn that setup time into a burst of stale USB packets. */ + recordWaitForInput(audio); audio->recordNextPacketTime = recordTime(); + atomic_store_explicit( + &audio->recordAccepting, true, memory_order_seq_cst); } -static void reconfigureRecord(LG_USBAudio * audio) +static void reconfigureRecord( + LG_USBAudio * audio, uint32_t sampleRate) { if (!atomic_load_explicit( &audio->recordStreaming, memory_order_acquire)) return; + recordDisableInput(audio); + atomic_store_explicit( + &audio->recordSampleRate, sampleRate, memory_order_relaxed); if (audio->events && audio->events->recordStart) audio->events->recordStart(audio->eventOpaque, - audio->recordSampleRate, USB_AUDIO_LAYOUT_STEREO); + sampleRate, USB_AUDIO_LAYOUT_STEREO); /* The provider restart has quiesced the old producer. Drop any frames * queued around the format transition and restart the USB sample phase. */ + recordWaitForInput(audio); resetRecordData(audio); audio->recordNextPacketTime = recordTime(); + atomic_store_explicit( + &audio->recordAccepting, true, memory_order_seq_cst); } static void stopStreams(LG_USBAudio * audio) @@ -577,17 +871,21 @@ static void setPlaybackSampleRate( static void setRecordSampleRate( LG_USBAudio * audio, uint32_t sampleRate) { - LG_LOCK(audio->recordLock); - const bool changed = audio->recordSampleRate != sampleRate; - audio->recordSampleRate = sampleRate; - LG_UNLOCK(audio->recordLock); - if (!changed) + if (atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed) == sampleRate) return; - const bool restartRecord = atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire); - if (restartRecord) - reconfigureRecord(audio); + if (atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire)) + reconfigureRecord(audio, sampleRate); + else + { + atomic_store_explicit( + &audio->recordSampleRate, sampleRate, memory_order_relaxed); + atomic_store_explicit(&audio->recordRateQ16, + (uint64_t)sampleRate * USB_AUDIO_RECORD_RATE_SCALE, + memory_order_relaxed); + } } static void resetDevice(LG_USBAudio * audio) @@ -597,9 +895,14 @@ static void resetDevice(LG_USBAudio * audio) audio->playbackAlt = 0; audio->recordAlt = 0; audio->playbackSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; - LG_LOCK(audio->recordLock); - audio->recordSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; - LG_UNLOCK(audio->recordLock); + audio->recordBatchPackets = 1; + audio->recordSafetyPackets = 1; + atomic_store_explicit(&audio->recordSampleRate, + LG_USB_AUDIO_DEFAULT_SAMPLE_RATE, memory_order_relaxed); + atomic_store_explicit(&audio->recordRateQ16, + (uint64_t)LG_USB_AUDIO_DEFAULT_SAMPLE_RATE * + USB_AUDIO_RECORD_RATE_SCALE, + memory_order_relaxed); resetFeedbackRate(audio); } @@ -718,6 +1021,7 @@ static void setAltSetting(void * opaque, uint64_t id, struct usb_redir_set_alt_setting_header * request) { LG_USBAudio * audio = getAudio(opaque); + flushPlayback(audio); struct usb_redir_alt_setting_status_header status = { .status = usb_redir_stall, @@ -812,40 +1116,19 @@ static void sendFeedbackPackets(LG_USBAudio * audio, uint32_t count) &packet, data, USB_AUDIO_FEEDBACK_SIZE); } -static void sendRecordPackets(LG_USBAudio * audio) +static void sendRecordPacketBatch(LG_USBAudio * audio, uint32_t count, + uint64_t rateQ16, bool forceSilence, uint64_t leadingSilenceFrames, + uint64_t availableFrames) { - if (!atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire)) - return; - - const uint64_t now = recordTime(); - if (audio->recordNextPacketTime > now) - return; - - uint64_t count = - (now - audio->recordNextPacketTime) / - USB_AUDIO_RECORD_PACKET_INTERVAL_NS + 1; - if (count > USB_AUDIO_RECORD_MAX_BATCH) - { - count = USB_AUDIO_RECORD_MAX_BATCH; - audio->recordNextPacketTime = now - - (count - 1) * USB_AUDIO_RECORD_PACKET_INTERVAL_NS; - } - uint8_t data[USB_AUDIO_RECORD_PACKET_SIZE]; - const int target = max( - (int)(audio->recordSampleRate / 50), - USB_AUDIO_RECORD_PACKET_FRAMES); - LG_LOCK(audio->recordLock); - const uint64_t rateQ16 = audio->recordRateQ16; - LG_UNLOCK(audio->recordLock); + uint64_t underflowFrames = 0; struct usb_redir_iso_packet_header packet = { .endpoint = USB_AUDIO_RECORD_DATA_ENDPOINT, .status = usb_redir_success, }; - for (uint64_t i = 0; i < count; ++i) + for (uint32_t i = 0; i < count; ++i) { audio->recordPacketPhase += rateQ16; const uint32_t frames = (uint32_t)(audio->recordPacketPhase / @@ -856,25 +1139,131 @@ static void sendRecordPackets(LG_USBAudio * audio) const size_t size = frames * USB_AUDIO_RECORD_FRAME_SIZE; memset(data, 0, size); - LG_LOCK(audio->recordLock); - const int queued = ringbuffer_getCount(audio->recordBuffer); - if (queued > target) - ringbuffer_consume(audio->recordBuffer, NULL, queued - target); - ringbuffer_consume(audio->recordBuffer, data, frames); - LG_UNLOCK(audio->recordLock); + if (!forceSilence) + { + const uint32_t leading = (uint32_t)min( + leadingSilenceFrames, (uint64_t)frames); + leadingSilenceFrames -= leading; + underflowFrames += leading; + + const uint32_t requested = frames - leading; + const uint32_t available = (uint32_t)min( + availableFrames, (uint64_t)requested); + const int consumed = ringbuffer_consume(audio->recordBuffer, + data + (size_t)leading * USB_AUDIO_RECORD_FRAME_SIZE, + available); + availableFrames -= consumed; + underflowFrames += requested - consumed; + } packet.length = (uint16_t)size; usbredirparser_send_iso_packet(audio->parser, audio->recordPacketId++, &packet, data, (int)size); - audio->recordNextPacketTime += - USB_AUDIO_RECORD_PACKET_INTERVAL_NS; } + + if (audio->debug && underflowFrames) + atomic_fetch_add_explicit(&audio->recordUnderflowFrames, + underflowFrames, memory_order_relaxed); +} + +static void sendRecordRefill(LG_USBAudio * audio, uint64_t rateQ16) +{ + const uint32_t count = min( + audio->recordRefillPackets, USB_AUDIO_RECORD_MAX_BATCH); + sendRecordPacketBatch(audio, count, rateQ16, true, 0, 0); + audio->recordRefillPackets -= count; + if (!audio->recordRefillPackets) + audio->recordNextPacketTime = + recordTime() + USB_AUDIO_RECORD_PACKET_INTERVAL_NS; +} + +static void sendRecordPackets(LG_USBAudio * audio) +{ + if (!atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire)) + return; + + const uint64_t rateQ16 = atomic_load_explicit( + &audio->recordRateQ16, memory_order_acquire); + if (audio->recordRefillPackets) + { + sendRecordRefill(audio, rateQ16); + return; + } + + const uint64_t now = recordTime(); + if (audio->recordNextPacketTime > now) + return; + + const bool overflow = atomic_exchange_explicit( + &audio->recordOverflowPending, false, memory_order_acq_rel); + const uint64_t lateness = now - audio->recordNextPacketTime; + uint64_t count = lateness / USB_AUDIO_RECORD_PACKET_INTERVAL_NS + 1; + if (audio->debug) + { + if (count > 1) + atomic_fetch_add_explicit(&audio->recordLatePackets, + count - 1, memory_order_relaxed); + recordCounterMax(&audio->recordMaxLateness, lateness); + } + + if (overflow || count > audio->recordSafetyPackets) + { + const int discarded = recordDiscardQueued(audio); + if (audio->debug) + { + if (discarded) + atomic_fetch_add_explicit(&audio->recordTrimmedFrames, + discarded, memory_order_relaxed); + atomic_fetch_add_explicit(&audio->recordDiscontinuities, + 1, memory_order_relaxed); + } + + audio->recordPacketPhase = 0; + audio->recordRefillPackets = audio->recordSafetyPackets; + sendRecordRefill(audio, rateQ16); + return; + } + + const uint32_t sendCount = (uint32_t)min( + count, (uint64_t)USB_AUDIO_RECORD_MAX_BATCH); + const uint64_t catchupFrames = + (audio->recordPacketPhase + rateQ16 * count) / + USB_AUDIO_RECORD_RATE_DENOMINATOR; + const uint64_t totalFrames = + (audio->recordPacketPhase + rateQ16 * sendCount) / + USB_AUDIO_RECORD_RATE_DENOMINATOR; + const uint32_t sampleRate = atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed); + /* Preserve the samples needed by every overdue packet. Trimming to the + * normal target first would replace valid catch-up audio with silence. */ + const int retainFrames = max( + (int)catchupFrames, + max((int)(sampleRate / 50), USB_AUDIO_RECORD_PACKET_FRAMES)); + int queued = ringbuffer_getCount(audio->recordBuffer); + if (queued > retainFrames) + { + const int discarded = ringbuffer_consume( + audio->recordBuffer, NULL, queued - retainFrames); + queued -= discarded; + if (audio->debug && discarded) + atomic_fetch_add_explicit(&audio->recordTrimmedFrames, + discarded, memory_order_relaxed); + } + + const uint64_t leadingSilenceFrames = + totalFrames > (uint64_t)queued ? totalFrames - queued : 0; + sendRecordPacketBatch(audio, sendCount, rateQ16, false, + leadingSilenceFrames, queued); + audio->recordNextPacketTime += + (uint64_t)sendCount * USB_AUDIO_RECORD_PACKET_INTERVAL_NS; } static void startISOStream(void * opaque, uint64_t id, struct usb_redir_start_iso_stream_header * request) { LG_USBAudio * audio = getAudio(opaque); + flushPlayback(audio); uint8_t result = usb_redir_stall; uint32_t feedbackPrefill = 0; @@ -893,7 +1282,8 @@ static void startISOStream(void * opaque, uint64_t id, if (!getLayout(audio->playbackAlt)) break; resetFeedbackRate(audio); - audio->feedbackStreaming = true; + atomic_store_explicit( + &audio->feedbackStreaming, true, memory_order_release); audio->feedbackDataPackets = 0; audio->feedbackPacketId = 0; feedbackPrefill = @@ -909,6 +1299,12 @@ static void startISOStream(void * opaque, uint64_t id, case USB_AUDIO_RECORD_DATA_ENDPOINT: if (audio->recordAlt != 1) break; + audio->recordBatchPackets = max(UINT32_C(1), + min((uint32_t)request->pkts_per_urb, + (uint32_t)USB_AUDIO_RECORD_MAX_BATCH)); + audio->recordSafetyPackets = max(UINT32_C(1), + min((uint32_t)request->pkts_per_urb * request->no_urbs, + (uint32_t)USB_AUDIO_RECORD_MAX_SAFETY_PACKETS)); result = usb_redir_success; startRecord(audio); break; @@ -924,6 +1320,7 @@ static void stopISOStream(void * opaque, uint64_t id, struct usb_redir_stop_iso_stream_header * request) { LG_USBAudio * audio = getAudio(opaque); + flushPlayback(audio); uint8_t result = usb_redir_stall; switch (request->endpoint) { @@ -1021,6 +1418,7 @@ static void controlPacket(void * opaque, uint64_t id, uint8_t * data, int dataLength) { LG_USBAudio * audio = getAudio(opaque); + flushPlayback(audio); uint8_t buffer[sizeof(l_configurationDescriptor)]; const uint8_t * response = NULL; size_t responseSize = 0; @@ -1151,7 +1549,8 @@ static void controlPacket(void * opaque, uint64_t id, control == USB_AUDIO_CONTROL_FREQUENCY) { writeLE32(buffer, playbackClock ? - audio->playbackSampleRate : audio->recordSampleRate); + audio->playbackSampleRate : atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed)); response = buffer; responseSize = 4; status = usb_redir_success; @@ -1220,6 +1619,7 @@ static void isoPacket(void * opaque, uint64_t id, if (packet->endpoint != USB_AUDIO_PLAYBACK_DATA_ENDPOINT || packet->status != usb_redir_success || dataLength < 0 || packet->length != dataLength || + (dataLength && !data) || !layout || dataLength > layoutPacketSize(layout) || dataLength % frameSize != 0) { @@ -1228,13 +1628,14 @@ static void isoPacket(void * opaque, uint64_t id, } else { - if (dataLength && audio->events && audio->events->playbackData) - audio->events->playbackData(audio->eventOpaque, data, - dataLength / frameSize); + queuePlaybackPacket( + audio, data, dataLength / frameSize, frameSize); - if (audio->feedbackStreaming && + if (atomic_load_explicit( + &audio->feedbackStreaming, memory_order_acquire) && ++audio->feedbackDataPackets == USB_AUDIO_FEEDBACK_INTERVAL) { + flushPlayback(audio); audio->feedbackDataPackets = 0; sendFeedbackPackets(audio, 1); } @@ -1287,9 +1688,54 @@ static void unplugDevice(void * opaque) resetDevice(opaque); } +static void reportRecordDebug(LG_USBAudio * audio) +{ + if (!audio->debug) + return; + + const uint64_t now = recordTime(); + if (!audio->recordNextDebugTime) + { + audio->recordNextDebugTime = + now + USB_AUDIO_RECORD_DEBUG_INTERVAL_NS; + return; + } + + if (now < audio->recordNextDebugTime) + return; + audio->recordNextDebugTime = now + USB_AUDIO_RECORD_DEBUG_INTERVAL_NS; + + const uint_fast64_t overflow = atomic_exchange_explicit( + &audio->recordOverflowFrames, 0, memory_order_relaxed); + const uint_fast64_t underflow = atomic_exchange_explicit( + &audio->recordUnderflowFrames, 0, memory_order_relaxed); + const uint_fast64_t trimmed = atomic_exchange_explicit( + &audio->recordTrimmedFrames, 0, memory_order_relaxed); + const uint_fast64_t late = atomic_exchange_explicit( + &audio->recordLatePackets, 0, memory_order_relaxed); + const uint_fast64_t maxLateness = atomic_exchange_explicit( + &audio->recordMaxLateness, 0, memory_order_relaxed); + const uint_fast64_t discontinuities = atomic_exchange_explicit( + &audio->recordDiscontinuities, 0, memory_order_relaxed); + if (!overflow && !underflow && !trimmed && !late && !discontinuities) + return; + + DEBUG_INFO( + "USB capture: overflow %" PRIuFAST64 + ", underflow %" PRIuFAST64 + ", trimmed %" PRIuFAST64 + " frames; late %" PRIuFAST64 + " packets, max %.3f ms, discontinuities %" PRIuFAST64, + overflow, underflow, trimmed, late, maxLateness / 1000000.0, + discontinuities); +} + static void processDevice(void * opaque) { - sendRecordPackets(opaque); + LG_USBAudio * audio = opaque; + flushPlayback(audio); + sendRecordPackets(audio); + reportRecordDebug(audio); } static const LG_USBRedirDeviceOps l_deviceOps = @@ -1300,29 +1746,53 @@ static const LG_USBRedirDeviceOps l_deviceOps = .unplug = unplugDevice, }; -LG_USBAudio * lgUsbAudio_create( - const LG_USBAudioEventOps * events, void * eventOpaque) +LG_USBAudio * lgUsbAudio_create(const LG_USBAudioEventOps * events, + void * eventOpaque, bool debug) { LG_USBAudio * audio = calloc(1, sizeof(*audio)); if (!audio) return NULL; - audio->events = events; - audio->eventOpaque = eventOpaque; - audio->playbackSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; - audio->recordSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; - LG_LOCK_INIT(audio->recordLock); - audio->recordBuffer = ringbuffer_new( + audio->events = events; + audio->eventOpaque = eventOpaque; + audio->debug = debug; + audio->playbackSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; + audio->recordBatchPackets = 1; + audio->recordSafetyPackets = 1; + audio->recordIdle = lgCreateEvent(true, 0); + if (!audio->recordIdle) + { + free(audio); + return NULL; + } + audio->recordBuffer = ringbuffer_new( USB_AUDIO_RECORD_QUEUE_FRAMES, USB_AUDIO_RECORD_FRAME_SIZE); if (!audio->recordBuffer) { + lgFreeEvent(audio->recordIdle); free(audio); return NULL; } atomic_init(&audio->feedbackValue, encodeFeedbackRate(audio->playbackSampleRate)); + atomic_init(&audio->feedbackStreaming, false); + atomic_init( + &audio->recordSampleRate, LG_USB_AUDIO_DEFAULT_SAMPLE_RATE); atomic_init(&audio->recordStreaming, false); + atomic_init(&audio->recordAccepting, false); + atomic_init(&audio->recordOverflowPending, false); + atomic_init(&audio->recordGeneration, 0); + atomic_init(&audio->recordInFlight, 0); + atomic_init(&audio->recordRateQ16, + (uint64_t)LG_USB_AUDIO_DEFAULT_SAMPLE_RATE * + USB_AUDIO_RECORD_RATE_SCALE); + atomic_init(&audio->recordOverflowFrames, 0); + atomic_init(&audio->recordUnderflowFrames, 0); + atomic_init(&audio->recordTrimmedFrames, 0); + atomic_init(&audio->recordLatePackets, 0); + atomic_init(&audio->recordMaxLateness, 0); + atomic_init(&audio->recordDiscontinuities, 0); return audio; } @@ -1335,89 +1805,101 @@ void lgUsbAudio_setFeedbackRate(LG_USBAudio * audio, double sampleRate) encodeFeedbackRate(sampleRate), memory_order_release); } -bool lgUsbAudio_recordData( - LG_USBAudio * audio, const void * data, size_t frames) +bool lgUsbAudio_feedbackActive(const LG_USBAudio * audio) { - if (!audio || frames > INT_MAX || (frames && !data) || - !atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire)) + return audio && atomic_load_explicit( + &audio->feedbackStreaming, memory_order_acquire); +} + +bool lgUsbAudio_recordData( + LG_USBAudio * audio, const void * data, size_t frames, + const LG_AudioClock * sourceClock) +{ + if (!audio || frames > INT_MAX || (frames && !data)) return false; - LG_LOCK(audio->recordLock); + const unsigned int generation = atomic_load_explicit( + &audio->recordGeneration, memory_order_seq_cst); if (!atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire)) + &audio->recordStreaming, memory_order_acquire) || + !atomic_load_explicit( + &audio->recordAccepting, memory_order_seq_cst)) + return false; + + atomic_fetch_add_explicit( + &audio->recordInFlight, 1, memory_order_seq_cst); + if (!atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire) || + !atomic_load_explicit( + &audio->recordAccepting, memory_order_seq_cst) || + atomic_load_explicit( + &audio->recordGeneration, memory_order_seq_cst) != generation) { - LG_UNLOCK(audio->recordLock); + const unsigned int previous = atomic_fetch_sub_explicit( + &audio->recordInFlight, 1, memory_order_seq_cst); + if (previous == 1 && !atomic_load_explicit( + &audio->recordAccepting, memory_order_seq_cst)) + lgSignalEvent(audio->recordIdle); return false; } const size_t receivedFrames = frames; - const uint64_t now = recordTime(); - /* Average complete capture batches over time so callback jitter does not - * become USB packet jitter. recordRateQ16 is frames/second in Q16. */ - const bool resetRate = - !audio->recordRateLastTime || - now - audio->recordRateLastTime > USB_AUDIO_RECORD_RATE_GAP_NS; - if (resetRate) - { - audio->recordRateQ16 = - (uint64_t)audio->recordSampleRate * USB_AUDIO_RECORD_RATE_SCALE; - audio->recordRateStartTime = now; - audio->recordRateFrames = 0; - } + const uint32_t sampleRate = atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed); + bool sourceContinuous = true; + if (sourceClock) + sourceContinuous = updateRecordSourceRate( + audio, sampleRate, sourceClock); else - audio->recordRateFrames += receivedFrames; - audio->recordRateLastTime = now; + updateRecordArrivalRate(audio, sampleRate, receivedFrames); - const uint64_t elapsed = now - audio->recordRateStartTime; - if (elapsed >= USB_AUDIO_RECORD_RATE_SETTLE_NS) + size_t overflowFrames = 0; + if (sourceContinuous) { - const double measured = - (double)audio->recordRateFrames * 1000000000.0 / elapsed; - if (measured >= audio->recordSampleRate * 0.995 && - measured <= audio->recordSampleRate * 1.005) + const int length = ringbuffer_getLength(audio->recordBuffer); + const uint8_t * input = data; + if (frames > (size_t)length) { - const uint64_t measuredQ16 = (uint64_t)( - measured * USB_AUDIO_RECORD_RATE_SCALE + 0.5); - audio->recordRateQ16 = - (audio->recordRateQ16 * 3 + measuredQ16) / 4; + input += (frames - length) * USB_AUDIO_RECORD_FRAME_SIZE; + frames = length; } - audio->recordRateStartTime = now; - audio->recordRateFrames = 0; + const int appended = ringbuffer_append( + audio->recordBuffer, input, (int)frames); + overflowFrames = receivedFrames - appended; } - const int length = ringbuffer_getLength(audio->recordBuffer); - const uint8_t * input = data; - bool dropped = false; - if (frames > (size_t)length) + if (!sourceContinuous || overflowFrames) { - input += (frames - length) * USB_AUDIO_RECORD_FRAME_SIZE; - frames = length; - ringbuffer_reset(audio->recordBuffer); - dropped = true; - } - else - { - const int overflow = - ringbuffer_getCount(audio->recordBuffer) + (int)frames - length; - if (overflow > 0) - { - ringbuffer_consume(audio->recordBuffer, NULL, overflow); - dropped = true; - } + atomic_store_explicit( + &audio->recordOverflowPending, true, memory_order_release); + if (audio->debug) + atomic_fetch_add_explicit(&audio->recordOverflowFrames, + overflowFrames, memory_order_relaxed); } - const int appended = ringbuffer_append( - audio->recordBuffer, input, (int)frames); - LG_UNLOCK(audio->recordLock); - return !dropped && appended == (int)frames; + const unsigned int previous = atomic_fetch_sub_explicit( + &audio->recordInFlight, 1, memory_order_seq_cst); + if (previous == 1 && !atomic_load_explicit( + &audio->recordAccepting, memory_order_seq_cst)) + lgSignalEvent(audio->recordIdle); + return !sourceContinuous || !overflowFrames; } -bool lgUsbAudio_recording(const LG_USBAudio * audio) +uint64_t lgUsbAudio_processDelayNs(const LG_USBAudio * audio) { - return audio && atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire); + if (!audio || !atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire)) + return UINT64_MAX; + + if (audio->recordRefillPackets) + return 0; + + const uint64_t now = recordTime(); + const uint64_t deadline = audio->recordNextPacketTime + + (uint64_t)(audio->recordBatchPackets - 1) * + USB_AUDIO_RECORD_PACKET_INTERVAL_NS; + return deadline > now ? deadline - now : 0; } void lgUsbAudio_destroy(LG_USBAudio * audio) @@ -1425,8 +1907,10 @@ void lgUsbAudio_destroy(LG_USBAudio * audio) if (!audio) return; + recordDisableInput(audio); + recordWaitForInput(audio); ringbuffer_free(&audio->recordBuffer); - LG_LOCK_FREE(audio->recordLock); + lgFreeEvent(audio->recordIdle); free(audio); } diff --git a/client/src/usb_audio.h b/client/src/usb_audio.h index 71d435b7..dc852bfd 100644 --- a/client/src/usb_audio.h +++ b/client/src/usb_audio.h @@ -21,6 +21,7 @@ #ifndef _H_LG_CLIENT_USB_AUDIO_ #define _H_LG_CLIENT_USB_AUDIO_ +#include "interface/audio.h" #include "usbredir.h" #include @@ -49,21 +50,25 @@ typedef struct LG_USBAudioEventOps LG_USBAudioEventOps; LG_USBAudio * lgUsbAudio_create( - const LG_USBAudioEventOps * events, void * eventOpaque); + const LG_USBAudioEventOps * events, void * eventOpaque, bool debug); /* Destroy the LG_USBRedir using this device before destroying the device. */ void lgUsbAudio_destroy(LG_USBAudio * audio); /* Publish the requested source rate without touching usbredir from the audio * feedback thread. */ void lgUsbAudio_setFeedbackRate(LG_USBAudio * audio, double sampleRate); +bool lgUsbAudio_feedbackActive(const LG_USBAudio * audio); -/* Queue interleaved packed signed 24-bit microphone frames. This may be - * called from the audio recording thread. */ +/* Queue interleaved packed signed 24-bit microphone frames. sourceClock is + * borrowed for the call and identifies the first frame when present. This may + * be called from the audio recording thread. */ bool lgUsbAudio_recordData( - LG_USBAudio * audio, const void * data, size_t frames); + LG_USBAudio * audio, const void * data, size_t frames, + const LG_AudioClock * sourceClock); -/* This must be queried on the PureSpice processing thread. */ -bool lgUsbAudio_recording(const LG_USBAudio * audio); +/* Return the time until ISO-IN processing is needed. This must be queried on + * the PureSpice processing thread. */ +uint64_t lgUsbAudio_processDelayNs(const LG_USBAudio * audio); const LG_USBRedirDeviceOps * lgUsbAudio_deviceOps(void); diff --git a/client/src/usbredir.c b/client/src/usbredir.c index 7925a4a3..9734f6dc 100644 --- a/client/src/usbredir.c +++ b/client/src/usbredir.c @@ -21,6 +21,7 @@ #include "usbredir.h" #include "common/debug.h" +#include "common/time.h" #include @@ -29,6 +30,9 @@ #include #define USB_REDIR_CHANNEL_COUNT 256 +#define USB_REDIR_DISCONNECT_TIMEOUT_NS INT64_C(500000000) +#define USB_REDIR_RECONNECT_DELAY_NS INT64_C(250000000) +#define USB_REDIR_MAX_OUTPUT_BYTES UINT64_C(1048576) struct LG_USBRedir { @@ -49,6 +53,8 @@ struct LG_USBRedir atomic_bool available; bool plugged; bool disconnectPending; + int64_t disconnectDeadline; + int64_t reconnectDeadline; }; static void setAvailable(LG_USBRedir * usbredir, bool available) @@ -123,6 +129,7 @@ static void deviceDisconnectAck(void * opaque) { LG_USBRedir * usbredir = opaque; usbredir->disconnectPending = false; + usbredir->disconnectDeadline = 0; } static void unplugDevice(LG_USBRedir * usbredir) @@ -139,6 +146,7 @@ static void destroyParser(LG_USBRedir * usbredir) setAvailable(usbredir, false); unplugDevice(usbredir); usbredir->disconnectPending = false; + usbredir->disconnectDeadline = 0; if (!usbredir->parser) return; @@ -149,8 +157,20 @@ static void destroyParser(LG_USBRedir * usbredir) static bool flushUSBRedir(LG_USBRedir * usbredir) { - return !usbredir->parser || - usbredirparser_do_write(usbredir->parser) == 0; + if (!usbredir->parser) + return true; + + if (usbredirparser_do_write(usbredir->parser) != 0) + return false; + + const uint64_t buffered = + usbredirparser_get_bufferered_output_size(usbredir->parser); + if (buffered <= USB_REDIR_MAX_OUTPUT_BYTES) + return true; + + DEBUG_ERROR("USB redirection output queue exceeded %u bytes", + (unsigned int)USB_REDIR_MAX_OUTPUT_BYTES); + return false; } static bool connectChannel(LG_USBRedir * usbredir, @@ -166,10 +186,10 @@ static bool connectChannel(LG_USBRedir * usbredir, return false; } -static void selectChannel(LG_USBRedir * usbredir) +static bool selectChannel(LG_USBRedir * usbredir) { if (usbredir->channel) - return; + return true; for (unsigned int i = 0; i < USB_REDIR_CHANNEL_COUNT; ++i) { @@ -178,8 +198,28 @@ static void selectChannel(LG_USBRedir * usbredir) continue; if (connectChannel(usbredir, channel)) - return; + { + usbredir->reconnectDeadline = 0; + return true; + } } + + usbredir->reconnectDeadline = + (int64_t)nanotime() + USB_REDIR_RECONNECT_DELAY_NS; + return false; +} + +static void resetChannel(LG_USBRedir * usbredir) +{ + PSUSBRedirChannel * channel = usbredir->channel; + usbredir->channel = NULL; + destroyParser(usbredir); + + if (channel && purespice_usbRedirConnected(channel)) + purespice_usbRedirDisconnect(channel); + + usbredir->reconnectDeadline = + (int64_t)nanotime() + USB_REDIR_RECONNECT_DELAY_NS; } static bool createParser(LG_USBRedir * usbredir) @@ -268,14 +308,37 @@ bool lgUsbRedir_disconnectPending(const LG_USBRedir * usbredir) return usbredir->disconnectPending; } -bool lgUsbRedir_process(LG_USBRedir * usbredir) +static bool processUSBRedir(LG_USBRedir * usbredir, bool recover) { + const int64_t now = (int64_t)nanotime(); + if (!usbredir->channel && + now >= usbredir->reconnectDeadline) + selectChannel(usbredir); + if (!usbredir->parser || !atomic_load_explicit(&usbredir->available, memory_order_acquire)) - return flushUSBRedir(usbredir); + { + const bool result = flushUSBRedir(usbredir); + if (!result && recover) + resetChannel(usbredir); + return result; + } if (usbredir->disconnectPending) - return flushUSBRedir(usbredir); + { + if (usbredir->disconnectDeadline && + now >= usbredir->disconnectDeadline) + { + DEBUG_WARN("USB redirection disconnect acknowledgement timed out"); + resetChannel(usbredir); + return true; + } + + const bool result = flushUSBRedir(usbredir); + if (!result && recover) + resetChannel(usbredir); + return result; + } const bool desired = atomic_load_explicit(&usbredir->desiredPlugged, memory_order_acquire); @@ -292,6 +355,8 @@ bool lgUsbRedir_process(LG_USBRedir * usbredir) unplugDevice(usbredir); usbredir->disconnectPending = usbredirparser_peer_has_cap( usbredir->parser, usb_redir_cap_device_disconnect_ack); + usbredir->disconnectDeadline = usbredir->disconnectPending ? + now + USB_REDIR_DISCONNECT_TIMEOUT_NS : 0; usbredirparser_send_device_disconnect(usbredir->parser); } } @@ -299,7 +364,15 @@ bool lgUsbRedir_process(LG_USBRedir * usbredir) if (usbredir->plugged && usbredir->deviceOps->process) usbredir->deviceOps->process(usbredir->deviceOpaque); - return flushUSBRedir(usbredir); + const bool result = flushUSBRedir(usbredir); + if (!result && recover) + resetChannel(usbredir); + return result; +} + +bool lgUsbRedir_process(LG_USBRedir * usbredir) +{ + return processUSBRedir(usbredir, true); } void lgUsbRedir_state(PSUSBRedirChannel * channel, @@ -328,7 +401,13 @@ void lgUsbRedir_state(PSUSBRedirChannel * channel, case PS_USB_REDIR_DISCONNECTED: if (channel == usbredir->channel) + { destroyParser(usbredir); + usbredir->channel = NULL; + if (purespice_usbRedirAvailable(channel)) + usbredir->reconnectDeadline = + (int64_t)nanotime() + USB_REDIR_RECONNECT_DELAY_NS; + } break; case PS_USB_REDIR_UNAVAILABLE: @@ -375,7 +454,9 @@ bool lgUsbRedir_data(PSUSBRedirChannel * channel, return false; } - return lgUsbRedir_process(usbredir); + /* Returning false lets PureSpice disconnect after this callback unwinds; + * resetting it here would destroy the channel during its own dispatch. */ + return processUSBRedir(usbredir, false); } void * lgUsbRedir_device(void * parserOpaque)