diff --git a/client/audiodevs/PipeWire/pipewire.c b/client/audiodevs/PipeWire/pipewire.c index ceaa895f..62d77367 100644 --- a/client/audiodevs/PipeWire/pipewire.c +++ b/client/audiodevs/PipeWire/pipewire.c @@ -21,6 +21,7 @@ #include "interface/audiodev.h" #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -38,29 +40,54 @@ #include "common/option.h" #include "common/ringbuffer.h" #include "common/thread.h" +#include "common/time.h" typedef enum { STREAM_STATE_INACTIVE, - STREAM_STATE_ACTIVE, - STREAM_STATE_DRAINING + STREAM_STATE_ACTIVE } StreamState; +typedef enum +{ + RECORD_LATEST_FREE, + RECORD_LATEST_WRITING, + RECORD_LATEST_READY, + RECORD_LATEST_READING +} +RecordLatestState; + +typedef struct +{ + int frames; + bool clockValid; + LG_AudioClock clock; +} +RecordBlock; + +#define RECORD_LATEST_SLOTS 2 +#define RECORD_CLOCK_TOLERANCE_FRAMES 2.0 +#define RECORD_ERROR_REPORT_INTERVAL_NS INT64_C(5000000000) +#define PIPEWIRE_CONNECT_TIMEOUT_NS INT64_C(5000000000) + struct PipeWire { struct pw_loop * loop; struct pw_context * context; + struct pw_core * core; struct pw_thread_loop * thread; struct { struct pw_stream * stream; - struct spa_io_rate_match * rateMatch; - _Atomic(int64_t) presentationDeadline; + _Atomic(struct spa_io_rate_match *) rateMatch; + _Atomic(int64_t) latencyNs; atomic_bool latencyUpdateRequested; atomic_uint bufferErrors; + atomic_uint pullErrors; atomic_uint timingErrors; + atomic_int streamError; #if PW_CHECK_VERSION(1, 4, 0) double appliedResampleRatio; atomic_int resampleError; @@ -71,32 +98,63 @@ struct PipeWire LG_AudioFormat format; int stride; LG_AudioPullFn pullFn; + LG_AudioFailureFn failureFn; + uint32_t failureCookie; int maxPeriodFrames; int startFrames; - StreamState state; + atomic_uint state; } playback; struct { struct pw_stream * stream; + enum pw_stream_state connectionState; LG_AudioFormat format; int stride; LG_AudioPushFn pushFn; + LG_AudioFailureFn failureFn; + uint32_t failureCookie; RingBuffer sendQueue; + RingBuffer sendBlocks; uint8_t * sendBuffer; int sendBufferFrames; + int maxQueuedFrames; sem_t sendWake; bool sendWakeInitialized; LGThread * sendThread; atomic_bool sendStop; + atomic_bool sendWakePending; + atomic_bool latestMode; + atomic_uint_fast64_t latestSerial; + uint8_t * latestBuffer; + struct + { + atomic_uint state; + atomic_uint_fast64_t serial; + int frames; + bool clockValid; + LG_AudioClock clock; + } + latest[RECORD_LATEST_SLOTS]; atomic_uint bufferErrors; atomic_uint droppedFrames; + atomic_uint rejectedBatches; + atomic_uint rejectedFrames; atomic_uint signalErrors; + atomic_int recycleError; + uint64_t clockPosition; + uint64_t clockLastPosition; + uint64_t clockLastTicks; + int64_t clockLastTime; + struct spa_fraction clockLastRate; + bool clockLastValid; + bool clockLastTimingValid; + bool clockDiscontinuityPending; bool active; } record; @@ -104,6 +162,17 @@ struct PipeWire static struct PipeWire pw = {0}; +static inline void pipewire_storeError(atomic_int * target, int result) +{ + if (result >= 0) + return; + + int expected = 0; + atomic_compare_exchange_strong_explicit( + target, &expected, result, + memory_order_relaxed, memory_order_relaxed); +} + static bool pipewire_audioFormatEqual(const LG_AudioFormat * a, const LG_AudioFormat * b) { @@ -198,8 +267,12 @@ static void pipewire_reportPlaybackErrors(void) { const unsigned int bufferErrors = atomic_exchange_explicit( &pw.playback.bufferErrors, 0, memory_order_relaxed); + const unsigned int pullErrors = atomic_exchange_explicit( + &pw.playback.pullErrors, 0, memory_order_relaxed); const unsigned int timingErrors = atomic_exchange_explicit( &pw.playback.timingErrors, 0, memory_order_relaxed); + const int streamError = atomic_exchange_explicit( + &pw.playback.streamError, 0, memory_order_relaxed); #if PW_CHECK_VERSION(1, 4, 0) const int resampleError = atomic_exchange_explicit( &pw.playback.resampleError, 0, memory_order_relaxed); @@ -208,9 +281,15 @@ static void pipewire_reportPlaybackErrors(void) if (bufferErrors) DEBUG_WARN("PipeWire playback ran out of buffers %u time(s)", bufferErrors); + if (pullErrors) + DEBUG_WARN("PipeWire playback returned an invalid frame count %u time(s)", + pullErrors); if (timingErrors) DEBUG_WARN("PipeWire playback timing query failed %u time(s)", timingErrors); + if (streamError) + DEBUG_WARN("PipeWire playback stream operation failed: %s", + spa_strerror(streamError)); #if PW_CHECK_VERSION(1, 4, 0) if (resampleError) DEBUG_WARN("PipeWire resampler rate update failed: %s", @@ -224,25 +303,49 @@ static void pipewire_reportRecordErrors(void) &pw.record.bufferErrors, 0, memory_order_relaxed); const unsigned int droppedFrames = atomic_exchange_explicit( &pw.record.droppedFrames, 0, memory_order_relaxed); + const unsigned int rejectedBatches = atomic_exchange_explicit( + &pw.record.rejectedBatches, 0, memory_order_relaxed); + const unsigned int rejectedFrames = atomic_exchange_explicit( + &pw.record.rejectedFrames, 0, memory_order_relaxed); const unsigned int signalErrors = atomic_exchange_explicit( &pw.record.signalErrors, 0, memory_order_relaxed); + const int recycleError = atomic_exchange_explicit( + &pw.record.recycleError, 0, memory_order_relaxed); if (bufferErrors) DEBUG_WARN("PipeWire recording encountered %u buffer error(s)", bufferErrors); if (droppedFrames) DEBUG_WARN("PipeWire recording dropped %u frame(s)", droppedFrames); + if (rejectedFrames) + DEBUG_WARN("PipeWire recording provider rejected %u frame(s) " + "in %u batch(es)", rejectedFrames, rejectedBatches); if (signalErrors) DEBUG_WARN("PipeWire recording worker notification failed %u time(s)", signalErrors); + if (recycleError) + DEBUG_WARN("PipeWire recording buffer recycle failed: %s", + spa_strerror(recycleError)); } +static void pipewire_reportRecordErrorsDue(uint64_t * nextReport) +{ + const uint64_t now = nanotime(); + if (now < *nextReport) + return; + + pipewire_reportRecordErrors(); + *nextReport = now + RECORD_ERROR_REPORT_INTERVAL_NS; +} + +#if !PW_CHECK_VERSION(1, 1, 0) static int64_t pipewire_monotonicTime(void) { struct timespec time; clock_gettime(CLOCK_MONOTONIC, &time); return SPA_TIMESPEC_TO_NSEC(&time); } +#endif static inline void pipewire_updatePlaybackLatency(void) { @@ -266,17 +369,27 @@ static inline void pipewire_updatePlaybackLatency(void) if (time.rate.num == 0 || time.rate.denom == 0) return; -#if PW_CHECK_VERSION(0, 3, 50) - const double latencyTicks = - time.delay + (double)time.queued + time.buffered; +#if PW_CHECK_VERSION(1, 1, 0) + const int64_t now = (int64_t)pw_stream_get_nsec(pw.playback.stream); #else - const double latencyTicks = - time.delay + (double)time.queued / pw.playback.stride; + const int64_t now = pipewire_monotonicTime(); #endif - const int64_t latencyNs = llrint(max(0.0, latencyTicks) * - time.rate.num * SPA_NSEC_PER_SEC / time.rate.denom); - atomic_store_explicit(&pw.playback.presentationDeadline, - time.now + latencyNs, memory_order_release); + const int64_t elapsedNs = max(INT64_C(0), now - time.now); + const double graphLatencyNs = + (double)time.delay * time.rate.num * + SPA_NSEC_PER_SEC / time.rate.denom - elapsedNs; +#if PW_CHECK_VERSION(0, 3, 50) + const double streamFrames = time.queued + time.buffered; +#else + const double streamFrames = + (double)time.queued / pw.playback.stride; +#endif + const double streamLatencyNs = streamFrames * SPA_NSEC_PER_SEC / + pw.playback.format.sampleRate; + const int64_t latencyNs = + llrint(max(0.0, graphLatencyNs + streamLatencyNs)); + atomic_store_explicit( + &pw.playback.latencyNs, latencyNs, memory_order_release); } #if PW_CHECK_VERSION(1, 4, 0) @@ -302,24 +415,20 @@ static bool pipewire_playbackSetRate(double * ratio) return true; } -static bool pipewire_configurePlaybackResampler(bool enable) +static bool pipewire_configurePlaybackResampler( + bool enable, bool * reusable) { - pw.playback.resamplerEnabled = false; - pw.playback.appliedResampleRatio = 0.0; - - if (!enable) - { - pw_stream_set_rate(pw.playback.stream, 0.0); - return false; - } - - const int result = pw_stream_set_rate(pw.playback.stream, 1.0); + const bool wasEnabled = pw.playback.resamplerEnabled; + const int result = pw_stream_set_rate( + pw.playback.stream, enable ? 1.0 : 0.0); + if (reusable) + *reusable = result >= 0 || !wasEnabled; if (result < 0) return false; - pw.playback.resamplerEnabled = true; - pw.playback.appliedResampleRatio = 1.0; - return true; + pw.playback.resamplerEnabled = enable; + pw.playback.appliedResampleRatio = enable ? 1.0 : 0.0; + return enable; } #endif @@ -329,7 +438,10 @@ static void pipewire_onPlaybackIoChanged(void * userdata, uint32_t id, switch (id) { case SPA_IO_RateMatch: - pw.playback.rateMatch = data; + atomic_store_explicit(&pw.playback.rateMatch, + data && size >= offsetof(struct spa_io_rate_match, size) + + sizeof(((struct spa_io_rate_match *)0)->size) ? data : NULL, + memory_order_release); break; } } @@ -338,10 +450,18 @@ static void pipewire_onPlaybackStateChanged(void * userdata, enum pw_stream_state old, enum pw_stream_state state, const char * error) { + (void)error; pw.playback.connectionState = state; - if (state == PW_STREAM_STATE_ERROR) - DEBUG_ERROR("PipeWire playback stream failed: %s", - error ? error : "unknown error"); + if ((state == PW_STREAM_STATE_ERROR || + state == PW_STREAM_STATE_UNCONNECTED) && + pw.playback.failureFn) + { + LG_AudioFailureFn failureFn = pw.playback.failureFn; + const uint32_t failureCookie = pw.playback.failureCookie; + pw.playback.failureFn = NULL; + pw.playback.failureCookie = 0; + failureFn(failureCookie); + } pw_thread_loop_signal(pw.thread, false); } @@ -359,34 +479,48 @@ static void pipewire_onPlaybackProcess(void * userdata) struct spa_buffer * sbuf = pbuf->buffer; uint8_t * dst; - if (sbuf->n_datas == 0 || !sbuf->datas[0].chunk || + if (!sbuf || sbuf->n_datas == 0 || !sbuf->datas || + !sbuf->datas[0].chunk || !(dst = sbuf->datas[0].data)) { #if PW_CHECK_VERSION(1, 4, 0) - pw_stream_return_buffer(pw.playback.stream, pbuf); + const int recycleResult = + pw_stream_return_buffer(pw.playback.stream, pbuf); #else - pw_stream_queue_buffer(pw.playback.stream, pbuf); + const int recycleResult = + pw_stream_queue_buffer(pw.playback.stream, pbuf); #endif + pipewire_storeError(&pw.playback.streamError, recycleResult); return; } int frames = sbuf->datas[0].maxsize / pw.playback.stride; - if (pw.playback.rateMatch && pw.playback.rateMatch->size > 0) - frames = min(frames, pw.playback.rateMatch->size); + struct spa_io_rate_match * rateMatch = atomic_load_explicit( + &pw.playback.rateMatch, memory_order_acquire); + if (rateMatch && rateMatch->size > 0) + frames = min(frames, rateMatch->size); #if PW_CHECK_VERSION(0, 3, 50) else if (pbuf->requested > 0) frames = min(frames, pbuf->requested); #endif - frames = pw.playback.pullFn(dst, frames); + const int requestedFrames = frames; + frames = pw.playback.pullFn(dst, requestedFrames); + if (frames < 0 || frames > requestedFrames) + { + atomic_fetch_add_explicit( + &pw.playback.pullErrors, 1, memory_order_relaxed); + frames = clamp(frames, 0, requestedFrames); + } + pipewire_updatePlaybackLatency(); if (!frames) { pbuf->size = 0; sbuf->datas[0].chunk->offset = 0; sbuf->datas[0].chunk->stride = pw.playback.stride; sbuf->datas[0].chunk->size = 0; - pw_stream_queue_buffer(pw.playback.stream, pbuf); - pipewire_updatePlaybackLatency(); + pipewire_storeError(&pw.playback.streamError, + pw_stream_queue_buffer(pw.playback.stream, pbuf)); return; } @@ -395,14 +529,8 @@ static void pipewire_onPlaybackProcess(void * userdata) sbuf->datas[0].chunk->stride = pw.playback.stride; sbuf->datas[0].chunk->size = frames * pw.playback.stride; - pw_stream_queue_buffer(pw.playback.stream, pbuf); - pipewire_updatePlaybackLatency(); -} - -static void pipewire_onPlaybackDrained(void * userdata) -{ - pw_stream_set_active(pw.playback.stream, false); - pw.playback.state = STREAM_STATE_INACTIVE; + pipewire_storeError(&pw.playback.streamError, + pw_stream_queue_buffer(pw.playback.stream, pbuf)); } static struct Option pipewire_options[] = @@ -428,11 +556,35 @@ static void pipewire_earlyInit(void) option_register(pipewire_options); } +static bool pipewire_waitForStream(enum pw_stream_state * state) +{ + int result = 0; +#if PW_CHECK_VERSION(0, 3, 7) + struct timespec deadline; + result = pw_thread_loop_get_time( + pw.thread, &deadline, PIPEWIRE_CONNECT_TIMEOUT_NS); + if (result < 0) + return false; + + while (*state == PW_STREAM_STATE_CONNECTING && result >= 0) + result = pw_thread_loop_timed_wait_full(pw.thread, &deadline); +#else + while (*state == PW_STREAM_STATE_CONNECTING && result >= 0) + result = pw_thread_loop_timed_wait(pw.thread, 5); +#endif + return result >= 0; +} + static bool pipewire_init(void) { pw_init(NULL, NULL); pw.loop = pw_loop_new(NULL); + if (!pw.loop) + { + DEBUG_ERROR("Failed to create a PipeWire loop"); + goto err; + } #if PW_CHECK_VERSION(1, 3, 81) pw.context = pw_context_new( pw.loop, @@ -455,8 +607,8 @@ static bool pipewire_init(void) } /* this is just to test for PipeWire availabillity */ - struct pw_core * core = pw_context_connect(pw.context, NULL, 0); - if (!core) + pw.core = pw_context_connect(pw.context, NULL, 0); + if (!pw.core) goto err_context; /* PipeWire is available so create the loop thread and start it */ @@ -464,17 +616,32 @@ static bool pipewire_init(void) if (!pw.thread) { DEBUG_ERROR("Failed to create the thread loop"); - goto err_context; + goto err_core; } - pw_thread_loop_start(pw.thread); + const int result = pw_thread_loop_start(pw.thread); + if (result < 0) + { + DEBUG_ERROR("Failed to start the PipeWire thread loop: %s", + spa_strerror(result)); + pw_thread_loop_destroy(pw.thread); + pw.thread = NULL; + goto err_core; + } return true; +err_core: + pw_core_disconnect(pw.core); + pw.core = NULL; + err_context: pw_context_destroy(pw.context); + pw.context = NULL; err: - pw_loop_destroy(pw.loop); + if (pw.loop) + pw_loop_destroy(pw.loop); + pw.loop = NULL; pw_deinit(); return false; } @@ -488,12 +655,17 @@ static void pipewire_playbackStopStream(void) } pw_thread_loop_lock(pw.thread); - pw_stream_destroy(pw.playback.stream); - pw.playback.stream = NULL; - pw.playback.rateMatch = NULL; - pw.playback.resamplerEnabled = false; atomic_store_explicit( - &pw.playback.presentationDeadline, 0, memory_order_release); + &pw.playback.rateMatch, NULL, memory_order_release); + pw.playback.failureFn = NULL; + pw.playback.failureCookie = 0; + pw_stream_destroy(pw.playback.stream); + pw.playback.stream = NULL; + pw.playback.resamplerEnabled = false; + atomic_store_explicit(&pw.playback.state, + STREAM_STATE_INACTIVE, memory_order_release); + atomic_store_explicit( + &pw.playback.latencyNs, 0, memory_order_release); atomic_store_explicit( &pw.playback.latencyUpdateRequested, false, memory_order_relaxed); pw_thread_loop_unlock(pw.thread); @@ -524,23 +696,36 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, .version = PW_VERSION_STREAM_EVENTS, .state_changed = pipewire_onPlaybackStateChanged, .io_changed = pipewire_onPlaybackIoChanged, - .process = pipewire_onPlaybackProcess, - .drained = pipewire_onPlaybackDrained + .process = pipewire_onPlaybackProcess }; + bool reuse = false; + pw_thread_loop_lock(pw.thread); if (pw.playback.stream && + pw.playback.connectionState == PW_STREAM_STATE_PAUSED && + atomic_load_explicit( + &pw.playback.state, memory_order_acquire) == STREAM_STATE_INACTIVE && pipewire_audioFormatEqual(&pw.playback.format, format)) { + pw.playback.pullFn = pullFn; + bool resamplerReusable = true; #if PW_CHECK_VERSION(1, 4, 0) atomic_store_explicit( &pw.playback.resampleError, 0, memory_order_relaxed); *resamplerEnabled = - pipewire_configurePlaybackResampler(requestResampler); + pipewire_configurePlaybackResampler( + requestResampler, &resamplerReusable); #endif - *maxPeriodFrames = pw.playback.maxPeriodFrames; - *startFrames = pw.playback.startFrames; - return true; + if (resamplerReusable) + { + *maxPeriodFrames = pw.playback.maxPeriodFrames; + *startFrames = pw.playback.startFrames; + reuse = true; + } } + pw_thread_loop_unlock(pw.thread); + if (reuse) + return true; pipewire_playbackStopStream(); @@ -553,6 +738,8 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, pw.playback.pullFn = pullFn; pw_thread_loop_lock(pw.thread); + atomic_store_explicit( + &pw.playback.rateMatch, NULL, memory_order_release); struct pw_properties * props = pw_properties_new( @@ -602,17 +789,19 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, // is using the audio device, but we can treat this value as a maximum const struct pw_properties * properties = pw_stream_get_properties(pw.playback.stream); - const char *actualNodeLatency = properties ? + const char * actualNodeLatency = properties ? pw_properties_get(properties, PW_KEY_NODE_LATENCY) : NULL; unsigned num, denom; + uint64_t actualPeriodFrames = 0; if (!actualNodeLatency || - sscanf(actualNodeLatency, "%u/%u", &num, &denom) != 2 || - num == 0 || num > INT_MAX || denom != sampleRate) + sscanf(actualNodeLatency, "%u/%u", &num, &denom) != 2 || + num == 0 || denom == 0 || + (actualPeriodFrames = + ((uint64_t)num * sampleRate + denom - 1) / denom) > INT_MAX) { DEBUG_WARN( - "PIPEWIRE_LATENCY value '%s' is invalid or does not match stream sample " - "rate; using %d/%d", + "PIPEWIRE_LATENCY value '%s' is invalid; using %d/%d", actualNodeLatency ? actualNodeLatency : "(unset)", requestedPeriodFrames, sampleRate); @@ -625,11 +814,12 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, pw.playback.maxPeriodFrames = requestedPeriodFrames; } else - pw.playback.maxPeriodFrames = num; + pw.playback.maxPeriodFrames = (int)actualPeriodFrames; // If the previous quantum size was very small, PipeWire can request two full // periods almost immediately at the start of playback - pw.playback.startFrames = pw.playback.maxPeriodFrames * 2; + pw.playback.startFrames = (int)min( + (int64_t)pw.playback.maxPeriodFrames * 2, (int64_t)INT_MAX); *maxPeriodFrames = pw.playback.maxPeriodFrames; *startFrames = pw.playback.startFrames; @@ -653,40 +843,46 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, if (result < 0) { DEBUG_ERROR("Failed to connect playback stream: %s", spa_strerror(result)); + atomic_store_explicit( + &pw.playback.rateMatch, NULL, memory_order_release); pw_stream_destroy(pw.playback.stream); pw.playback.stream = NULL; - pw.playback.rateMatch = NULL; pw_thread_loop_unlock(pw.thread); return false; } - while (pw.playback.connectionState == PW_STREAM_STATE_CONNECTING) - pw_thread_loop_wait(pw.thread); - - if (pw.playback.connectionState != PW_STREAM_STATE_PAUSED) + if (!pipewire_waitForStream(&pw.playback.connectionState) || + pw.playback.connectionState != PW_STREAM_STATE_PAUSED) { DEBUG_ERROR("PipeWire playback stream did not become ready"); + atomic_store_explicit( + &pw.playback.rateMatch, NULL, memory_order_release); pw_stream_destroy(pw.playback.stream); pw.playback.stream = NULL; - pw.playback.rateMatch = NULL; pw_thread_loop_unlock(pw.thread); return false; } - pw.playback.state = STREAM_STATE_INACTIVE; + atomic_store_explicit(&pw.playback.state, + STREAM_STATE_INACTIVE, memory_order_release); atomic_store_explicit( - &pw.playback.presentationDeadline, 0, memory_order_release); + &pw.playback.latencyNs, 0, memory_order_release); atomic_store_explicit( &pw.playback.latencyUpdateRequested, false, memory_order_relaxed); atomic_store_explicit( &pw.playback.bufferErrors, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.playback.pullErrors, 0, memory_order_relaxed); atomic_store_explicit( &pw.playback.timingErrors, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.playback.streamError, 0, memory_order_relaxed); #if PW_CHECK_VERSION(1, 4, 0) atomic_store_explicit( &pw.playback.resampleError, 0, memory_order_relaxed); *resamplerEnabled = - pipewire_configurePlaybackResampler(requestResampler); + pipewire_configurePlaybackResampler( + requestResampler, NULL); #else (void)requestResampler; #endif @@ -694,113 +890,386 @@ static bool pipewire_playbackSetup(const LG_AudioFormat * format, return true; } -static void pipewire_playbackStart(void) +static int pipewire_playbackControl(bool active, + LG_AudioFailureFn failureFn, uint32_t failureCookie) { - if (!pw.playback.stream) - return; - + int error = 0; pw_thread_loop_lock(pw.thread); - if (pw.playback.state != STREAM_STATE_ACTIVE) + if (!active) { - switch (pw.playback.state) + pw.playback.failureFn = NULL; + pw.playback.failureCookie = 0; + } + + if (!pw.playback.stream) + error = active ? -ENODEV : 0; + else if (active) + { + if (pw.playback.connectionState == PW_STREAM_STATE_ERROR || + pw.playback.connectionState == PW_STREAM_STATE_UNCONNECTED) + error = -EPIPE; + else { - case STREAM_STATE_INACTIVE: - pw_stream_set_active(pw.playback.stream, true); - pw.playback.state = STREAM_STATE_ACTIVE; - break; + pw.playback.failureFn = failureFn; + pw.playback.failureCookie = failureCookie; + if (atomic_load_explicit( + &pw.playback.state, memory_order_acquire) != STREAM_STATE_ACTIVE) + { + /* A stopped stream must not replay data queued before reactivation. */ + error = pw_stream_flush(pw.playback.stream, false); + if (error >= 0) + error = pw_stream_set_active(pw.playback.stream, true); + } + if (error >= 0) + { + atomic_store_explicit(&pw.playback.state, + STREAM_STATE_ACTIVE, memory_order_release); + } + } - case STREAM_STATE_DRAINING: - // We are in the middle of draining the PipeWire buffers; we need to - // wait for this to complete before allowing the new playback to start - break; - - default: - DEBUG_UNREACHABLE(); + if (error < 0 || + pw.playback.connectionState == PW_STREAM_STATE_ERROR || + pw.playback.connectionState == PW_STREAM_STATE_UNCONNECTED) + { + pw.playback.failureFn = NULL; + pw.playback.failureCookie = 0; + if (error >= 0) + error = -EPIPE; } } + else if (atomic_load_explicit( + &pw.playback.state, memory_order_acquire) == STREAM_STATE_ACTIVE) + { + const int activeError = + pw_stream_set_active(pw.playback.stream, false); + const int flushError = + pw_stream_flush(pw.playback.stream, false); + + error = activeError < 0 ? activeError : flushError; + if (activeError >= 0) + atomic_store_explicit(&pw.playback.state, + STREAM_STATE_INACTIVE, memory_order_release); + } + pw_thread_loop_unlock(pw.thread); + if (error < 0) + { + int expected = 0; + atomic_compare_exchange_strong_explicit( + &pw.playback.streamError, &expected, error, + memory_order_relaxed, memory_order_relaxed); + } + return error; +} + +static bool pipewire_playbackStart( + LG_AudioFailureFn failureFn, uint32_t failureCookie) +{ + return pipewire_playbackControl( + true, failureFn, failureCookie) >= 0; } static void pipewire_playbackStop(void) { - const bool inThread = pw_thread_loop_in_thread(pw.thread); - if (!inThread) - pw_thread_loop_lock(pw.thread); - - if (pw.playback.state != STREAM_STATE_ACTIVE) - goto done; - - if (inThread) + if (pipewire_playbackControl(false, NULL, 0) < 0) { - pw_stream_set_active(pw.playback.stream, false); - pw.playback.state = STREAM_STATE_INACTIVE; + pipewire_playbackStopStream(); + return; } - else - { - pw_stream_flush(pw.playback.stream, true); - pw.playback.state = STREAM_STATE_DRAINING; - } - -done: - if (!inThread) - pw_thread_loop_unlock(pw.thread); + atomic_store_explicit( + &pw.playback.latencyNs, 0, memory_order_release); + atomic_store_explicit( + &pw.playback.latencyUpdateRequested, false, memory_order_relaxed); } static void pipewire_playbackVolume(int channels, const uint16_t volume[]) { - if (channels != pw.playback.format.channelCount) + if (channels <= 0 || channels > LG_AUDIO_MAX_CHANNELS) return; float param[channels]; for(int i = 0; i < channels; ++i) - param[i] = 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787; + param[i] = max(0.0, + 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787); + int result = 0; pw_thread_loop_lock(pw.thread); - pw_stream_set_control(pw.playback.stream, SPA_PROP_channelVolumes, - channels, param, 0); + if (pw.playback.stream && + channels == pw.playback.format.channelCount) + result = pw_stream_set_control(pw.playback.stream, + SPA_PROP_channelVolumes, channels, param, 0); pw_thread_loop_unlock(pw.thread); + + if (result < 0) + DEBUG_WARN("Failed to set PipeWire playback volume: %s", + spa_strerror(result)); } static void pipewire_playbackMute(bool mute) { + int result = 0; pw_thread_loop_lock(pw.thread); - float val = mute ? 1.0f : 0.0f; - pw_stream_set_control(pw.playback.stream, SPA_PROP_mute, 1, &val, 0); + if (pw.playback.stream) + { + float val = mute ? 1.0f : 0.0f; + result = pw_stream_set_control( + pw.playback.stream, SPA_PROP_mute, 1, &val, 0); + } pw_thread_loop_unlock(pw.thread); + + if (result < 0) + DEBUG_WARN("Failed to set PipeWire playback mute: %s", + spa_strerror(result)); } static uint64_t pipewire_playbackLatency(void) { - pipewire_reportPlaybackErrors(); atomic_store_explicit( &pw.playback.latencyUpdateRequested, true, memory_order_release); - const int64_t deadline = atomic_load_explicit( - &pw.playback.presentationDeadline, memory_order_acquire); - if (deadline <= 0) + const int64_t latencyNs = atomic_load_explicit( + &pw.playback.latencyNs, memory_order_acquire); + if (latencyNs <= 0) return 0; - return max(INT64_C(0), deadline - pipewire_monotonicTime()) / 1000; + return latencyNs / 1000; +} + +static void pipewire_recordSignalSender(void) +{ + if (atomic_exchange_explicit( + &pw.record.sendWakePending, true, memory_order_acq_rel)) + return; + + if (sem_post(&pw.record.sendWake) < 0) + { + atomic_fetch_add_explicit( + &pw.record.signalErrors, 1, memory_order_relaxed); + if (errno != EOVERFLOW) + atomic_store_explicit( + &pw.record.sendWakePending, false, memory_order_release); + } +} + +static void pipewire_recordRejected(int frames) +{ + atomic_fetch_add_explicit( + &pw.record.rejectedBatches, 1, memory_order_relaxed); + atomic_fetch_add_explicit( + &pw.record.rejectedFrames, frames, memory_order_relaxed); +} + +static void pipewire_recordAdvanceClock( + LG_AudioClock * clock, int frames) +{ + const double rate = clock->rate > 0.0 ? + clock->rate : pw.record.format.sampleRate; + clock->position += frames; + if (rate > 0.0) + clock->time += (int64_t)llrint( + (double)frames * SPA_NSEC_PER_SEC / rate); + clock->discontinuity = false; +} + +static bool pipewire_recordClockJump( + double elapsedNs, double expectedNs) +{ + const double tolerance = + RECORD_CLOCK_TOLERANCE_FRAMES * SPA_NSEC_PER_SEC / + pw.record.format.sampleRate; + return elapsedNs <= 0 || + fabs(elapsedNs - expectedNs) > tolerance; +} + +static RecordBlock pipewire_recordMakeBlock( + struct pw_buffer * pbuf, const struct spa_buffer * sbuf, int frames) +{ + RecordBlock block = + { + .frames = frames + }; + struct pw_time timing = {0}; +#if PW_CHECK_VERSION(0, 3, 50) + const int timingResult = pw_stream_get_time_n( + pw.record.stream, &timing, sizeof(timing)); +#else + const int timingResult = pw_stream_get_time( + pw.record.stream, &timing); +#endif + const bool timingValid = timingResult >= 0 && + timing.rate.num != 0 && timing.rate.denom != 0; + + int64_t clockTime = 0; +#if PW_CHECK_VERSION(1, 0, 5) + if (pbuf->time) + clockTime = (int64_t)pbuf->time; +#endif + if (!clockTime && timingResult >= 0) + clockTime = timing.now; + + const uint64_t position = pw.record.clockPosition; + pw.record.clockPosition += frames; + + bool discontinuity = pw.record.clockDiscontinuityPending; + const struct spa_meta_header * header = spa_buffer_find_meta_data( + sbuf, SPA_META_Header, sizeof(*header)); + if (header && + (header->flags & (SPA_META_HEADER_FLAG_DISCONT | + SPA_META_HEADER_FLAG_CORRUPTED))) + discontinuity = true; + if (sbuf->datas[0].chunk->flags & SPA_CHUNK_FLAG_CORRUPTED) + discontinuity = true; + + block.clockValid = clockTime > 0; + if (block.clockValid) + { + const bool clockStable = pw.record.clockLastValid; + if (pw.record.clockLastValid) + { + const uint64_t elapsedFrames = + position - pw.record.clockLastPosition; + const double expectedNs = + (double)elapsedFrames * SPA_NSEC_PER_SEC / + pw.record.format.sampleRate; + if (pipewire_recordClockJump( + clockTime - pw.record.clockLastTime, expectedNs)) + discontinuity = true; + } + + if (timingValid && pw.record.clockLastTimingValid) + { + if (timing.rate.num != pw.record.clockLastRate.num || + timing.rate.denom != pw.record.clockLastRate.denom || + timing.ticks <= pw.record.clockLastTicks) + discontinuity = true; + else + { + const double elapsedTickNs = + (double)(timing.ticks - pw.record.clockLastTicks) * + timing.rate.num * SPA_NSEC_PER_SEC / timing.rate.denom; + const double expectedNs = + (double)(position - pw.record.clockLastPosition) * + SPA_NSEC_PER_SEC / pw.record.format.sampleRate; + if (pipewire_recordClockJump(elapsedTickNs, expectedNs)) + discontinuity = true; + } + } + + block.clock = (LG_AudioClock) + { + .position = position, + .time = clockTime, + .rate = 0.0, + .stable = clockStable && !discontinuity, + .discontinuity = discontinuity + }; + pw.record.clockDiscontinuityPending = false; + pw.record.clockLastPosition = position; + pw.record.clockLastTime = clockTime; + pw.record.clockLastValid = true; + } + else + { + pw.record.clockDiscontinuityPending |= discontinuity; + pw.record.clockLastValid = false; + } + + if (timingValid) + { + pw.record.clockLastTicks = timing.ticks; + pw.record.clockLastRate = timing.rate; + pw.record.clockLastTimingValid = block.clockValid; + } + else + pw.record.clockLastTimingValid = false; + + return block; +} + +static bool pipewire_recordLatestPending(void) +{ + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + if (atomic_load_explicit( + &pw.record.latest[i].state, memory_order_acquire) != + RECORD_LATEST_FREE) + return true; + + return false; +} + +static bool pipewire_recordSendLatest(void) +{ + int chosen = -1; + uint64_t chosenSerial = 0; + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + if (atomic_load_explicit( + &pw.record.latest[i].state, memory_order_acquire) != + RECORD_LATEST_READY) + continue; + + const uint64_t serial = atomic_load_explicit( + &pw.record.latest[i].serial, memory_order_relaxed); + if (chosen < 0 || serial > chosenSerial) + { + chosen = i; + chosenSerial = serial; + } + } + + if (chosen < 0) + return false; + + unsigned int expected = RECORD_LATEST_READY; + if (!atomic_compare_exchange_strong_explicit( + &pw.record.latest[chosen].state, &expected, + RECORD_LATEST_READING, memory_order_acq_rel, memory_order_acquire)) + return true; + + /* Only the newest overload block is useful. Drop older blocks before + * delivering it so recovery cannot build another backlog. */ + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + if (i == chosen || + atomic_load_explicit( + &pw.record.latest[i].serial, memory_order_relaxed) > chosenSerial) + continue; + + expected = RECORD_LATEST_READY; + if (atomic_compare_exchange_strong_explicit( + &pw.record.latest[i].state, &expected, + RECORD_LATEST_READING, + memory_order_acq_rel, memory_order_acquire)) + { + const int frames = pw.record.latest[i].frames; + atomic_store_explicit(&pw.record.latest[i].state, + RECORD_LATEST_FREE, memory_order_release); + atomic_fetch_add_explicit(&pw.record.droppedFrames, + frames, memory_order_relaxed); + } + } + + const int frames = pw.record.latest[chosen].frames; + const LG_AudioClock * clock = + pw.record.latest[chosen].clockValid ? + &pw.record.latest[chosen].clock : NULL; + if (!pw.record.pushFn( + pw.record.latestBuffer + + (size_t)chosen * pw.record.sendBufferFrames * pw.record.stride, + frames, clock)) + pipewire_recordRejected(frames); + atomic_store_explicit(&pw.record.latest[chosen].state, + RECORD_LATEST_FREE, memory_order_release); + return true; } static int pipewire_recordSendThread(void * opaque) { + uint64_t nextErrorReport = nanotime() + + RECORD_ERROR_REPORT_INTERVAL_NS; + for (;;) { - int available; - while ((available = ringbuffer_getCount(pw.record.sendQueue)) > 0) - { - const int frames = min(available, pw.record.sendBufferFrames); - const int consumed = ringbuffer_consume( - pw.record.sendQueue, pw.record.sendBuffer, frames); - DEBUG_ASSERT(consumed == frames); - pw.record.pushFn(pw.record.sendBuffer, frames); - } - - pipewire_reportRecordErrors(); - if (atomic_load_explicit(&pw.record.sendStop, memory_order_acquire)) - break; - int result; do result = sem_wait(&pw.record.sendWake); @@ -811,6 +1280,87 @@ static int pipewire_recordSendThread(void * opaque) DEBUG_ERROR("Failed to wait for the PipeWire recording worker"); break; } + + /* Clear before draining. A producer racing before this point is covered by + * this wake, while a producer racing after it publishes the sole next + * wake. */ + atomic_store_explicit( + &pw.record.sendWakePending, false, memory_order_release); + if (atomic_load_explicit(&pw.record.sendStop, memory_order_acquire)) + break; + + for (;;) + { + pipewire_reportRecordErrorsDue(&nextErrorReport); + + if (atomic_load_explicit( + &pw.record.sendStop, memory_order_acquire)) + break; + + if (atomic_load_explicit( + &pw.record.latestMode, memory_order_acquire)) + { + const int queued = ringbuffer_getCount(pw.record.sendQueue); + if (queued > 0) + { + ringbuffer_consume(pw.record.sendQueue, NULL, queued); + atomic_fetch_add_explicit( + &pw.record.droppedFrames, queued, memory_order_relaxed); + } + const int queuedBlocks = ringbuffer_getCount(pw.record.sendBlocks); + if (queuedBlocks > 0) + ringbuffer_consume( + pw.record.sendBlocks, NULL, queuedBlocks); + + while (!atomic_load_explicit( + &pw.record.sendStop, memory_order_acquire) && + pipewire_recordSendLatest()) + ; + + atomic_store_explicit( + &pw.record.latestMode, false, memory_order_release); + if (pipewire_recordLatestPending()) + { + atomic_store_explicit( + &pw.record.latestMode, true, memory_order_release); + continue; + } + } + + RecordBlock block; + if (ringbuffer_consume( + pw.record.sendBlocks, &block, 1) != 1) + break; + + LG_AudioClock clock = block.clock; + int remaining = block.frames; + while (remaining > 0 && !atomic_load_explicit( + &pw.record.sendStop, memory_order_acquire)) + { + const int frames = min( + remaining, pw.record.sendBufferFrames); + const int consumed = ringbuffer_consume( + pw.record.sendQueue, pw.record.sendBuffer, frames); + DEBUG_ASSERT(consumed == frames); + if (consumed != frames) + { + atomic_fetch_add_explicit( + &pw.record.bufferErrors, 1, memory_order_relaxed); + break; + } + + if (!pw.record.pushFn(pw.record.sendBuffer, frames, + block.clockValid ? &clock : NULL)) + pipewire_recordRejected(frames); + remaining -= frames; + if (block.clockValid) + pipewire_recordAdvanceClock(&clock, frames); + } + } + + if (atomic_load_explicit( + &pw.record.sendStop, memory_order_acquire)) + break; } pipewire_reportRecordErrors(); @@ -819,12 +1369,12 @@ static int pipewire_recordSendThread(void * opaque) static void pipewire_recordStopSender(void) { + const bool hadThread = pw.record.sendThread != NULL; if (pw.record.sendThread) { atomic_store_explicit( &pw.record.sendStop, true, memory_order_release); - if (sem_post(&pw.record.sendWake) < 0) - DEBUG_ERROR("Failed to stop the PipeWire recording worker"); + pipewire_recordSignalSender(); lgJoinThread(pw.record.sendThread, NULL); pw.record.sendThread = NULL; } @@ -836,10 +1386,15 @@ static void pipewire_recordStopSender(void) } ringbuffer_free(&pw.record.sendQueue); + ringbuffer_free(&pw.record.sendBlocks); free(pw.record.sendBuffer); - pw.record.sendBuffer = NULL; + free(pw.record.latestBuffer); + pw.record.sendBuffer = NULL; + pw.record.latestBuffer = NULL; pw.record.sendBufferFrames = 0; - pipewire_reportRecordErrors(); + pw.record.maxQueuedFrames = 0; + if (!hadThread) + pipewire_reportRecordErrors(); } static bool pipewire_recordStartSender(int sampleRate) @@ -851,10 +1406,16 @@ static bool pipewire_recordStartSender(int sampleRate) pw.record.sendQueue = ringbuffer_new( queueFrames, pw.record.stride); + pw.record.sendBlocks = ringbuffer_new( + queueFrames, sizeof(RecordBlock)); pw.record.sendBuffer = malloc( (size_t)sendFrames * pw.record.stride); + pw.record.latestBuffer = malloc( + (size_t)RECORD_LATEST_SLOTS * sendFrames * pw.record.stride); pw.record.sendBufferFrames = sendFrames; - if (!pw.record.sendQueue || !pw.record.sendBuffer) + pw.record.maxQueuedFrames = max(sampleRate / 50, 1); + if (!pw.record.sendQueue || !pw.record.sendBlocks || + !pw.record.sendBuffer || !pw.record.latestBuffer) { DEBUG_ERROR("Failed to allocate the PipeWire recording queue"); pipewire_recordStopSender(); @@ -871,12 +1432,42 @@ static bool pipewire_recordStartSender(int sampleRate) atomic_store_explicit( &pw.record.sendStop, false, memory_order_relaxed); + atomic_store_explicit( + &pw.record.sendWakePending, false, memory_order_relaxed); + atomic_store_explicit( + &pw.record.latestMode, false, memory_order_relaxed); + atomic_store_explicit( + &pw.record.latestSerial, 0, memory_order_relaxed); + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + atomic_store_explicit(&pw.record.latest[i].state, + RECORD_LATEST_FREE, memory_order_relaxed); + atomic_store_explicit( + &pw.record.latest[i].serial, 0, memory_order_relaxed); + pw.record.latest[i].frames = 0; + pw.record.latest[i].clockValid = false; + pw.record.latest[i].clock = (LG_AudioClock) {0}; + } + pw.record.clockPosition = 0; + pw.record.clockLastPosition = 0; + pw.record.clockLastTicks = 0; + pw.record.clockLastTime = 0; + pw.record.clockLastRate = (struct spa_fraction) {0}; + pw.record.clockLastValid = false; + pw.record.clockLastTimingValid = false; + pw.record.clockDiscontinuityPending = false; atomic_store_explicit( &pw.record.bufferErrors, 0, memory_order_relaxed); atomic_store_explicit( &pw.record.droppedFrames, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.record.rejectedBatches, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.record.rejectedFrames, 0, memory_order_relaxed); atomic_store_explicit( &pw.record.signalErrors, 0, memory_order_relaxed); + atomic_store_explicit( + &pw.record.recycleError, 0, memory_order_relaxed); if (!lgCreateThread("pwRecordSend", pipewire_recordSendThread, NULL, &pw.record.sendThread)) { @@ -889,38 +1480,143 @@ static bool pipewire_recordStartSender(int sampleRate) static void pipewire_recordStopStream(void) { + pw_thread_loop_lock(pw.thread); + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; if (pw.record.stream) { - pw_thread_loop_lock(pw.thread); pw_stream_destroy(pw.record.stream); pw.record.stream = NULL; - pw_thread_loop_unlock(pw.thread); } - pw.record.active = false; + pw_thread_loop_unlock(pw.thread); + pipewire_recordStopSender(); + pw.record.pushFn = NULL; } -static void pipewire_recordQueueFrames(const void * data, int frames) +static void pipewire_recordQueueFrames( + const void * data, const RecordBlock * block) { + const int frames = block->frames; + int chosen = -1; + if (atomic_load_explicit( + &pw.record.latestMode, memory_order_acquire)) + goto latest; + const int occupancy = ringbuffer_getCount(pw.record.sendQueue); const int available = max(0, ringbuffer_getLength(pw.record.sendQueue) - occupancy); - const int append = min(frames, available); + const bool exceedsBacklog = occupancy > 0 && + (occupancy >= pw.record.maxQueuedFrames || + frames > pw.record.maxQueuedFrames - occupancy); + if (frames > available || exceedsBacklog) + goto latest; + const int advanced = - ringbuffer_append(pw.record.sendQueue, data, append); - DEBUG_ASSERT(advanced == append); + ringbuffer_append(pw.record.sendQueue, data, frames); + DEBUG_ASSERT(advanced == frames); + if (advanced == frames) + { + const int blocks = ringbuffer_append( + pw.record.sendBlocks, block, 1); + DEBUG_ASSERT(blocks == 1); + if (blocks == 1) + { + pipewire_recordSignalSender(); + return; + } + } - if (append != frames) - atomic_fetch_add_explicit( - &pw.record.droppedFrames, frames - append, memory_order_relaxed); +latest: + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + unsigned int expected = RECORD_LATEST_FREE; + if (atomic_compare_exchange_strong_explicit( + &pw.record.latest[i].state, &expected, + RECORD_LATEST_WRITING, memory_order_acq_rel, memory_order_acquire)) + { + chosen = i; + break; + } + } - /* Posting only when the producer observes an empty queue can lose a wake - * while the consumer drains the previous batch. Always notify for appended - * data; the sender coalesces all queued frames when it wakes. */ - if (append > 0 && sem_post(&pw.record.sendWake) < 0) + if (chosen < 0) + { + int oldest = -1; + uint64_t oldestSerial = UINT64_MAX; + for (int i = 0; i < RECORD_LATEST_SLOTS; ++i) + { + if (atomic_load_explicit( + &pw.record.latest[i].state, memory_order_acquire) != + RECORD_LATEST_READY) + continue; + + const uint64_t serial = atomic_load_explicit( + &pw.record.latest[i].serial, memory_order_relaxed); + if (serial < oldestSerial) + { + oldest = i; + oldestSerial = serial; + } + } + + if (oldest >= 0) + { + unsigned int expected = RECORD_LATEST_READY; + if (atomic_compare_exchange_strong_explicit( + &pw.record.latest[oldest].state, &expected, + RECORD_LATEST_WRITING, + memory_order_acq_rel, memory_order_acquire)) + { + chosen = oldest; + const int dropped = pw.record.latest[oldest].frames; + atomic_fetch_add_explicit(&pw.record.droppedFrames, + dropped, memory_order_relaxed); + } + } + } + + if (chosen < 0) + { atomic_fetch_add_explicit( - &pw.record.signalErrors, 1, memory_order_relaxed); + &pw.record.droppedFrames, frames, memory_order_relaxed); + pw.record.clockDiscontinuityPending = true; + atomic_store_explicit( + &pw.record.latestMode, true, memory_order_release); + pipewire_recordSignalSender(); + return; + } + + const int keep = min(frames, pw.record.sendBufferFrames); + memcpy( + pw.record.latestBuffer + + (size_t)chosen * pw.record.sendBufferFrames * pw.record.stride, + (const uint8_t *)data + (size_t)(frames - keep) * pw.record.stride, + (size_t)keep * pw.record.stride); + pw.record.latest[chosen].frames = keep; + pw.record.latest[chosen].clockValid = block->clockValid; + if (block->clockValid) + { + pw.record.latest[chosen].clock = block->clock; + pipewire_recordAdvanceClock( + &pw.record.latest[chosen].clock, frames - keep); + pw.record.latest[chosen].clock.discontinuity = true; + pw.record.latest[chosen].clock.stable = false; + } + atomic_store_explicit(&pw.record.latest[chosen].serial, + atomic_fetch_add_explicit( + &pw.record.latestSerial, 1, memory_order_relaxed) + 1, + memory_order_relaxed); + atomic_store_explicit(&pw.record.latest[chosen].state, + RECORD_LATEST_READY, memory_order_release); + atomic_store_explicit( + &pw.record.latestMode, true, memory_order_release); + + if (keep != frames) + atomic_fetch_add_explicit( + &pw.record.droppedFrames, frames - keep, memory_order_relaxed); + pipewire_recordSignalSender(); } static void pipewire_onRecordProcess(void * userdata) @@ -931,6 +1627,7 @@ static void pipewire_onRecordProcess(void * userdata) { atomic_fetch_add_explicit( &pw.record.bufferErrors, 1, memory_order_relaxed); + pw.record.clockDiscontinuityPending = true; return; } @@ -941,65 +1638,111 @@ static void pipewire_onRecordProcess(void * userdata) { atomic_fetch_add_explicit( &pw.record.bufferErrors, 1, memory_order_relaxed); + pw.record.clockDiscontinuityPending = true; #if PW_CHECK_VERSION(1, 4, 0) - pw_stream_return_buffer(pw.record.stream, pbuf); + const int recycleResult = + pw_stream_return_buffer(pw.record.stream, pbuf); #else - pw_stream_queue_buffer(pw.record.stream, pbuf); + const int recycleResult = + pw_stream_queue_buffer(pw.record.stream, pbuf); #endif + pipewire_storeError(&pw.record.recycleError, recycleResult); return; } const uint32_t offset = sbuf->datas[0].chunk->offset; - const uint32_t bytes = min( + const uint32_t flags = sbuf->datas[0].chunk->flags; + const bool empty = flags & SPA_CHUNK_FLAG_EMPTY; + if (flags & (SPA_CHUNK_FLAG_EMPTY | SPA_CHUNK_FLAG_CORRUPTED)) + pw.record.clockDiscontinuityPending = true; + + const uint32_t bytes = empty ? 0 : min( sbuf->datas[0].chunk->size, sbuf->datas[0].maxsize - offset); const int frames = bytes / pw.record.stride; + if (bytes % pw.record.stride) + pw.record.clockDiscontinuityPending = true; if (frames > 0) + { + const RecordBlock block = + pipewire_recordMakeBlock(pbuf, sbuf, frames); pipewire_recordQueueFrames( - (uint8_t *)sbuf->datas[0].data + offset, frames); + (uint8_t *)sbuf->datas[0].data + offset, &block); + } - pw_stream_queue_buffer(pw.record.stream, pbuf); + pipewire_storeError(&pw.record.recycleError, + pw_stream_queue_buffer(pw.record.stream, pbuf)); } -static void pipewire_recordStart(const LG_AudioFormat * format, - LG_AudioPushFn pushFn) +static void pipewire_onRecordStateChanged(void * userdata, + enum pw_stream_state old, enum pw_stream_state state, + const char * error) { + (void)error; + pw.record.connectionState = state; + if (state == PW_STREAM_STATE_ERROR || + state == PW_STREAM_STATE_UNCONNECTED) + { + LG_AudioFailureFn failureFn = pw.record.failureFn; + const uint32_t failureCookie = pw.record.failureCookie; + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; + + if (failureFn && failureCookie) + failureFn(failureCookie); + } + pw_thread_loop_signal(pw.thread, false); +} + +static int pipewire_recordControl(bool active) +{ + int error = 0; + pw_thread_loop_lock(pw.thread); + if (!pw.record.stream) + error = active ? -ENODEV : 0; + else if (pw.record.active != active) + { + error = pw_stream_set_active(pw.record.stream, active); + if (error >= 0) + pw.record.active = active; + } + pw_thread_loop_unlock(pw.thread); + return error; +} + +static bool pipewire_recordStart(const LG_AudioFormat * format, + LG_AudioPushFn pushFn, LG_AudioFailureFn failureFn, + uint32_t failureCookie) +{ + pipewire_recordStopStream(); + const int channels = format->channelCount; const int sampleRate = format->sampleRate; const int sampleSize = pipewire_sampleSize(format->sampleFormat); + const enum spa_audio_format sampleFormat = + pipewire_sampleFormat(format->sampleFormat); + if (!sampleSize || sampleFormat == SPA_AUDIO_FORMAT_UNKNOWN) + return false; + const int periodFrames = max(sampleRate / 1000, 1); char requestedNodeLatency[32]; snprintf(requestedNodeLatency, sizeof(requestedNodeLatency), "%d/%d", periodFrames, sampleRate); - const struct spa_pod * params[1]; + const struct spa_pod * params[2]; uint8_t buffer[1024]; struct spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer)); static const struct pw_stream_events events = { - .version = PW_VERSION_STREAM_EVENTS, - .process = pipewire_onRecordProcess + .version = PW_VERSION_STREAM_EVENTS, + .state_changed = pipewire_onRecordStateChanged, + .process = pipewire_onRecordProcess }; - if (pw.record.stream && - pipewire_audioFormatEqual(&pw.record.format, format)) - { - if (!pw.record.active) - { - pw_thread_loop_lock(pw.thread); - pw_stream_set_active(pw.record.stream, true); - pw.record.active = true; - pw_thread_loop_unlock(pw.thread); - } - return; - } - - pipewire_recordStopStream(); - pw.record.format = *format; pw.record.stride = sampleSize * channels; pw.record.pushFn = pushFn; if (!pipewire_recordStartSender(sampleRate)) - return; + return false; struct pw_properties * props = pw_properties_new( @@ -1014,7 +1757,7 @@ static void pipewire_recordStart(const LG_AudioFormat * format, { DEBUG_ERROR("Failed to create recording stream properties"); pipewire_recordStopSender(); - return; + return false; } const char * device = option_get_string("pipewire", "recDevice"); @@ -1028,6 +1771,8 @@ static void pipewire_recordStart(const LG_AudioFormat * format, } pw_thread_loop_lock(pw.thread); + pw.record.failureFn = failureFn; + pw.record.failureCookie = failureCookie; pw.record.stream = pw_stream_new_simple( pw.loop, "Looking Glass", @@ -1038,40 +1783,71 @@ static void pipewire_recordStart(const LG_AudioFormat * format, if (!pw.record.stream) { + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; pw_thread_loop_unlock(pw.thread); DEBUG_ERROR("Failed to create the stream"); pipewire_recordStopSender(); - return; + return false; } struct spa_audio_info_raw info = - pipewire_audioInfo(format, - pipewire_sampleFormat(format->sampleFormat)); + pipewire_audioInfo(format, sampleFormat); params[0] = spa_format_audio_raw_build( &b, SPA_PARAM_EnumFormat, &info); + params[1] = spa_pod_builder_add_object(&b, + SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, + SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header), + SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_header))); - const int result = pw_stream_connect( + pw.record.connectionState = PW_STREAM_STATE_CONNECTING; + int result = pw_stream_connect( pw.record.stream, PW_DIRECTION_INPUT, PW_ID_ANY, PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_MAP_BUFFERS | - PW_STREAM_FLAG_RT_PROCESS, - params, 1); + PW_STREAM_FLAG_RT_PROCESS | + PW_STREAM_FLAG_INACTIVE, + params, 2); if (result < 0) { DEBUG_ERROR("Failed to connect recording stream: %s", spa_strerror(result)); + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; pw_stream_destroy(pw.record.stream); pw.record.stream = NULL; pw_thread_loop_unlock(pw.thread); pipewire_recordStopSender(); - return; + return false; + } + + if (!pipewire_waitForStream(&pw.record.connectionState) || + pw.record.connectionState != PW_STREAM_STATE_PAUSED) + { + DEBUG_ERROR("PipeWire recording stream did not become ready"); + pw.record.failureFn = NULL; + pw.record.failureCookie = 0; + pw_stream_destroy(pw.record.stream); + pw.record.stream = NULL; + pw_thread_loop_unlock(pw.thread); + pipewire_recordStopSender(); + return false; } pw_thread_loop_unlock(pw.thread); - pw.record.active = true; + result = pipewire_recordControl(true); + if (result < 0) + { + DEBUG_ERROR("Failed to activate recording stream: %s", + spa_strerror(result)); + pipewire_recordStopStream(); + return false; + } + + return true; } static void pipewire_recordStop(void) @@ -1081,31 +1857,56 @@ static void pipewire_recordStop(void) static void pipewire_recordVolume(int channels, const uint16_t volume[]) { - if (channels != pw.record.format.channelCount) + if (channels <= 0 || channels > LG_AUDIO_MAX_CHANNELS) return; float param[channels]; for(int i = 0; i < channels; ++i) - param[i] = 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787; + param[i] = max(0.0, + 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787); + int result = 0; pw_thread_loop_lock(pw.thread); - pw_stream_set_control(pw.record.stream, SPA_PROP_channelVolumes, - channels, param, 0); + if (pw.record.stream && + channels == pw.record.format.channelCount) + result = pw_stream_set_control(pw.record.stream, + SPA_PROP_channelVolumes, channels, param, 0); pw_thread_loop_unlock(pw.thread); + + if (result < 0) + DEBUG_WARN("Failed to set PipeWire recording volume: %s", + spa_strerror(result)); } static void pipewire_recordMute(bool mute) { + int result = 0; pw_thread_loop_lock(pw.thread); - float val = mute ? 1.0f : 0.0f; - pw_stream_set_control(pw.record.stream, SPA_PROP_mute, 1, &val, 0); + if (pw.record.stream) + { + float val = mute ? 1.0f : 0.0f; + result = pw_stream_set_control( + pw.record.stream, SPA_PROP_mute, 1, &val, 0); + } pw_thread_loop_unlock(pw.thread); + + if (result < 0) + DEBUG_WARN("Failed to set PipeWire recording mute: %s", + spa_strerror(result)); } static void pipewire_free(void) { pipewire_playbackStopStream(); pipewire_recordStopStream(); + + pw_thread_loop_lock(pw.thread); + if (pw.core) + { + pw_core_disconnect(pw.core); + pw.core = NULL; + } + pw_thread_loop_unlock(pw.thread); pw_thread_loop_stop(pw.thread); pw_thread_loop_destroy(pw.thread); pw_context_destroy(pw.context); diff --git a/client/audiodevs/PulseAudio/pulseaudio.c b/client/audiodevs/PulseAudio/pulseaudio.c index 06d4c5dc..6630e6b0 100644 --- a/client/audiodevs/PulseAudio/pulseaudio.c +++ b/client/audiodevs/PulseAudio/pulseaudio.c @@ -28,18 +28,23 @@ #include "common/debug.h" #include "common/time.h" +#define PULSEAUDIO_ERROR_REPORT_INTERVAL_NS INT64_C(1000000000) +#define PULSEAUDIO_OPERATION_TIMEOUT_US UINT64_C(250000) +#define PULSEAUDIO_READY_TIMEOUT_US UINT64_C(5000000) +#define PULSEAUDIO_RATE_UPDATE_INTERVAL_NS INT64_C(50000000) +#define PULSEAUDIO_RATE_UPDATE_MAX_HOLD_NS INT64_C(250000000) +#define PULSEAUDIO_RATE_UPDATE_DEADBAND_HZ UINT32_C(1) + struct PulseAudio { pa_threaded_mainloop * loop; pa_mainloop_api * api; pa_context * context; - pa_operation * contextSub; + bool loopStarted; pa_stream * sink; - int sinkIndex; + uint32_t sinkIndex; bool sinkCorked; - bool sinkMuted; - bool sinkStarting; bool sinkResamplerEnabled; bool sinkRateFailed; int sinkMaxPeriodFrames; @@ -50,9 +55,26 @@ struct PulseAudio uint32_t sinkAppliedRate; uint32_t sinkPendingRate; uint32_t sinkRequestedRate; + uint32_t sinkDeferredRate; + int64_t sinkNextRateUpdate; + int64_t sinkRateDeferredSince; + bool sinkRateUpdateArmed; pa_operation * sinkRateOperation; + pa_operation * sinkStartOperation; + pa_time_event * sinkStartTimer; + uint32_t sinkStartSerial; LG_AudioPullFn sinkPullFn; - _Atomic(int64_t) sinkPresentationDeadline; + LG_AudioFailureFn sinkFailureFn; + uint32_t sinkFailureCookie; + _Atomic(int64_t) sinkLatencyNs; + atomic_bool sinkLatencyUpdateRequested; + atomic_bool sinkErrorsPending; + _Atomic(int64_t) sinkNextErrorReport; + atomic_uint sinkUnderflows; + atomic_uint sinkOverflows; + atomic_uint sinkWriteErrors; + atomic_uint sinkRateErrors; + atomic_uint sinkControlErrors; }; static struct PulseAudio pa = {0}; @@ -136,13 +158,83 @@ static pa_channel_position_t pulseaudio_channel( return (pa_channel_position_t)(PA_CHANNEL_POSITION_AUX0 + index); } -static void pulseaudio_unrefOperation(pa_operation * operation) +static void pulseaudio_noteError(atomic_uint * counter) { - if (operation) + atomic_fetch_add_explicit(counter, 1, memory_order_relaxed); + atomic_store_explicit( + &pa.sinkErrorsPending, true, memory_order_release); +} + +static void pulseaudio_trackControlOperation(pa_operation * operation) +{ + if (!operation) + pulseaudio_noteError(&pa.sinkControlErrors); + else pa_operation_unref(operation); } -static bool pulseaudio_submitRateUpdate(void); +static void pulseaudio_streamControl_cb( + pa_stream * stream, int success, void * userdata) +{ + if (!success) + pulseaudio_noteError(&pa.sinkControlErrors); +} + +static void pulseaudio_contextControl_cb( + pa_context * context, int success, void * userdata) +{ + if (!success) + pulseaudio_noteError(&pa.sinkControlErrors); +} + +static void pulseaudio_reportErrors(bool force) +{ + if (!atomic_load_explicit( + &pa.sinkErrorsPending, memory_order_acquire)) + return; + + const int64_t now = (int64_t)nanotime(); + if (!force) + { + int64_t next = atomic_load_explicit( + &pa.sinkNextErrorReport, memory_order_relaxed); + if (now < next || !atomic_compare_exchange_strong_explicit( + &pa.sinkNextErrorReport, &next, + now + PULSEAUDIO_ERROR_REPORT_INTERVAL_NS, + memory_order_relaxed, memory_order_relaxed)) + return; + } + + if (!atomic_exchange_explicit( + &pa.sinkErrorsPending, false, memory_order_acq_rel)) + return; + + const unsigned int underflows = atomic_exchange_explicit( + &pa.sinkUnderflows, 0, memory_order_relaxed); + const unsigned int overflows = atomic_exchange_explicit( + &pa.sinkOverflows, 0, memory_order_relaxed); + const unsigned int writeErrors = atomic_exchange_explicit( + &pa.sinkWriteErrors, 0, memory_order_relaxed); + const unsigned int rateErrors = atomic_exchange_explicit( + &pa.sinkRateErrors, 0, memory_order_relaxed); + const unsigned int controlErrors = atomic_exchange_explicit( + &pa.sinkControlErrors, 0, memory_order_relaxed); + + if (underflows) + DEBUG_WARN("PulseAudio playback underflowed %u time(s)", underflows); + if (overflows) + DEBUG_WARN("PulseAudio playback overflowed %u time(s)", overflows); + if (writeErrors) + DEBUG_WARN("PulseAudio playback write failed %u time(s)", writeErrors); + if (rateErrors) + DEBUG_WARN("PulseAudio sample rate update failed %u time(s)", + rateErrors); + if (controlErrors) + DEBUG_WARN("PulseAudio control operation failed %u time(s)", + controlErrors); +} + +static bool pulseaudio_submitRateUpdate(bool force); static void pulseaudio_rateUpdate_cb(pa_stream * stream, int success, void * userdata) @@ -156,33 +248,46 @@ static void pulseaudio_rateUpdate_cb(pa_stream * stream, int success, if (!success) { - DEBUG_ERROR("Failed to update PulseAudio sample rate: %s", - pa_strerror(pa_context_errno(pa.context))); + pulseaudio_noteError(&pa.sinkRateErrors); pa.sinkRateFailed = true; return; } pa.sinkAppliedRate = pa.sinkPendingRate; if (pa.sinkCorked) - pulseaudio_submitRateUpdate(); + pulseaudio_submitRateUpdate(true); } -static bool pulseaudio_submitRateUpdate(void) +static bool pulseaudio_submitRateUpdate(bool force) { if (pa.sinkRateFailed) return false; - if (pa.sinkRateOperation || - pa.sinkRequestedRate == pa.sinkAppliedRate) + if (pa.sinkRateOperation) return true; - pa.sinkPendingRate = pa.sinkRequestedRate; + if (pa.sinkRequestedRate == pa.sinkAppliedRate) + { + pa.sinkDeferredRate = pa.sinkRequestedRate; + pa.sinkRateUpdateArmed = false; + pa.sinkRateDeferredSince = 0; + return true; + } + + if (!force && !pa.sinkRateUpdateArmed) + return true; + + pa.sinkPendingRate = pa.sinkRequestedRate; + pa.sinkDeferredRate = pa.sinkPendingRate; + pa.sinkRateUpdateArmed = false; + pa.sinkRateDeferredSince = 0; + pa.sinkNextRateUpdate = + (int64_t)nanotime() + PULSEAUDIO_RATE_UPDATE_INTERVAL_NS; pa.sinkRateOperation = pa_stream_update_sample_rate(pa.sink, pa.sinkPendingRate, pulseaudio_rateUpdate_cb, NULL); if (!pa.sinkRateOperation) { - DEBUG_ERROR("Failed to request a PulseAudio sample rate update: %s", - pa_strerror(pa_context_errno(pa.context))); + pulseaudio_noteError(&pa.sinkRateErrors); pa.sinkRateFailed = true; return false; } @@ -190,31 +295,43 @@ static bool pulseaudio_submitRateUpdate(void) return true; } -static void pulseaudio_sink_input_cb(pa_context *c, const pa_sink_input_info *i, - int eol, void *userdata) +static void pulseaudio_cancelStartTimer_nl(void) { - if (eol < 0 || eol == 1) + if (!pa.sinkStartTimer) return; - pa.sinkIndex = i->index; + pa_time_event * timer = pa.sinkStartTimer; + pa.sinkStartTimer = NULL; + pa.api->time_free(timer); } -static void pulseaudio_subscribe_cb(pa_context *c, - pa_subscription_event_type_t t, uint32_t index, void *userdata) +static void pulseaudio_cancelStartOperation_nl(void) { - switch (t & PA_SUBSCRIPTION_EVENT_FACILITY_MASK) - { - case PA_SUBSCRIPTION_EVENT_SINK_INPUT: - if ((t & PA_SUBSCRIPTION_EVENT_TYPE_MASK) == PA_SUBSCRIPTION_EVENT_REMOVE) - pa.sinkIndex = 0; - else - { - pa_operation *o = pa_context_get_sink_input_info(c, index, - pulseaudio_sink_input_cb, NULL); - pulseaudio_unrefOperation(o); - } - break; - } + if (!pa.sinkStartOperation) + return; + + pa_operation * operation = pa.sinkStartOperation; + pa.sinkStartOperation = NULL; + pa_operation_cancel(operation); + pa_operation_unref(operation); +} + +static void pulseaudio_sinkDisarm_nl(void) +{ + ++pa.sinkStartSerial; + pulseaudio_cancelStartTimer_nl(); + pulseaudio_cancelStartOperation_nl(); + pa.sinkFailureFn = NULL; + pa.sinkFailureCookie = 0; +} + +static void pulseaudio_sinkFailed_nl(void) +{ + LG_AudioFailureFn failureFn = pa.sinkFailureFn; + const uint32_t failureCookie = pa.sinkFailureCookie; + pulseaudio_sinkDisarm_nl(); + if (failureFn) + failureFn(failureCookie); } static void pulseaudio_ctx_state_change_cb(pa_context * c, void * userdata) @@ -228,28 +345,120 @@ static void pulseaudio_ctx_state_change_cb(pa_context * c, void * userdata) case PA_CONTEXT_READY: DEBUG_INFO("Connected to PulseAudio server"); - pa_context_set_subscribe_callback(c, pulseaudio_subscribe_cb, NULL); - pa_context_subscribe(c, PA_SUBSCRIPTION_MASK_SINK_INPUT, NULL, NULL); pa_threaded_mainloop_signal(pa.loop, 0); break; case PA_CONTEXT_TERMINATED: - if (pa.contextSub) - { - pa_operation_unref(pa.contextSub); - pa.contextSub = NULL; - } + if (c == pa.context) + pulseaudio_sinkFailed_nl(); + pa_threaded_mainloop_signal(pa.loop, 0); break; case PA_CONTEXT_FAILED: default: + if (c == pa.context) + pulseaudio_sinkFailed_nl(); DEBUG_ERROR("context error: %s", pa_strerror(pa_context_errno(c))); + pa_threaded_mainloop_signal(pa.loop, 0); break; } } +static void pulseaudio_context_close_nl(void) +{ + if (!pa.context) + return; + + pa_context_set_state_callback(pa.context, NULL, NULL); + pa_context_disconnect(pa.context); + pa_context_unref(pa.context); + pa.context = NULL; +} + +struct PulseWait +{ + bool timedOut; +}; + +static void pulseaudio_timeout_cb(pa_mainloop_api * api, + pa_time_event * event, const struct timeval * tv, void * userdata) +{ + (void)api; + (void)event; + (void)tv; + struct PulseWait * wait = userdata; + wait->timedOut = true; + pa_threaded_mainloop_signal(pa.loop, 0); +} + +/* The PulseAudio threaded mainloop must be locked. */ +static bool pulseaudio_contextConnect_nl(void) +{ + pa_proplist * propList = pa_proplist_new(); + if (!propList) + { + DEBUG_ERROR("Failed to create the PulseAudio property list"); + return false; + } + pa_proplist_sets(propList, PA_PROP_MEDIA_ROLE, "video"); + + pa.context = pa_context_new_with_proplist( + pa.api, "Looking Glass", propList); + pa_proplist_free(propList); + if (!pa.context) + { + DEBUG_ERROR("Failed to create the PulseAudio context"); + return false; + } + + pa_context_set_state_callback(pa.context, + pulseaudio_ctx_state_change_cb, NULL); + if (pa_context_connect( + pa.context, NULL, PA_CONTEXT_NOAUTOSPAWN, NULL) < 0) + { + DEBUG_ERROR("Failed to connect to the PulseAudio server: %s", + pa_strerror(pa_context_errno(pa.context))); + pulseaudio_context_close_nl(); + return false; + } + + struct PulseWait wait = {0}; + pa_time_event * timer = pa_context_rttime_new(pa.context, + pa_rtclock_now() + PULSEAUDIO_READY_TIMEOUT_US, + pulseaudio_timeout_cb, &wait); + if (!timer) + { + DEBUG_ERROR("Failed to create the PulseAudio connection timer"); + pulseaudio_context_close_nl(); + return false; + } + + pa_context_state_t state; + while (!wait.timedOut) + { + state = pa_context_get_state(pa.context); + if (state == PA_CONTEXT_READY || !PA_CONTEXT_IS_GOOD(state)) + break; + pa_threaded_mainloop_wait(pa.loop); + } + state = pa_context_get_state(pa.context); + pa.api->time_free(timer); + + if (state == PA_CONTEXT_READY) + return true; + + if (wait.timedOut) + DEBUG_ERROR("Timed out connecting to the PulseAudio server"); + else + DEBUG_ERROR("PulseAudio context did not become ready: %s", + pa_strerror(pa_context_errno(pa.context))); + pulseaudio_context_close_nl(); + return false; +} + static bool pulseaudio_init(void) { + pa.sinkIndex = PA_INVALID_INDEX; pa.loop = pa_threaded_mainloop_new(); if (!pa.loop) { @@ -258,74 +467,33 @@ static bool pulseaudio_init(void) } pa.api = pa_threaded_mainloop_get_api(pa.loop); - if (pa_signal_init(pa.api) != 0) - { - DEBUG_ERROR("Failed to init signals"); - goto err_loop; - } - if (pa_threaded_mainloop_start(pa.loop) < 0) { DEBUG_ERROR("Failed to start the main loop"); goto err_loop; } - - pa_proplist * propList = pa_proplist_new(); - if (!propList) - { - DEBUG_ERROR("Failed to create the proplist"); - goto err_thread; - } - pa_proplist_sets(propList, PA_PROP_MEDIA_ROLE, "video"); + pa.loopStarted = true; pa_threaded_mainloop_lock(pa.loop); - pa.context = pa_context_new_with_proplist( - pa.api, - "Looking Glass", - propList); - if (!pa.context) + if (!pulseaudio_contextConnect_nl()) { - DEBUG_ERROR("Failed to create the context"); - goto err_context; + pa_threaded_mainloop_unlock(pa.loop); + goto err_thread; } - - pa_context_set_state_callback(pa.context, - pulseaudio_ctx_state_change_cb, NULL); - - if (pa_context_connect(pa.context, NULL, PA_CONTEXT_NOAUTOSPAWN, NULL) < 0) - { - DEBUG_ERROR("Failed to connect to the context server"); - goto err_context; - } - - for(;;) - { - pa_context_state_t state = pa_context_get_state(pa.context); - if(!PA_CONTEXT_IS_GOOD(state)) - { - DEBUG_ERROR("Context is bad"); - goto err_context; - } - - if (state == PA_CONTEXT_READY) - break; - - pa_threaded_mainloop_wait(pa.loop); - } - pa_threaded_mainloop_unlock(pa.loop); - pa_proplist_free(propList); return true; -err_context: - pa_threaded_mainloop_unlock(pa.loop); - pa_proplist_free(propList); - err_thread: - pa_threaded_mainloop_stop(pa.loop); + if (pa.loopStarted) + { + pa_threaded_mainloop_stop(pa.loop); + pa.loopStarted = false; + } err_loop: pa_threaded_mainloop_free(pa.loop); + pa.loop = NULL; + pa.api = NULL; err: return false; @@ -333,6 +501,7 @@ err: static void pulseaudio_sink_close_nl(void) { + pulseaudio_sinkDisarm_nl(); if (!pa.sink) return; @@ -347,51 +516,92 @@ static void pulseaudio_sink_close_nl(void) pa_operation_cancel(operation); pa_operation_unref(operation); } - pulseaudio_unrefOperation(pa_stream_flush(pa.sink, NULL, NULL)); + pa_stream_disconnect(pa.sink); pa_stream_unref(pa.sink); - pa.sink = NULL; - pa.sinkResamplerEnabled = false; - pa.sinkRateFailed = false; - pa.sinkNominalRate = 0; - pa.sinkAppliedRate = 0; - pa.sinkPendingRate = 0; - pa.sinkRequestedRate = 0; + pa.sink = NULL; + pa.sinkIndex = PA_INVALID_INDEX; + pa.sinkResamplerEnabled = false; + pa.sinkRateFailed = false; + pa.sinkNominalRate = 0; + pa.sinkAppliedRate = 0; + pa.sinkPendingRate = 0; + pa.sinkRequestedRate = 0; + pa.sinkDeferredRate = 0; + pa.sinkNextRateUpdate = 0; + pa.sinkRateDeferredSince = 0; + pa.sinkRateUpdateArmed = false; atomic_store_explicit( - &pa.sinkPresentationDeadline, 0, memory_order_release); + &pa.sinkLatencyNs, 0, memory_order_release); + atomic_store_explicit( + &pa.sinkLatencyUpdateRequested, false, memory_order_relaxed); } static void pulseaudio_free(void) { + if (!pa.loop) + return; + pa_threaded_mainloop_lock(pa.loop); pulseaudio_sink_close_nl(); - - pa_context_set_state_callback(pa.context, NULL, NULL); - pa_context_set_subscribe_callback(pa.context, NULL, NULL); - pa_context_disconnect(pa.context); - pa_context_unref(pa.context); - - if (pa.contextSub) - { - pa_operation_unref(pa.contextSub); - pa.contextSub = NULL; - } - + pulseaudio_context_close_nl(); pa_threaded_mainloop_unlock(pa.loop); + pulseaudio_reportErrors(true); + + if (pa.loopStarted) + { + pa_threaded_mainloop_stop(pa.loop); + pa.loopStarted = false; + } + pa_threaded_mainloop_free(pa.loop); + pa.loop = NULL; + pa.api = NULL; } static void pulseaudio_state_cb(pa_stream * p, void * userdata) { - if (pa.sinkStarting && pa_stream_get_state(pa.sink) == PA_STREAM_READY) - { - pulseaudio_unrefOperation(pa_stream_cork(pa.sink, 0, NULL, NULL)); - pa.sinkCorked = false; - pa.sinkStarting = false; - } - + const pa_stream_state_t state = pa_stream_get_state(p); + if (p == pa.sink && + (state == PA_STREAM_FAILED || state == PA_STREAM_TERMINATED)) + pulseaudio_sinkFailed_nl(); pa_threaded_mainloop_signal(pa.loop, 0); } +static void pulseaudio_start_cb( + pa_stream * stream, int success, void * userdata) +{ + const uint32_t serial = (uint32_t)(uintptr_t)userdata; + if (stream != pa.sink || serial != pa.sinkStartSerial) + return; + + pa_operation * operation = pa.sinkStartOperation; + pa.sinkStartOperation = NULL; + if (operation) + pa_operation_unref(operation); + pulseaudio_cancelStartTimer_nl(); + if (!success || pa_stream_get_state(stream) != PA_STREAM_READY || + !pa.context || + pa_context_get_state(pa.context) != PA_CONTEXT_READY) + { + pa.sinkCorked = true; + pulseaudio_sinkFailed_nl(); + } +} + +static void pulseaudio_startTimeout_cb(pa_mainloop_api * api, + pa_time_event * event, const struct timeval * tv, void * userdata) +{ + (void)tv; + const uint32_t serial = (uint32_t)(uintptr_t)userdata; + if (event != pa.sinkStartTimer || serial != pa.sinkStartSerial) + return; + + pa.sinkStartTimer = NULL; + api->time_free(event); + pa.sinkCorked = true; + pulseaudio_sinkFailed_nl(); +} + static void pulseaudio_write_cb(pa_stream * p, size_t nbytes, void * userdata) { // PulseAudio tries to pull data from the stream as soon as it is created for @@ -403,11 +613,17 @@ static void pulseaudio_write_cb(pa_stream * p, size_t nbytes, void * userdata) if (pa_stream_begin_write(p, (void **)&dst, &nbytes) < 0) { - DEBUG_ERROR("pa_stream_begin_write failed: %s", - pa_strerror(pa_context_errno(pa.context))); + pulseaudio_noteError(&pa.sinkWriteErrors); return; } + pa_usec_t latency = 0; + int negative = 0; + bool latencyValid = false; + if (atomic_exchange_explicit( + &pa.sinkLatencyUpdateRequested, false, memory_order_acquire)) + latencyValid = pa_stream_get_latency(p, &latency, &negative) == 0; + int frames = nbytes / pa.sinkStride; frames = pa.sinkPullFn(dst, frames); if (frames <= 0) @@ -419,8 +635,7 @@ static void pulseaudio_write_cb(pa_stream * p, size_t nbytes, void * userdata) if (pa_stream_write( p, dst, frames * pa.sinkStride, NULL, 0, PA_SEEK_RELATIVE) < 0) { - DEBUG_ERROR("pa_stream_write failed: %s", - pa_strerror(pa_context_errno(pa.context))); + pulseaudio_noteError(&pa.sinkWriteErrors); pa_stream_cancel_write(p); return; } @@ -428,26 +643,24 @@ static void pulseaudio_write_cb(pa_stream * p, size_t nbytes, void * userdata) /* Queue rate changes after the current audio block. This keeps the rate * reported by the preceding pull aligned with the block PulseAudio has * already received. */ - pulseaudio_submitRateUpdate(); + pulseaudio_submitRateUpdate(false); - pa_usec_t latency; - int negative; - if (pa_stream_get_latency(p, &latency, &negative) == 0) + if (latencyValid) { const int64_t latencyNs = negative ? 0 : (int64_t)latency * 1000; - atomic_store_explicit(&pa.sinkPresentationDeadline, - (int64_t)nanotime() + latencyNs, memory_order_release); + atomic_store_explicit( + &pa.sinkLatencyNs, latencyNs, memory_order_release); } } static void pulseaudio_underflow_cb(pa_stream * p, void * userdata) { - DEBUG_WARN("Underflow"); + pulseaudio_noteError(&pa.sinkUnderflows); } static void pulseaudio_overflow_cb(pa_stream * p, void * userdata) { - DEBUG_WARN("Overflow"); + pulseaudio_noteError(&pa.sinkOverflows); } static bool pulseaudio_setup(const LG_AudioFormat * format, @@ -456,6 +669,7 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, LG_AudioPullFn pullFn) { *resamplerEnabled = false; + pulseaudio_reportErrors(false); const int channels = format->channelCount; const int sampleRate = format->sampleRate; @@ -488,10 +702,25 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, }; pa_threaded_mainloop_lock(pa.loop); + if (!pa.context || + pa_context_get_state(pa.context) != PA_CONTEXT_READY) + { + pulseaudio_sink_close_nl(); + pulseaudio_context_close_nl(); + if (!pulseaudio_contextConnect_nl()) + { + pa_threaded_mainloop_unlock(pa.loop); + return false; + } + } + /* pa_stream_update_sample_rate requires protocol version 12. */ const bool enableResampler = requestResampler && pa_context_get_server_protocol_version(pa.context) >= 12; - if (pa.sink && !pa.sinkRateFailed && + if (pa.sink && pa.context && + pa_stream_get_state(pa.sink) == PA_STREAM_READY && + pa_context_get_state(pa.context) == PA_CONTEXT_READY && + !pa.sinkRateFailed && pa.sinkResamplerEnabled == enableResampler && !pa.sinkRateOperation && pa.sinkAppliedRate == pa.sinkNominalRate && @@ -511,13 +740,16 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, pa.sinkStride = stride; pa.sinkPullFn = pullFn; pa.sinkCorked = true; - pa.sinkStarting = false; pa.sinkResamplerEnabled = enableResampler; pa.sinkRateFailed = false; pa.sinkNominalRate = sampleRate; pa.sinkAppliedRate = sampleRate; pa.sinkPendingRate = sampleRate; pa.sinkRequestedRate = sampleRate; + pa.sinkDeferredRate = sampleRate; + pa.sinkNextRateUpdate = 0; + pa.sinkRateDeferredSince = 0; + pa.sinkRateUpdateArmed = false; pa.sink = pa_stream_new( pa.context, "Looking Glass", &spec, &channelMap); @@ -550,18 +782,44 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, return false; } - while (pa_stream_get_state(pa.sink) == PA_STREAM_CREATING) - pa_threaded_mainloop_wait(pa.loop); - - if (pa_stream_get_state(pa.sink) != PA_STREAM_READY) + pa_stream * sink = pa.sink; + pa_context * context = pa.context; + struct PulseWait wait = {0}; + pa_time_event * timer = pa_context_rttime_new(context, + pa_rtclock_now() + PULSEAUDIO_READY_TIMEOUT_US, + pulseaudio_timeout_cb, &wait); + if (!timer) { - DEBUG_ERROR("PulseAudio stream did not become ready: %s", - pa_strerror(pa_context_errno(pa.context))); + DEBUG_ERROR("Failed to create the PulseAudio stream setup timer"); pulseaudio_sink_close_nl(); pa_threaded_mainloop_unlock(pa.loop); return false; } + while (!wait.timedOut && pa.sink == sink && + pa_stream_get_state(sink) == PA_STREAM_CREATING && + pa.context == context && + PA_CONTEXT_IS_GOOD(pa_context_get_state(context))) + pa_threaded_mainloop_wait(pa.loop); + + const bool ready = pa.sink == sink && pa.context == context && + pa_stream_get_state(sink) == PA_STREAM_READY && + pa_context_get_state(context) == PA_CONTEXT_READY; + pa.api->time_free(timer); + + if (!ready) + { + if (wait.timedOut) + DEBUG_ERROR("Timed out setting up the PulseAudio stream"); + else + DEBUG_ERROR("PulseAudio stream did not become ready: %s", + pa_strerror(pa_context_errno(context))); + pulseaudio_sink_close_nl(); + pa_threaded_mainloop_unlock(pa.loop); + return false; + } + pa.sinkIndex = pa_stream_get_index(pa.sink); + const pa_buffer_attr * actual = pa_stream_get_buffer_attr(pa.sink); const uint64_t minRequestFrames = actual && actual->minreq != UINT32_MAX ? @@ -584,79 +842,123 @@ static bool pulseaudio_setup(const LG_AudioFormat * format, *resamplerEnabled = pa.sinkResamplerEnabled; atomic_store_explicit( - &pa.sinkPresentationDeadline, 0, memory_order_release); + &pa.sinkLatencyNs, 0, memory_order_release); + atomic_store_explicit( + &pa.sinkLatencyUpdateRequested, false, memory_order_relaxed); pa_threaded_mainloop_unlock(pa.loop); return true; } -static void pulseaudio_start(void) +static bool pulseaudio_start( + LG_AudioFailureFn failureFn, uint32_t failureCookie) { - if (!pa.sink) - return; + pulseaudio_reportErrors(false); + if (!pa.loop || pa_threaded_mainloop_in_thread(pa.loop)) + return false; pa_threaded_mainloop_lock(pa.loop); - pa_stream_state_t state = pa_stream_get_state(pa.sink); - if (state == PA_STREAM_CREATING) - pa.sinkStarting = true; - else + pa_stream * sink = pa.sink; + pa_context * context = pa.context; + if (!sink || !context || + pa_stream_get_state(sink) != PA_STREAM_READY || + pa_context_get_state(context) != PA_CONTEXT_READY) { - pulseaudio_unrefOperation(pa_stream_cork(pa.sink, 0, NULL, NULL)); - pa.sinkCorked = false; + pa_threaded_mainloop_unlock(pa.loop); + return false; } + pulseaudio_sinkDisarm_nl(); + pa.sinkFailureFn = failureFn; + pa.sinkFailureCookie = failureCookie; + const uint32_t serial = pa.sinkStartSerial; + pa.sinkStartTimer = pa_context_rttime_new(context, + pa_rtclock_now() + PULSEAUDIO_OPERATION_TIMEOUT_US, + pulseaudio_startTimeout_cb, (void *)(uintptr_t)serial); + if (!pa.sinkStartTimer) + { + pulseaudio_sinkDisarm_nl(); + pa_threaded_mainloop_unlock(pa.loop); + return false; + } + + pa.sinkStartOperation = pa_stream_cork(sink, 0, + pulseaudio_start_cb, (void *)(uintptr_t)serial); + if (!pa.sinkStartOperation) + { + pulseaudio_sinkDisarm_nl(); + pa_threaded_mainloop_unlock(pa.loop); + return false; + } + pa.sinkCorked = false; + pa_threaded_mainloop_unlock(pa.loop); + return true; } static void pulseaudio_stop(void) { - if (!pa.sink) + if (!pa.loop) return; bool needLock = !pa_threaded_mainloop_in_thread(pa.loop); if (needLock) pa_threaded_mainloop_lock(pa.loop); - pulseaudio_unrefOperation(pa_stream_cork(pa.sink, 1, NULL, NULL)); - pa.sinkCorked = true; - pa.sinkStarting = false; + pulseaudio_sinkDisarm_nl(); + if (!pa.sink) + { + if (needLock) + pa_threaded_mainloop_unlock(pa.loop); + return; + } + + pa.sinkCorked = true; + if (pa_stream_get_state(pa.sink) == PA_STREAM_READY) + { + pulseaudio_trackControlOperation( + pa_stream_cork(pa.sink, 1, pulseaudio_streamControl_cb, NULL)); + pulseaudio_trackControlOperation( + pa_stream_flush(pa.sink, pulseaudio_streamControl_cb, NULL)); + } if (pa.sinkResamplerEnabled) { pa.sinkRequestedRate = pa.sinkNominalRate; - pulseaudio_submitRateUpdate(); + pulseaudio_submitRateUpdate(true); } atomic_store_explicit( - &pa.sinkPresentationDeadline, 0, memory_order_release); + &pa.sinkLatencyNs, 0, memory_order_release); + atomic_store_explicit( + &pa.sinkLatencyUpdateRequested, false, memory_order_relaxed); if (needLock) + { pa_threaded_mainloop_unlock(pa.loop); + pulseaudio_reportErrors(false); + } } static void pulseaudio_volume(int channels, const uint16_t volume[]) { - if (!pa.sink || !pa.sinkIndex) - return; - struct pa_cvolume v = { .channels = channels }; for(int i = 0; i < channels; ++i) v.values[i] = pa_sw_volume_from_linear( - 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787); + max(0.0, + 9.3234e-7 * pow(1.000211902, volume[i]) - 0.000172787)); pa_threaded_mainloop_lock(pa.loop); - pulseaudio_unrefOperation(pa_context_set_sink_input_volume( - pa.context, pa.sinkIndex, &v, NULL, NULL)); + if (pa.sink && pa.sinkIndex != PA_INVALID_INDEX) + pulseaudio_trackControlOperation(pa_context_set_sink_input_volume( + pa.context, pa.sinkIndex, &v, pulseaudio_contextControl_cb, NULL)); pa_threaded_mainloop_unlock(pa.loop); } static void pulseaudio_mute(bool mute) { - if (!pa.sink || !pa.sinkIndex || pa.sinkMuted == mute) - return; - - pa.sinkMuted = mute; pa_threaded_mainloop_lock(pa.loop); - pulseaudio_unrefOperation(pa_context_set_sink_input_mute( - pa.context, pa.sinkIndex, mute, NULL, NULL)); + if (pa.sink && pa.sinkIndex != PA_INVALID_INDEX) + pulseaudio_trackControlOperation(pa_context_set_sink_input_mute( + pa.context, pa.sinkIndex, mute, pulseaudio_contextControl_cb, NULL)); pa_threaded_mainloop_unlock(pa.loop); } @@ -671,8 +973,53 @@ static bool pulseaudio_setRate(double * ratio) return false; pa.sinkRequestedRate = (uint32_t)llround(requestedRate); - const uint32_t scheduledRate = pa.sinkRateOperation ? - pa.sinkPendingRate : pa.sinkRequestedRate; + + uint32_t scheduledRate; + if (pa.sinkRateOperation) + { + scheduledRate = pa.sinkPendingRate; + if (pa.sinkRequestedRate == scheduledRate) + pa.sinkRateDeferredSince = 0; + else if (!pa.sinkRateDeferredSince || + pa.sinkDeferredRate != pa.sinkRequestedRate) + { + pa.sinkDeferredRate = pa.sinkRequestedRate; + pa.sinkRateDeferredSince = (int64_t)nanotime(); + } + } + else + { + scheduledRate = pa.sinkAppliedRate; + pa.sinkRateUpdateArmed = false; + if (pa.sinkRequestedRate == scheduledRate) + pa.sinkRateDeferredSince = 0; + else + { + const int64_t now = (int64_t)nanotime(); + if (!pa.sinkRateDeferredSince || + pa.sinkDeferredRate != pa.sinkRequestedRate) + { + pa.sinkDeferredRate = pa.sinkRequestedRate; + pa.sinkRateDeferredSince = now; + } + + const uint32_t difference = pa.sinkRequestedRate > scheduledRate ? + pa.sinkRequestedRate - scheduledRate : + scheduledRate - pa.sinkRequestedRate; + /* Coalesce controller noise around an integer-Hz boundary while still + * applying a persistent one-Hz correction. Larger corrections only + * wait for the operation-rate limit. */ + if (now >= pa.sinkNextRateUpdate && + (difference > PULSEAUDIO_RATE_UPDATE_DEADBAND_HZ || + now - pa.sinkRateDeferredSince >= + PULSEAUDIO_RATE_UPDATE_MAX_HOLD_NS)) + { + pa.sinkRateUpdateArmed = true; + scheduledRate = pa.sinkRequestedRate; + } + } + } + if (!scheduledRate) return false; @@ -682,12 +1029,15 @@ static bool pulseaudio_setRate(double * ratio) static uint64_t pulseaudio_latency(void) { - const int64_t deadline = atomic_load_explicit( - &pa.sinkPresentationDeadline, memory_order_acquire); - if (deadline <= 0) + atomic_store_explicit( + &pa.sinkLatencyUpdateRequested, true, memory_order_release); + + const int64_t latencyNs = atomic_load_explicit( + &pa.sinkLatencyNs, memory_order_acquire); + if (latencyNs <= 0) return 0; - return max(INT64_C(0), deadline - (int64_t)nanotime()) / 1000; + return latencyNs / 1000; } struct LG_AudioDevOps LGAD_PulseAudio = diff --git a/client/include/interface/audio.h b/client/include/interface/audio.h index d1564736..2b80ac6b 100644 --- a/client/include/interface/audio.h +++ b/client/include/interface/audio.h @@ -106,7 +106,8 @@ typedef struct LG_AudioEventOps { /* Format and clock pointers are borrowed for the duration of each call. * A NULL source clock indicates that the provider has no usable clock. - * Providers must serialize event delivery for an attachment. Stream + * Providers must serialize event delivery within each stream direction. + * Playback and recording events may be delivered concurrently. Stream * generations are nonzero and uniquely identify each stream instance. */ void (*playbackStart)(void * opaque, uint32_t generation, const LG_AudioFormat * format, const LG_AudioClock * sourceClock); @@ -163,7 +164,9 @@ typedef struct LG_AudioOps * audio callback with the measured playback device clock. Its position uses * the device's independent output-frame timeline and its time includes the * backend's estimated presentation latency. targetRate is the source frame - * rate requested by the client's buffer controller. */ + * rate requested by the client's buffer controller. Return false while + * active rate feedback is unavailable; sustained failure makes the client + * restart playback with local rate control. */ bool (*clockFeedback)(void * opaque, uint32_t generation, const LG_AudioClock * playbackClock, double targetRate); } diff --git a/client/include/interface/audiodev.h b/client/include/interface/audiodev.h index 548dfec0..b08bf404 100644 --- a/client/include/interface/audiodev.h +++ b/client/include/interface/audiodev.h @@ -28,7 +28,9 @@ #include "interface/audio.h" typedef int (*LG_AudioPullFn)(uint8_t * dst, int frames); -typedef void (*LG_AudioPushFn)(uint8_t * src, int frames); +typedef bool (*LG_AudioPushFn)(uint8_t * src, int frames, + const LG_AudioClock * sourceClock); +typedef void (*LG_AudioFailureFn)(uint32_t cookie); struct LG_AudioDevOps { @@ -55,10 +57,13 @@ struct LG_AudioDevOps bool requestResampler, bool * resamplerEnabled, int * maxPeriodFrames, int * startFrames, LG_AudioPullFn pullFn); - /* called when there is data available to start playback */ - void (*start)(void); + /* Called when there is data available to start playback. failureFn may + * report an asynchronous failure using the supplied cookie. Returning + * false means the failure callback is already quiescent. */ + bool (*start)(LG_AudioFailureFn failureFn, uint32_t cookie); - /* called when the source reports the audio stream has stopped */ + /* Called when the source reports the audio stream has stopped. This must + * synchronously quiesce the failure callback before returning. */ void (*stop)(void); /* [optional] called to set the volume of the channels */ @@ -80,10 +85,17 @@ struct LG_AudioDevOps struct { - /* start the record stream using the requested interleaved format */ - void (*start)(const LG_AudioFormat * format, LG_AudioPushFn pushFn); + /* Start the record stream using the requested interleaved format. + * sourceClock is borrowed for the duration of pushFn and describes the + * first frame. It is NULL when the backend has no source clock. pushFn + * returns false when the active provider rejects the frames. + * failureFn may report an asynchronous failure using the supplied cookie. + * Returning false means both callbacks are already quiescent. */ + bool (*start)(const LG_AudioFormat * format, LG_AudioPushFn pushFn, + LG_AudioFailureFn failureFn, uint32_t cookie); - /* called when the source reports the audio stream has stopped */ + /* Called when the source reports the audio stream has stopped. This must + * synchronously quiesce both callbacks before returning. */ void (*stop)(void); /* [optional] called to set the volume of the channels */ diff --git a/client/src/audio.c b/client/src/audio.c index 26c578cb..2211d640 100644 --- a/client/src/audio.c +++ b/client/src/audio.c @@ -32,10 +32,12 @@ #include "dynamic/audiodev.h" +#include #include #include #include #include +#include #include #include #include @@ -49,6 +51,10 @@ #define PLAYBACK_MAX_RATE_CORRECTION 0.005 #define PLAYBACK_MAX_RATE_SLEW_PER_SEC 0.005 #define PLAYBACK_MAX_JITTER_SEC 0.1 +#define PLAYBACK_JITTER_DECAY_SEC 10.0 +#define AUDIO_START_RETRY_MIN_NS INT64_C(5000000) +#define AUDIO_START_RETRY_MAX_NS INT64_C(250000000) +#define AUDIO_RETRY_RESET_NS INT64_C(1000000000) #define PLAYBACK_PHASE_BASELINE_TIME_SEC 5.0 #define PLAYBACK_PHASE_RESERVE_DECAY_SEC 60.0 /* libsamplerate does not expose its buffered-frame delay. SRC_SINC_FASTEST @@ -65,7 +71,9 @@ #define PLAYBACK_DEVICE_RATE_STABLE_SEC 2.0 #define PLAYBACK_DEVICE_RATE_STABLE_DELTA_PPM 50.0 #define PLAYBACK_DEVICE_RATE_MAX_ACQUIRE_SEC 20.0 +#define PLAYBACK_MAX_SOURCE_PACKET_MS 100 #define PLAYBACK_FEEDBACK_INTERVAL_NS INT64_C(1000000) +#define PLAYBACK_FEEDBACK_FAILURE_NS INT64_C(100000000) #define PLAYBACK_GRAPH_INTERVAL_NS INT64_C(25000000) typedef enum @@ -80,6 +88,15 @@ typedef enum } StreamState; +typedef enum +{ + PLAYBACK_DATA_DROP, + PLAYBACK_DATA_PROCESSED, + PLAYBACK_DATA_RETRY, + PLAYBACK_DATA_RETRY_NOW +} +PlaybackDataResult; + typedef enum { PLAYBACK_RATE_SOFTWARE, @@ -88,6 +105,15 @@ typedef enum } PlaybackRateControl; +typedef enum +{ + PLAYBACK_DIAGNOSTIC_REGISTER_GRAPH = 1U << 0, + PLAYBACK_DIAGNOSTIC_INVALIDATE = 1U << 1, + PLAYBACK_DIAGNOSTIC_START_LOG = 1U << 2, + PLAYBACK_DIAGNOSTIC_SYNC_LOG = 1U << 3, +} +PlaybackDiagnosticFlags; + #define STREAM_ACTIVE(state) \ (state == STREAM_STATE_RUN || \ state == STREAM_STATE_KEEP_ALIVE || \ @@ -95,6 +121,8 @@ PlaybackRateControl; #define PLAYBACK_CALLBACK_DISABLED (UINT32_C(1) << 31) #define PLAYBACK_CALLBACK_COUNT_MASK (PLAYBACK_CALLBACK_DISABLED - 1) +#define ACTIVE_CALLBACK_WAITING (UINT32_C(1) << 31) +#define ACTIVE_CALLBACK_COUNT_MASK (ACTIVE_CALLBACK_WAITING - 1) typedef struct { @@ -129,6 +157,7 @@ typedef struct float * framesOut; int framesInSize; int framesOutSize; + int maxSourcePacketFrames; int64_t inputPosition; int64_t outputPosition; @@ -195,11 +224,43 @@ typedef struct } PlaybackDeviceTiming; +typedef struct +{ + double targetLatencyMs; + double queuedLatencyMs; +} +PlaybackStartDiagnostics; + +typedef struct +{ + double softwareLatencyMs; + double targetLatencyMs; + double sourcePpm; + double devicePpm; + double controlPpm; + double jitterMs; + unsigned int underruns; + unsigned int overruns; + PlaybackRateControl rateControl; +} +PlaybackSyncDiagnostics; + +typedef struct +{ + unsigned int epoch; + unsigned int pending; + float graphMax; + PlaybackStartDiagnostics start; + PlaybackSyncDiagnostics sync; +} +PlaybackDiagnostics; + typedef struct { const LG_AudioOps * ops; void * opaque; bool available; + uint32_t epoch; uint32_t generation; } AudioBinding; @@ -207,9 +268,14 @@ AudioBinding; typedef struct { struct LG_AudioDevOps * audioDev; + atomic_bool ready; - LG_Lock providerLock; - LG_RWLock activeLock; + LG_Lock bindingLock; + LG_Lock providerLock; + LG_RWLock activeLock; + uint32_t nextBindingEpoch; + atomic_uint activeCallbacks; + LGEvent * activeIdle; AudioBinding fallback; AudioBinding transport; AudioBinding active; @@ -217,25 +283,54 @@ typedef struct struct { LG_Lock sourceLock; + LG_Lock deviceLock; + struct + { + sem_t wake; + bool wakeInitialized; + atomic_bool wakePending; + LGThread * thread; + atomic_bool stop; + } + worker; _Atomic(StreamState) state; atomic_uint callbackState; + sem_t callbackIdle; + bool callbackIdleInitialized; atomic_uint streamGeneration; - int volumeChannels; - uint16_t volume[LG_AUDIO_MAX_CHANNELS]; - bool mute; - LG_AudioFormat format; - LG_AudioFormat lastFormat; - bool lastFormatValid; - int channels; - int sampleRate; - int stride; - bool convertToFloat; - int deviceMaxPeriodFrames; - int deviceStartFrames; - int targetStartFrames; - int startupLowWaterFrames; - int64_t startupPacketDeadline; - int64_t startupPacketPeriod; + uint64_t requestSerial; + uint32_t requestedGeneration; + LG_AudioFormat requestedFormat; + bool requestedFormatValid; + bool requestedProviderRateControl; + bool forceSoftwareResampler; + bool startPending; + bool startInProgress; + unsigned int startFailures; + int64_t nextStartRetry; + atomic_uint activeAttemptSerial; + atomic_uint failedAttemptSerial; + uint32_t nextAttemptSerial; + uint64_t backendRequestSerial; + int64_t backendStartTime; + uint64_t controlSerial; + bool controlsPending; + int volumeChannels; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool mute; + LG_AudioFormat format; + LG_AudioFormat lastFormat; + bool lastFormatValid; + int channels; + int sampleRate; + int stride; + bool convertToFloat; + int deviceMaxPeriodFrames; + int deviceStartFrames; + int targetStartFrames; + int startupLowWaterFrames; + int64_t startupPacketDeadline; + int64_t startupPacketPeriod; PlaybackRateControl rateControl; bool lastProviderRateControl; _Atomic(double) backendResampleRatio; @@ -244,8 +339,12 @@ typedef struct PlaybackDeviceTiming deviceTiming; atomic_uint underruns; - RingBuffer timings; - GraphHandle graph; + RingBuffer timings; + GraphHandle graph; + atomic_uint diagnosticsEpoch; + atomic_bool graphReady; + bool graphRegistrationAttempted; + PlaybackDiagnostics diagnostics; /* These two structs contain data specifically for use in the device and * source data threads respectively. Keep them on separate cache lines to @@ -257,33 +356,47 @@ typedef struct struct { - LG_Lock lock; - atomic_uint streamGeneration; - bool shuttingDown; - bool requested; - bool started; - int volumeChannels; - uint16_t volume[LG_AUDIO_MAX_CHANNELS]; - bool mute; - LG_AudioFormat format; - LG_AudioFormat lastFormat; - MsgBoxHandle confirmHandle; - uint64_t confirmGeneration; - bool confirmPending; - LG_AudioFormat confirmFormat; + LG_RWLock lock; + LGEvent * wake; + LGThread * thread; + atomic_bool workerAlive; + atomic_uint deliverySerial; + atomic_uint failedAttemptSerial; + uint32_t nextAttemptSerial; + AudioBinding deliveryBinding; + uint32_t deliveryGeneration; + bool shuttingDown; + bool requested; + bool enabled; + unsigned int startFailures; + int64_t nextStartRetry; + uint64_t requestSerial; + AudioBinding requestedBinding; + uint32_t requestedGeneration; + LG_AudioFormat requestedFormat; + int volumeChannels; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool mute; + uint64_t controlSerial; + MsgBoxHandle confirmHandle; + uint64_t confirmGeneration; + bool confirmPending; } record; struct { - LG_Lock lock; - LGEvent * event; - LGThread * thread; + LG_Lock lock; + sem_t wake; + bool wakeInitialized; + atomic_bool wakePending; + LGThread * thread; atomic_bool stop; bool pending; const LG_AudioOps * ops; void * opaque; + uint32_t bindingEpoch; uint32_t bindingGeneration; uint32_t generation; LG_AudioClock clock; @@ -703,7 +816,8 @@ static void playbackPublishDeviceTiming( double outputPosition) { PlaybackDeviceTiming * timing = &audio.playback.deviceTiming; - atomic_fetch_add_explicit(&timing->sequence, 1, memory_order_relaxed); + /* Publish the odd writer marker before any snapshot field. */ + atomic_fetch_add_explicit(&timing->sequence, 1, memory_order_seq_cst); atomic_store_explicit( &timing->periodFrames, periodFrames, memory_order_relaxed); atomic_store_explicit(&timing->time, time, memory_order_relaxed); @@ -824,10 +938,14 @@ static int64_t playbackMapMediaTime(PlaybackSourceData * sourceData, } static void playbackStop(void); +static bool playbackEnsureConversionBuffers( + PlaybackSourceData * sourceData, int frames); static MsgBoxHandle recordCancelConfirmLocked(void); -static void realRecordStartLocked(const LG_AudioFormat * format); -static void realRecordStopLocked(void); -static void recordStop(void); +static void recordStop( + const AudioBinding * binding, uint32_t generation); +static bool eventBegin( + const AudioBinding * binding, AudioBinding * active); +static void eventEnd(void); static StreamState playbackGetState(void) { @@ -861,8 +979,12 @@ static bool playbackCallbackEnter(void) static void playbackCallbackExit(void) { - atomic_fetch_sub_explicit( + const unsigned int previous = atomic_fetch_sub_explicit( &audio.playback.callbackState, 1, memory_order_release); + if ((previous & PLAYBACK_CALLBACK_DISABLED) && + (previous & PLAYBACK_CALLBACK_COUNT_MASK) == 1 && + audio.playback.callbackIdleInitialized) + sem_post(&audio.playback.callbackIdle); } static void playbackDisableCallbacks(void) @@ -877,12 +999,91 @@ static void playbackWaitForCallbacks(void) while((atomic_load_explicit( &audio.playback.callbackState, memory_order_acquire) & PLAYBACK_CALLBACK_COUNT_MASK) != 0) - ; + { + int result; + do + result = sem_wait(&audio.playback.callbackIdle); + while (result < 0 && errno == EINTR); + + if (result < 0) + break; + } +} + +static void playbackWorkerWake(void) +{ + if (!audio.playback.worker.wakeInitialized || + !audio.playback.worker.thread || + atomic_exchange_explicit( + &audio.playback.worker.wakePending, true, memory_order_acq_rel)) + return; + + if (sem_post(&audio.playback.worker.wake) < 0 && errno != EOVERFLOW) + atomic_store_explicit( + &audio.playback.worker.wakePending, false, memory_order_release); +} + +static void feedbackWorkerWake(void) +{ + if (!audio.feedback.wakeInitialized || !audio.feedback.thread || + atomic_exchange_explicit( + &audio.feedback.wakePending, true, memory_order_acq_rel)) + return; + + if (sem_post(&audio.feedback.wake) < 0 && errno != EOVERFLOW) + atomic_store_explicit( + &audio.feedback.wakePending, false, memory_order_release); +} + +static void playbackQueueStop(void) +{ + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + playbackDisableCallbacks(); + playbackSetState(STREAM_STATE_STOP_PENDING); + playbackWorkerWake(); +} + +static void playbackQueueSourceStop(void) +{ + atomic_store_explicit( + &audio.playback.activeAttemptSerial, 0, memory_order_release); + playbackQueueStop(); +} + +static void playbackBackendFailed(uint32_t attemptSerial) +{ + unsigned int expected = attemptSerial; + if (!attemptSerial || !atomic_compare_exchange_strong_explicit( + &audio.playback.activeAttemptSerial, &expected, 0, + memory_order_acq_rel, memory_order_acquire)) + return; + + atomic_store_explicit(&audio.playback.failedAttemptSerial, + attemptSerial, memory_order_release); + playbackQueueStop(); +} + +/* sourceLock must be held. */ +static uint32_t playbackArmBackend(void) +{ + if (!++audio.playback.nextAttemptSerial) + ++audio.playback.nextAttemptSerial; + + const uint32_t attemptSerial = audio.playback.nextAttemptSerial; + atomic_store_explicit( + &audio.playback.failedAttemptSerial, 0, memory_order_release); + audio.playback.backendRequestSerial = audio.playback.requestSerial; + audio.playback.backendStartTime = 0; + atomic_store_explicit(&audio.playback.activeAttemptSerial, + attemptSerial, memory_order_release); + return attemptSerial; } bool lgAudio_supportsPlayback(void) { - return audio.audioDev && audio.audioDev->playback.start; + return atomic_load_explicit(&audio.ready, memory_order_acquire) && + audio.audioDev && audio.audioDev->playback.start; } static const char * audioGraphFormatFn(const char * name, @@ -895,15 +1096,39 @@ static const char * audioGraphFormatFn(const char * name, return title; } +/* sourceLock must be held. */ +static PlaybackDiagnostics * playbackDiagnosticsLocked(void) +{ + PlaybackDiagnostics * diagnostics = &audio.playback.diagnostics; + const unsigned int epoch = atomic_load_explicit( + &audio.playback.diagnosticsEpoch, memory_order_acquire); + if (diagnostics->epoch != epoch) + *diagnostics = (PlaybackDiagnostics) { .epoch = epoch }; + + return diagnostics; +} + static void playbackStop(void) { - if (playbackGetState() == STREAM_STATE_STOP) + const bool alreadyStopped = playbackGetState() == STREAM_STATE_STOP; + playbackDisableCallbacks(); + atomic_store_explicit( + &audio.playback.activeAttemptSerial, 0, memory_order_release); + if (!alreadyStopped) + audio.audioDev->playback.stop(); + playbackWaitForCallbacks(); + atomic_store_explicit( + &audio.playback.failedAttemptSerial, 0, memory_order_release); + audio.playback.backendRequestSerial = 0; + audio.playback.backendStartTime = 0; + + if (alreadyStopped) return; - playbackDisableCallbacks(); - audio.audioDev->playback.stop(); - playbackWaitForCallbacks(); - + atomic_store_explicit( + &audio.playback.graphReady, false, memory_order_release); + atomic_fetch_add_explicit( + &audio.playback.diagnosticsEpoch, 1, memory_order_release); playbackSetState(STREAM_STATE_STOP); ringbuffer_free(&audio.playback.buffer); audio.playback.sourceData.src = src_delete(audio.playback.sourceData.src); @@ -917,14 +1142,16 @@ static void playbackStop(void) audio.playback.sourceData.framesInSize = 0; audio.playback.sourceData.framesOutSize = 0; } + audio.playback.sourceData.maxSourcePacketFrames = 0; if (audio.playback.timings) { if (audio.playback.graph) app_unregisterGraph(audio.playback.graph); - audio.playback.graph = NULL; ringbuffer_free(&audio.playback.timings); } + audio.playback.graph = NULL; + audio.playback.graphRegistrationAttempted = false; } static int playbackPullFrames(uint8_t * dst, int frames) @@ -998,7 +1225,10 @@ static int playbackPullFrames(uint8_t * dst, int frames) data->startupSilenceFrames = -offset; } - playbackSetState(STREAM_STATE_RUN); + StreamState expected = STREAM_STATE_SETUP_DEVICE; + atomic_compare_exchange_strong_explicit( + &audio.playback.state, &expected, STREAM_STATE_RUN, + memory_order_acq_rel, memory_order_acquire); } /* Timestamp the dequeue boundary before the current pull. The logical @@ -1035,6 +1265,8 @@ static int playbackPullFrames(uint8_t * dst, int frames) // Close the stream if nothing has played for a while if (audio.playback.buffer && + audio.playback.worker.wakeInitialized && + audio.playback.worker.thread && playbackGetState() == STREAM_STATE_KEEP_ALIVE) { int stopTimeSec = 30; @@ -1047,7 +1279,7 @@ static int playbackPullFrames(uint8_t * dst, int frames) memory_order_acq_rel, memory_order_acquire)) { playbackDisableCallbacks(); - audio.audioDev->playback.stop(); + playbackWorkerWake(); frames = 0; } } @@ -1074,25 +1306,26 @@ static bool playbackSetupDevice(const LG_AudioFormat * format, audio.playback.deviceStartFrames >= 0; } -static void playbackStart(const LG_AudioFormat * format, - const LG_AudioClock * sourceClock, bool providerRateControl) +static bool playbackStart(const LG_AudioFormat * format, + const LG_AudioClock * sourceClock, bool providerRateControl, + bool forceSoftwareResampler) { if (!audio.audioDev) - return; + return false; if (!audioFormatValid(format)) { DEBUG_ERROR("Invalid playback format"); if (playbackGetState() != STREAM_STATE_STOP) playbackStop(); - return; + return false; } const int channels = format->channelCount; const int sampleRate = format->sampleRate; StreamState state = playbackGetState(); - if (state == STREAM_STATE_KEEP_ALIVE && + if (!forceSoftwareResampler && state == STREAM_STATE_KEEP_ALIVE && audio.playback.lastFormatValid && audio.playback.lastProviderRateControl == providerRateControl && audioFormatEqual(format, &audio.playback.lastFormat)) @@ -1104,7 +1337,7 @@ static void playbackStart(const LG_AudioFormat * format, { playbackPrepareMediaClock( &audio.playback.sourceData, sourceClock); - return; + return true; } state = expected; @@ -1120,6 +1353,8 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.channels = channels; audio.playback.sampleRate = sampleRate; + audio.playback.sourceData.maxSourcePacketFrames = max( + (sampleRate * PLAYBACK_MAX_SOURCE_PACKET_MS + 999) / 1000, 1); playbackSetState(STREAM_STATE_SETUP_SOURCE); audio.playback.deviceData.nextPosition = 0; @@ -1180,6 +1415,7 @@ static void playbackStart(const LG_AudioFormat * format, audio.playback.startupPacketDeadline = 0; audio.playback.startupPacketPeriod = 0; const bool requestBackendResampler = !providerRateControl && + !forceSoftwareResampler && g_params.audioResampler != AUDIO_RESAMPLER_LIBSAMPLERATE; LG_AudioFormat deviceFormat = *format; @@ -1211,7 +1447,7 @@ static void playbackStart(const LG_AudioFormat * format, { DEBUG_ERROR("Failed to configure audio playback device"); playbackStop(); - return; + return false; } audio.playback.stride = channels * @@ -1223,12 +1459,21 @@ static void playbackStart(const LG_AudioFormat * format, PLAYBACK_RATE_PROVIDER : backendResampler ? PLAYBACK_RATE_BACKEND : PLAYBACK_RATE_SOFTWARE; + const int conversionBufferFrames = max(requestedPeriodFrames, + audio.playback.sourceData.maxSourcePacketFrames); + if (!playbackEnsureConversionBuffers( + &audio.playback.sourceData, conversionBufferFrames)) + { + playbackStop(); + return false; + } + audio.playback.buffer = ringbuffer_newUnbounded( sampleRate, audio.playback.stride); if (!audio.playback.buffer) { playbackStop(); - return; + return false; } if (g_params.audioResampler == AUDIO_RESAMPLER_BACKEND && @@ -1245,7 +1490,7 @@ static void playbackStart(const LG_AudioFormat * format, { DEBUG_ERROR("Failed to create resampler: %s", src_strerror(srcError)); playbackStop(); - return; + return false; } } else @@ -1266,22 +1511,383 @@ static void playbackStart(const LG_AudioFormat * format, break; } - // if a volume level was stored, set it before we return - if (audio.playback.volumeChannels) - audio.audioDev->playback.volume( - audio.playback.volumeChannels, - audio.playback.volume); - - // set the inital mute state - if (audio.audioDev->playback.mute) - audio.audioDev->playback.mute(audio.playback.mute); - // Set up synchronization instrumentation only when explicitly requested. if (g_params.audioDebug) audio.playback.timings = ringbuffer_new(1200, sizeof(float)); atomic_store_explicit( &audio.playback.callbackState, 0, memory_order_release); + return true; +} + +static int64_t audioStartRetryDelay(unsigned int failures) +{ + const unsigned int shift = min(failures, 6U); + return min(AUDIO_START_RETRY_MIN_NS << shift, + AUDIO_START_RETRY_MAX_NS); +} + +/* sourceLock must be held. */ +static bool playbackDelayStart(void) +{ + const unsigned int failures = audio.playback.startFailures; + audio.playback.nextStartRetry = nanotime() + + audioStartRetryDelay(failures); + if (audio.playback.startFailures < 7U) + ++audio.playback.startFailures; + + return failures == 0U || failures == 6U; +} + +/* sourceLock must be held. */ +static void playbackScheduleStart(void) +{ + if (!audio.playback.requestedGeneration || + !audio.playback.requestedFormatValid || + audio.playback.startPending || + audio.playback.startInProgress || + (atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == + audio.playback.requestedGeneration && + playbackGetState() != STREAM_STATE_STOP) || + nanotime() < audio.playback.nextStartRetry) + return; + + audio.playback.startPending = true; + playbackWorkerWake(); +} + +static void playbackProcessControls(void) +{ + uint64_t controlSerial; + uint32_t generation; + int volumeChannels; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool mute; + + LG_LOCK(audio.playback.sourceLock); + generation = atomic_load_explicit( + &audio.playback.streamGeneration, memory_order_acquire); + if (!audio.playback.controlsPending || !generation || + generation != audio.playback.requestedGeneration) + { + LG_UNLOCK(audio.playback.sourceLock); + return; + } + + controlSerial = audio.playback.controlSerial; + volumeChannels = audio.playback.volumeChannels; + if (volumeChannels) + memcpy(volume, audio.playback.volume, + sizeof(*volume) * volumeChannels); + mute = audio.playback.mute; + LG_UNLOCK(audio.playback.sourceLock); + + bool applied = false; + LG_LOCK(audio.playback.deviceLock); + const StreamState state = playbackGetState(); + if (audio.audioDev && state != STREAM_STATE_STOP && + state != STREAM_STATE_STOP_PENDING && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + { + if (volumeChannels && audio.audioDev->playback.volume) + audio.audioDev->playback.volume(volumeChannels, volume); + + if (audio.audioDev->playback.mute) + audio.audioDev->playback.mute(mute); + applied = true; + } + LG_UNLOCK(audio.playback.deviceLock); + + if (!applied) + return; + + LG_LOCK(audio.playback.sourceLock); + if (audio.playback.requestedGeneration == generation && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation && + audio.playback.controlSerial == controlSerial) + audio.playback.controlsPending = false; + LG_UNLOCK(audio.playback.sourceLock); +} + +static void playbackProcessStopPending(void) +{ + bool pending = false; + bool reportFailure = false; + uint32_t failedAttempt = 0; + uint64_t backendRequestSerial = 0; + int64_t backendStartTime = 0; + + LG_LOCK(audio.playback.sourceLock); + if (audio.audioDev && + playbackGetState() == STREAM_STATE_STOP_PENDING) + { + failedAttempt = atomic_load_explicit( + &audio.playback.failedAttemptSerial, memory_order_acquire); + backendRequestSerial = audio.playback.backendRequestSerial; + backendStartTime = audio.playback.backendStartTime; + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + pending = true; + } + LG_UNLOCK(audio.playback.sourceLock); + + if (!pending) + return; + + LG_LOCK(audio.playback.deviceLock); + playbackStop(); + LG_UNLOCK(audio.playback.deviceLock); + + LG_LOCK(audio.playback.sourceLock); + if (failedAttempt && audio.playback.requestedGeneration && + audio.playback.requestedFormatValid && + backendRequestSerial == audio.playback.requestSerial) + { + if (backendStartTime && + nanotime() - backendStartTime >= AUDIO_RETRY_RESET_NS) + { + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + } + + audio.playback.startPending = false; + reportFailure = playbackDelayStart(); + } + LG_UNLOCK(audio.playback.sourceLock); + + if (reportFailure) + DEBUG_ERROR("Audio playback device failed; retrying"); +} + +static void playbackProcessStart(void) +{ + LG_AudioFormat format; + uint64_t requestSerial; + uint32_t generation; + bool providerRateControl; + bool forceSoftwareResampler; + + LG_LOCK(audio.playback.sourceLock); + if (!audio.playback.startPending || + audio.playback.startInProgress || + !audio.playback.requestedGeneration || + !audio.playback.requestedFormatValid || + nanotime() < audio.playback.nextStartRetry) + { + LG_UNLOCK(audio.playback.sourceLock); + return; + } + + audio.playback.startPending = false; + audio.playback.startInProgress = true; + format = audio.playback.requestedFormat; + requestSerial = audio.playback.requestSerial; + generation = audio.playback.requestedGeneration; + providerRateControl = + audio.playback.requestedProviderRateControl; + forceSoftwareResampler = + audio.playback.forceSoftwareResampler; + LG_UNLOCK(audio.playback.sourceLock); + + LG_LOCK(audio.playback.deviceLock); + const bool started = playbackStart(&format, NULL, + providerRateControl, forceSoftwareResampler); + LG_UNLOCK(audio.playback.deviceLock); + + bool stopStale = false; + bool wake = false; + LG_LOCK(audio.playback.sourceLock); + const bool current = + requestSerial == audio.playback.requestSerial && + generation == audio.playback.requestedGeneration && + audio.playback.requestedFormatValid && + audioFormatEqual(&format, &audio.playback.requestedFormat) && + providerRateControl == + audio.playback.requestedProviderRateControl && + forceSoftwareResampler == + audio.playback.forceSoftwareResampler; + + audio.playback.startInProgress = false; + if (started && current) + { + audio.playback.startPending = false; + LG_LOCK(audio.playback.deviceLock); + if (playbackGetState() != STREAM_STATE_STOP_PENDING) + { + if (atomic_load_explicit(&audio.playback.activeAttemptSerial, + memory_order_acquire)) + audio.playback.backendRequestSerial = requestSerial; + atomic_store_explicit(&audio.playback.streamGeneration, + generation, memory_order_release); + audio.playback.controlsPending = true; + audio.playback.nextStartRetry = 0; + } + LG_UNLOCK(audio.playback.deviceLock); + } + else + { + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + if (started) + stopStale = true; + else if (current) + playbackDelayStart(); + } + + wake = audio.playback.startPending; + LG_UNLOCK(audio.playback.sourceLock); + + if (stopStale) + { + LG_LOCK(audio.playback.deviceLock); + playbackStop(); + LG_UNLOCK(audio.playback.deviceLock); + } + + if (wake) + playbackWorkerWake(); +} + +static void playbackProcessDeviceStart(void) +{ + uint64_t requestSerial = 0; + uint32_t generation = 0; + uint32_t attemptSerial = 0; + + LG_LOCK(audio.playback.sourceLock); + const StreamState state = playbackGetState(); + attemptSerial = atomic_load_explicit( + &audio.playback.activeAttemptSerial, memory_order_acquire); + generation = atomic_load_explicit( + &audio.playback.streamGeneration, memory_order_acquire); + if (audio.audioDev && state == STREAM_STATE_SETUP_DEVICE && + attemptSerial && !audio.playback.backendStartTime && generation && + generation == audio.playback.requestedGeneration) + requestSerial = audio.playback.requestSerial; + else + attemptSerial = 0; + LG_UNLOCK(audio.playback.sourceLock); + + if (!attemptSerial) + return; + + bool started = false; + LG_LOCK(audio.playback.deviceLock); + if (audio.audioDev && + playbackGetState() == STREAM_STATE_SETUP_DEVICE && + atomic_load_explicit(&audio.playback.activeAttemptSerial, + memory_order_acquire) == attemptSerial && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + started = audio.audioDev->playback.start( + playbackBackendFailed, attemptSerial); + LG_UNLOCK(audio.playback.deviceLock); + + LG_LOCK(audio.playback.sourceLock); + const StreamState completedState = playbackGetState(); + const bool attemptCurrent = attemptSerial == atomic_load_explicit( + &audio.playback.activeAttemptSerial, memory_order_acquire); + const bool current = started && + attemptCurrent && + requestSerial == audio.playback.requestSerial && + requestSerial == audio.playback.backendRequestSerial && + generation == audio.playback.requestedGeneration && + generation == atomic_load_explicit( + &audio.playback.streamGeneration, memory_order_acquire) && + (completedState == STREAM_STATE_SETUP_DEVICE || + completedState == STREAM_STATE_RUN); + const bool keepAlive = started && attemptCurrent && + completedState == STREAM_STATE_KEEP_ALIVE; + if (current || keepAlive) + audio.playback.backendStartTime = nanotime(); + LG_UNLOCK(audio.playback.sourceLock); + + if (!current && !keepAlive) + playbackBackendFailed(attemptSerial); +} + +static void playbackProcessDiagnostics(void) +{ + PlaybackDiagnostics diagnostics; + RingBuffer timings; + + LG_LOCK(audio.playback.sourceLock); + if (!audio.playback.diagnostics.pending) + { + LG_UNLOCK(audio.playback.sourceLock); + return; + } + + diagnostics = audio.playback.diagnostics; + timings = audio.playback.timings; + audio.playback.diagnostics.pending = 0; + LG_UNLOCK(audio.playback.sourceLock); + + bool current = false; + double backendLatencyMs = 0.0; + LG_LOCK(audio.playback.deviceLock); + const StreamState state = playbackGetState(); + current = diagnostics.epoch == atomic_load_explicit( + &audio.playback.diagnosticsEpoch, memory_order_acquire) && + state != STREAM_STATE_STOP && state != STREAM_STATE_STOP_PENDING; + if (current) + { + const bool timingsCurrent = timings && + timings == audio.playback.timings; + if ((diagnostics.pending & PLAYBACK_DIAGNOSTIC_REGISTER_GRAPH) && + timingsCurrent && !audio.playback.graph && + !audio.playback.graphRegistrationAttempted) + { + audio.playback.graphRegistrationAttempted = true; + audio.playback.graph = app_registerGraph("PLAYBACK RING", + timings, 0.0f, diagnostics.graphMax, audioGraphFormatFn); + atomic_store_explicit(&audio.playback.graphReady, + audio.playback.graph != NULL, memory_order_release); + } + + if ((diagnostics.pending & PLAYBACK_DIAGNOSTIC_INVALIDATE) && + timingsCurrent && audio.playback.graph) + app_invalidateGraph(audio.playback.graph); + + if ((diagnostics.pending & PLAYBACK_DIAGNOSTIC_SYNC_LOG) && + audio.audioDev && audio.audioDev->playback.latency) + backendLatencyMs = + audio.audioDev->playback.latency() / 1000.0; + } + LG_UNLOCK(audio.playback.deviceLock); + + if (!current) + return; + + if (diagnostics.pending & PLAYBACK_DIAGNOSTIC_START_LOG) + DEBUG_INFO( + "Audio start: %.2f/%.2f ms target/queued", + diagnostics.start.targetLatencyMs, + diagnostics.start.queuedLatencyMs); + + if (diagnostics.pending & PLAYBACK_DIAGNOSTIC_SYNC_LOG) + { + const bool providerControl = + diagnostics.sync.rateControl == PLAYBACK_RATE_PROVIDER; + const char * controlName = providerControl ? "feedback" : + diagnostics.sync.rateControl == PLAYBACK_RATE_BACKEND ? + "backend" : "software"; + const char * sourceRateName = providerControl ? "arrival" : "source"; + + DEBUG_INFO( + "Audio sync: ring %.2f/%.2f ms, backend %.2f ms, " + "%s %+.1f ppm, rates %s/device %+.1f/%+.1f ppm, " + "jitter %.2f ms, xruns %u/%u", + diagnostics.sync.softwareLatencyMs, + diagnostics.sync.targetLatencyMs, + backendLatencyMs, controlName, diagnostics.sync.controlPpm, + sourceRateName, diagnostics.sync.sourcePpm, + diagnostics.sync.devicePpm, diagnostics.sync.jitterMs, + diagnostics.sync.underruns, diagnostics.sync.overruns); + } } static void playbackSourceStop(void) @@ -1289,14 +1895,18 @@ static void playbackSourceStop(void) if (!audio.audioDev) return; - switch (playbackGetState()) + StreamState state = playbackGetState(); + switch (state) { case STREAM_STATE_RUN: case STREAM_STATE_RESUMING: { // Keep the audio device open for a while to reduce startup latency if // playback starts again - playbackSetState(STREAM_STATE_KEEP_ALIVE); + if (!atomic_compare_exchange_strong_explicit( + &audio.playback.state, &state, STREAM_STATE_KEEP_ALIVE, + memory_order_acq_rel, memory_order_acquire)) + break; // Reset the software resampler so it is safe for the next playback if (audio.playback.sourceData.src) @@ -1326,34 +1936,27 @@ static void playbackSourceStop(void) } } -static void playbackVolume(int channels, const uint16_t volume[]) +static int playbackStoreVolume(int channels, const uint16_t volume[]) { if (!audio.audioDev || !audio.audioDev->playback.volume || !g_params.audioSyncVolume) - return; + return 0; // store the values so we can restore the state if the stream is restarted channels = min(ARRAY_LENGTH(audio.playback.volume), channels); memcpy(audio.playback.volume, volume, sizeof(uint16_t) * channels); audio.playback.volumeChannels = channels; - - if (!STREAM_ACTIVE(playbackGetState())) - return; - - audio.audioDev->playback.volume(channels, volume); + return channels; } -static void playbackMute(bool mute) +static bool playbackStoreMute(bool mute) { if (!audio.audioDev || !audio.audioDev->playback.mute) - return; + return false; // store the value so we can restore it if the stream is restarted audio.playback.mute = mute; - if (!STREAM_ACTIVE(playbackGetState())) - return; - - audio.audioDev->playback.mute(mute); + return true; } static double computeDevicePosition(int64_t curTime) @@ -1384,8 +1987,10 @@ static bool playbackEnsureConversionBuffers( if (audio.playback.convertToFloat && frames > sourceData->framesInSize) { + const int capacity = max(frames, + max(sourceData->framesInSize * 2, 64)); float * framesIn = realloc(sourceData->framesIn, - (size_t)frames * audio.playback.channels * sizeof(float)); + (size_t)capacity * audio.playback.channels * sizeof(float)); if (!framesIn) { DEBUG_ERROR("Failed to grow playback input buffer"); @@ -1393,7 +1998,7 @@ static bool playbackEnsureConversionBuffers( } sourceData->framesIn = framesIn; - sourceData->framesInSize = frames; + sourceData->framesInSize = capacity; } if (audio.playback.rateControl == PLAYBACK_RATE_SOFTWARE) @@ -1402,8 +2007,10 @@ static bool playbackEnsureConversionBuffers( (int)ceil(frames * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) + 64; if (framesOut > sourceData->framesOutSize) { + const int capacity = max(framesOut, + max(sourceData->framesOutSize * 2, 64)); float * output = realloc(sourceData->framesOut, - (size_t)framesOut * audio.playback.channels * sizeof(float)); + (size_t)capacity * audio.playback.channels * sizeof(float)); if (!output) { DEBUG_ERROR("Failed to grow playback output buffer"); @@ -1411,7 +2018,7 @@ static bool playbackEnsureConversionBuffers( } sourceData->framesOut = output; - sourceData->framesOutSize = framesOut; + sourceData->framesOutSize = capacity; } } @@ -1463,30 +2070,37 @@ static int playbackSlewBuffer( return advanced; } -static void playbackData(const void * data, size_t frameCount, - const LG_AudioClock * sourceClock) +static PlaybackDataResult playbackData(const void * data, size_t frameCount, + const LG_AudioClock * sourceClock, int64_t arrivalTime) { StreamState state = playbackGetState(); if (state == STREAM_STATE_STOP_PENDING) - { - playbackStop(); - return; - } + return PLAYBACK_DATA_DROP; - if (state == STREAM_STATE_STOP || !audio.audioDev || frameCount == 0) - return; + if (frameCount == 0) + return PLAYBACK_DATA_DROP; + if (state == STREAM_STATE_STOP || !audio.audioDev) + return PLAYBACK_DATA_RETRY_NOW; + + PlaybackSourceData * sourceData = &audio.playback.sourceData; + if (!data || frameCount > (size_t)sourceData->maxSourcePacketFrames) + { + DEBUG_ERROR("Invalid playback packet length: %zu frames", frameCount); + return PLAYBACK_DATA_DROP; + } + const int frames = frameCount; if (audio.playback.rateControl == PLAYBACK_RATE_BACKEND && atomic_exchange_explicit( &audio.playback.backendResamplerFailed, false, memory_order_acq_rel)) { - DEBUG_ERROR("Audio backend resampler failed"); - playbackStop(); - return; + DEBUG_WARN("Audio backend resampler failed; using libsamplerate"); + audio.playback.forceSoftwareResampler = true; + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY_NOW; } - PlaybackSourceData * sourceData = &audio.playback.sourceData; /* Backend resampling changes how many source frames PipeWire requests per * device period. Use the command-normalized output clock for rate matching, * while deviceClock remains in the ring's source-frame domain for latency. */ @@ -1494,23 +2108,15 @@ static void playbackData(const void * data, size_t frameCount, audio.playback.rateControl == PLAYBACK_RATE_BACKEND ? &sourceData->outputClock : &sourceData->deviceClock; const int64_t now = nanotime(); + if (audio.playback.startFailures && + audio.playback.backendStartTime && + now - audio.playback.backendStartTime >= AUDIO_RETRY_RESET_NS) + { + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + } const double nominalFrameSec = 1.0 / audio.playback.sampleRate; - if (!data || frameCount > INT_MAX || - frameCount > (size_t)audio.playback.sampleRate * 2) - { - DEBUG_ERROR("Invalid playback packet length: %zu frames", frameCount); - playbackStop(); - return; - } - const int frames = frameCount; - - if (!playbackEnsureConversionBuffers(sourceData, frames)) - { - playbackStop(); - return; - } - const void * inputFrames = data; if (audio.playback.convertToFloat) { @@ -1519,54 +2125,57 @@ static void playbackData(const void * data, size_t frameCount, audio.playback.format.sampleFormat)) { DEBUG_ERROR("Failed to convert playback samples"); - playbackStop(); - return; + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY; } inputFrames = sourceData->framesIn; } const bool providerRateControl = audio.playback.rateControl == PLAYBACK_RATE_PROVIDER; - /* Do not carry an underrun as buffer debt. Provider feedback is bounded, - * so repaying a guest stall by rate correction would discard resumed audio - * for many seconds. Rebase at the first packet instead. */ - const bool providerUnderrun = - providerRateControl && playbackGetState() == STREAM_STATE_RUN && + /* An unbounded ring represents an underrun by advancing the reader beyond + * the writer. Do not carry that logical debt into resumed playback: bounded + * rate correction would otherwise discard fresh audio for many seconds. */ + const bool bufferUnderrun = + STREAM_ACTIVE(playbackGetState()) && ringbuffer_getCount(audio.playback.buffer) < 0; bool discontinuity = - providerUnderrun || (sourceClock && sourceClock->discontinuity); + bufferUnderrun || (sourceClock && sourceClock->discontinuity); const int64_t packetTime = playbackMapMediaTime(sourceData, sourceClock, frames, - audio.playback.sampleRate, now, &discontinuity); + audio.playback.sampleRate, arrivalTime, &discontinuity); if (sourceData->bufferOverrunPending) { discontinuity = true; sourceData->bufferOverrunPending = false; } - if (sourceData->lastPacketTime != INT64_MIN && + if (!discontinuity && + sourceData->lastPacketTime != INT64_MIN && sourceData->lastArrivalTime != INT64_MIN) { const double mediaDelta = (packetTime - sourceData->lastPacketTime) * 1.0e-9; const double arrivalDelta = - (now - sourceData->lastArrivalTime) * 1.0e-9; + (arrivalTime - sourceData->lastArrivalTime) * 1.0e-9; const double jitter = fabs(arrivalDelta - mediaDelta); /* Keep a slowly decaying peak rather than feeding arrival jitter into the * virtual clock. This lets the buffer absorb real delivery jitter while * the rate controller follows only the source media clock. */ + const double decaySec = max(arrivalDelta, 0.0); sourceData->arrivalJitterSec = min(PLAYBACK_MAX_JITTER_SEC, - max(jitter, sourceData->arrivalJitterSec * 0.999)); + max(jitter, sourceData->arrivalJitterSec * + exp(-decaySec / PLAYBACK_JITTER_DECAY_SEC))); } sourceData->lastPacketTime = packetTime; - sourceData->lastArrivalTime = now; + sourceData->lastArrivalTime = arrivalTime; const bool sourceRateWasValid = sourceData->sourceRateValid; const int64_t sourceRateTimeMs = providerRateControl ? - (now - sourceData->mediaLocalOrigin) / INT64_C(1000000) : + (arrivalTime - sourceData->mediaLocalOrigin) / INT64_C(1000000) : sourceData->mediaTimeMs; playbackSourceRateAdd( sourceData, sourceRateTimeMs, nominalFrameSec); @@ -1578,6 +2187,9 @@ static void playbackData(const void * data, size_t frameCount, frameSec <= nominalFrameSec * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) { + if (!sourceData->sourceRateValid || + sourceData->rateFilterTimeMs == INT64_MIN) + sourceData->rateFilterTimeMs = sourceRateTimeMs; sourceData->sourceRateFrameSec = frameSec; sourceData->sourceRateValid = true; } @@ -1679,6 +2291,17 @@ static void playbackData(const void * data, size_t frameCount, (providerRateControl ? 0.0 : sourceReserveFrames); } + if (discontinuity && sourceData->src) + { + const int error = src_reset(sourceData->src); + if (error) + { + DEBUG_ERROR("Failed to reset resampler: %s", src_strerror(error)); + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY; + } + } + const int maxPeriodFrames = max(audio.playback.deviceMaxPeriodFrames, sourceData->devPeriodFrames); /* The device period, delivery jitter, packet phase, and resampler delay @@ -1710,7 +2333,7 @@ static void playbackData(const void * data, size_t frameCount, double devPosition = DBL_MIN; state = playbackGetState(); - if (providerRateControl && + if ((providerRateControl || bufferUnderrun) && (discontinuity || state == STREAM_STATE_KEEP_ALIVE || state == STREAM_STATE_RESUMING)) @@ -1726,12 +2349,16 @@ static void playbackData(const void * data, size_t frameCount, sourceData->offsetError = 0.0; sourceData->offsetErrorIntegral = 0.0; sourceData->ratioIntegral = 0.0; - if (providerUnderrun) + if (providerRateControl && bufferUnderrun) { sourceData->lastRatio = 1.0; sourceData->nextFeedbackTime = 0; } - playbackSetState(STREAM_STATE_RUN); + if (state == STREAM_STATE_KEEP_ALIVE || + state == STREAM_STATE_RESUMING) + atomic_compare_exchange_strong_explicit( + &audio.playback.state, &state, STREAM_STATE_RUN, + memory_order_acq_rel, memory_order_acquire); } else if ((discontinuity || state == STREAM_STATE_KEEP_ALIVE || @@ -1750,7 +2377,11 @@ static void playbackData(const void * data, size_t frameCount, sourceData->offsetError = 0.0; sourceData->offsetErrorIntegral = 0.0; sourceData->ratioIntegral = 0.0; - playbackSetState(STREAM_STATE_RUN); + if (state == STREAM_STATE_KEEP_ALIVE || + state == STREAM_STATE_RESUMING) + atomic_compare_exchange_strong_explicit( + &audio.playback.state, &state, STREAM_STATE_RUN, + memory_order_acq_rel, memory_order_acquire); } double actualLatencyFrames = 0.0; @@ -1955,15 +2586,15 @@ static void playbackData(const void * data, size_t frameCount, if (error) { DEBUG_ERROR("Resampling failed: %s", src_strerror(error)); - playbackStop(); - return; + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY; } if (srcData.input_frames_used == 0 && srcData.output_frames_gen == 0) { DEBUG_ERROR("Resampler made no progress"); - playbackStop(); - return; + playbackQueueSourceStop(); + return PLAYBACK_DATA_RETRY; } const int outputFrames = playbackAppendFrames( @@ -1995,16 +2626,25 @@ static void playbackData(const void * data, size_t frameCount, if (ringbuffer_getCount(audio.playback.buffer) >= audio.playback.targetStartFrames) { - if (audio.playback.timings && !audio.playback.graph) + if (g_params.audioDebug) { - const float graphMax = + PlaybackDiagnostics * diagnostics = playbackDiagnosticsLocked(); + if (audio.playback.timings) + { + diagnostics->graphMax = + targetLatencyFrames * 1000.0 / + audio.playback.sampleRate * 2; + diagnostics->pending |= + PLAYBACK_DIAGNOSTIC_REGISTER_GRAPH; + } + + diagnostics->start.targetLatencyMs = targetLatencyFrames * 1000.0 / - audio.playback.sampleRate * 2; - audio.playback.graph = app_registerGraph("PLAYBACK RING", - audio.playback.timings, 0.0f, graphMax, - audioGraphFormatFn); - if (!audio.playback.graph) - ringbuffer_free(&audio.playback.timings); + audio.playback.sampleRate; + diagnostics->start.queuedLatencyMs = + audio.playback.targetStartFrames * 1000.0 / + audio.playback.sampleRate; + diagnostics->pending |= PLAYBACK_DIAGNOSTIC_START_LOG; } audio.playback.startupLowWaterFrames = @@ -2012,40 +2652,36 @@ static void playbackData(const void * data, size_t frameCount, audio.playback.startupPacketPeriod = max(llrint(packetSec * 1.0e9), INT64_C(1)); audio.playback.startupPacketDeadline = - now + audio.playback.startupPacketPeriod; - - if (g_params.audioDebug) - DEBUG_INFO( - "Audio start: %.2f/%.2f ms target/queued", - targetLatencyFrames * 1000.0 / - audio.playback.sampleRate, - audio.playback.targetStartFrames * 1000.0 / - audio.playback.sampleRate); + arrivalTime + audio.playback.startupPacketPeriod; playbackSetState(STREAM_STATE_SETUP_DEVICE); - audio.audioDev->playback.start(); + playbackArmBackend(); + playbackWorkerWake(); } } if (!g_params.audioDebug) - return; + return PLAYBACK_DATA_PROCESSED; const double softwareLatencyMs = actualLatencyFrames * 1000.0 / audio.playback.sampleRate; + bool wakeDiagnostics = false; - if (audio.playback.graph && now >= sourceData->nextGraphTime) + if (audio.playback.timings && + atomic_load_explicit( + &audio.playback.graphReady, memory_order_acquire) && + now >= sourceData->nextGraphTime) { sourceData->nextGraphTime = now + PLAYBACK_GRAPH_INTERVAL_NS; const float latency = softwareLatencyMs; ringbuffer_push(audio.playback.timings, &latency); - app_invalidateGraph(audio.playback.graph); + playbackDiagnosticsLocked()->pending |= + PLAYBACK_DIAGNOSTIC_INVALIDATE; + wakeDiagnostics = true; } if (now >= sourceData->nextLogTime) { - const double backendLatencyMs = - audio.audioDev->playback.latency ? - audio.audioDev->playback.latency() / 1000.0 : 0.0; const double sourcePpm = sourceData->sourceRateValid ? (nominalFrameSec / sourceData->sourceRateFrameSec - 1.0) * 1.0e6 : 0.0; @@ -2058,27 +2694,39 @@ static void playbackData(const void * data, size_t frameCount, (playbackProviderRate(sourceData) / audio.playback.sampleRate - 1.0) * 1.0e6 : (ratio - 1.0) * 1.0e6; - const char * controlName = providerControl ? "feedback" : - audio.playback.rateControl == PLAYBACK_RATE_BACKEND ? - "backend" : "software"; - const char * sourceRateName = providerControl ? "arrival" : "source"; const unsigned int underruns = atomic_exchange_explicit( &audio.playback.underruns, 0, memory_order_relaxed); - - DEBUG_INFO( - "Audio sync: ring %.2f/%.2f ms, backend %.2f ms, " - "%s %+.1f ppm, rates %s/device %+.1f/%+.1f ppm, " - "jitter %.2f ms, xruns %u/%u", - softwareLatencyMs, + PlaybackDiagnostics * diagnostics = playbackDiagnosticsLocked(); + const unsigned int pendingUnderruns = + diagnostics->pending & PLAYBACK_DIAGNOSTIC_SYNC_LOG ? + diagnostics->sync.underruns : 0; + const unsigned int pendingOverruns = + diagnostics->pending & PLAYBACK_DIAGNOSTIC_SYNC_LOG ? + diagnostics->sync.overruns : 0; + diagnostics->sync = (PlaybackSyncDiagnostics) + { + .softwareLatencyMs = softwareLatencyMs, + .targetLatencyMs = targetLatencyFrames * 1000.0 / audio.playback.sampleRate, - backendLatencyMs, controlName, controlPpm, - sourceRateName, sourcePpm, devicePpm, - sourceData->arrivalJitterSec * 1000.0, - underruns, sourceData->bufferOverruns); + .sourcePpm = sourcePpm, + .devicePpm = devicePpm, + .controlPpm = controlPpm, + .jitterMs = sourceData->arrivalJitterSec * 1000.0, + .underruns = pendingUnderruns + underruns, + .overruns = pendingOverruns + sourceData->bufferOverruns, + .rateControl = audio.playback.rateControl, + }; + diagnostics->pending |= PLAYBACK_DIAGNOSTIC_SYNC_LOG; sourceData->bufferOverruns = 0; sourceData->nextLogTime = now + INT64_C(5000000000); + wakeDiagnostics = true; } + + if (wakeDiagnostics) + playbackWorkerWake(); + + return PLAYBACK_DATA_PROCESSED; } static bool playbackGetFeedback( @@ -2115,28 +2763,148 @@ static bool playbackGetFeedback( return true; } -bool lgAudio_supportsRecord(void) +static bool audioBindingEqual( + const AudioBinding * a, const AudioBinding * b) { - return audio.audioDev && audio.audioDev->record.start; + return + a->ops == b->ops && + a->opaque == b->opaque && + a->epoch == b->epoch && + a->generation == b->generation; } -static void recordPushFrames(uint8_t * data, int frames) +static void recordWorkerWake(void) +{ + LG_LOCK_SHARED(audio.record.lock); + if (audio.record.wake && audio.record.thread) + lgSignalEvent(audio.record.wake); + LG_UNLOCK_SHARED(audio.record.lock); +} + +static void recordDisarmLocked(void) +{ + atomic_store_explicit( + &audio.record.deliverySerial, 0, memory_order_release); +} + +static void recordBackendFailed(uint32_t attemptSerial) +{ + unsigned int expected = attemptSerial; + if (!attemptSerial || !atomic_compare_exchange_strong_explicit( + &audio.record.deliverySerial, &expected, 0, + memory_order_acq_rel, memory_order_acquire)) + return; + + atomic_store_explicit(&audio.record.failedAttemptSerial, + attemptSerial, memory_order_release); + lgSignalEvent(audio.record.wake); +} + +static void recordAdvanceRequestLocked(void) +{ + if (!++audio.record.requestSerial) + ++audio.record.requestSerial; +} + +static void recordResetStartRetryLocked(void) +{ + audio.record.startFailures = 0; + audio.record.nextStartRetry = 0; +} + +/* record.lock must be held exclusively. */ +static bool recordDelayStartLocked(void) +{ + const unsigned int failures = audio.record.startFailures; + audio.record.nextStartRetry = nanotime() + + audioStartRetryDelay(failures); + if (audio.record.startFailures < 7U) + ++audio.record.startFailures; + + return failures == 0U || failures == 6U; +} + +/* record.lock must be held exclusively. */ +static bool recordDelayRequestLocked( + uint64_t requestSerial, bool resetRetry) +{ + if (audio.record.shuttingDown || !audio.record.requested || + !audio.record.enabled || !audio.record.requestedGeneration || + requestSerial != audio.record.requestSerial) + return false; + + if (resetRetry) + recordResetStartRetryLocked(); + return recordDelayStartLocked(); +} + +static unsigned int recordWorkerWaitTimeout(void) +{ + unsigned int timeout = TIMEOUT_INFINITE; + + LG_LOCK_SHARED(audio.record.lock); + if (!audio.record.shuttingDown && audio.record.requested && + audio.record.enabled && audio.record.requestedGeneration && + audio.record.nextStartRetry) + { + const int64_t remaining = + audio.record.nextStartRetry - nanotime(); + timeout = remaining <= 0 ? 0 : + (unsigned int)((remaining + INT64_C(999999)) / + INT64_C(1000000)); + } + LG_UNLOCK_SHARED(audio.record.lock); + + return timeout; +} + +bool lgAudio_supportsRecord(void) +{ + return atomic_load_explicit(&audio.ready, memory_order_acquire) && + audio.audioDev && audio.audioDev->record.start && + audio.audioDev->record.stop && + audio.record.wake && audio.record.thread && + atomic_load_explicit( + &audio.record.workerAlive, memory_order_acquire); +} + +static bool recordPushFrames(uint8_t * data, int frames, + const LG_AudioClock * sourceClock) { if (frames <= 0) - return; + return true; - const uint32_t generation = atomic_load_explicit( - &audio.record.streamGeneration, memory_order_acquire); - if (!generation) - return; + const uint32_t serial = atomic_load_explicit( + &audio.record.deliverySerial, memory_order_acquire); + if (!serial) + return true; - LG_LOCK_SHARED(audio.activeLock); - if (generation == atomic_load_explicit( - &audio.record.streamGeneration, memory_order_acquire) && - audio.active.ops && audio.active.ops->recordData) - audio.active.ops->recordData(audio.active.opaque, - generation, data, frames, NULL); - LG_UNLOCK_SHARED(audio.activeLock); + AudioBinding binding; + uint32_t generation; + + LG_LOCK_SHARED(audio.record.lock); + if (serial != atomic_load_explicit( + &audio.record.deliverySerial, memory_order_acquire)) + { + LG_UNLOCK_SHARED(audio.record.lock); + return true; + } + binding = audio.record.deliveryBinding; + generation = audio.record.deliveryGeneration; + LG_UNLOCK_SHARED(audio.record.lock); + + AudioBinding active; + if (!eventBegin(&binding, &active)) + return true; + + bool accepted = true; + if (serial == atomic_load_explicit( + &audio.record.deliverySerial, memory_order_acquire) && + active.ops->recordData) + accepted = active.ops->recordData(active.opaque, + generation, data, frames, sourceClock); + eventEnd(); + return accepted; } static MsgBoxHandle recordCancelConfirmLocked(void) @@ -2148,36 +2916,16 @@ static MsgBoxHandle recordCancelConfirmLocked(void) return handle; } -static void realRecordStartLocked(const LG_AudioFormat * format) -{ - audio.record.started = true; - audio.record.format = *format; - - audio.audioDev->record.start(format, recordPushFrames); - - // if a volume level was stored, set it before we return - if (audio.record.volumeChannels && audio.audioDev->record.volume) - audio.audioDev->record.volume( - audio.record.volumeChannels, - audio.record.volume); - - // set the inital mute state - if (audio.audioDev->record.mute) - audio.audioDev->record.mute(audio.record.mute); - - if (g_params.micShowIndicator) - app_showRecord(true); -} - static void recordConfirm(bool yes, void * opaque) { const uint64_t generation = (uint64_t)(uintptr_t)opaque; + bool wake = false; - LG_LOCK(audio.record.lock); + LG_LOCK_EXCLUSIVE(audio.record.lock); if (!audio.record.confirmPending || generation != audio.record.confirmGeneration) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } @@ -2185,419 +2933,818 @@ static void recordConfirm(bool yes, void * opaque) audio.record.confirmHandle = NULL; if (yes && audio.record.requested && - !audio.record.shuttingDown && audio.audioDev) + audio.record.requestedGeneration && + !audio.record.shuttingDown && audio.audioDev && + audio.record.thread && atomic_load_explicit( + &audio.record.workerAlive, memory_order_acquire)) { DEBUG_INFO("Microphone access granted"); - realRecordStartLocked(&audio.record.confirmFormat); + audio.record.enabled = true; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + wake = true; } else if (yes) DEBUG_INFO("Ignoring stale microphone access confirmation"); else DEBUG_INFO("Microphone access denied"); - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (wake) + recordWorkerWake(); } -static void recordStart(const LG_AudioFormat * format) +static void recordCreateConfirm( + MsgBoxHandle oldConfirm, uint64_t generation) { - LG_LOCK(audio.record.lock); - if (!audio.audioDev || !audio.audioDev->record.start || - audio.record.shuttingDown || - !audioFormatValid(format)) + app_msgBoxClose(oldConfirm); + + MsgBoxHandle handle = app_confirmMsgBox( + "Microphone", recordConfirm, (void *)(uintptr_t)generation, + "An application just opened the microphone!\n" + "Do you want it to access your microphone?"); + MsgBoxHandle stale = NULL; + + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (handle && audio.record.confirmPending && + generation == audio.record.confirmGeneration) + audio.record.confirmHandle = handle; + else { - if (format && !audioFormatValid(format)) - DEBUG_ERROR("Invalid recording format"); - LG_UNLOCK(audio.record.lock); + stale = handle; + if (!handle && audio.record.confirmPending && + generation == audio.record.confirmGeneration) + { + audio.record.confirmPending = false; + ++audio.record.confirmGeneration; + } + } + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + + app_msgBoxClose(stale); +} + +static void recordStart(const AudioBinding * binding, + uint32_t generation, const LG_AudioFormat * format) +{ + MsgBoxHandle oldConfirm = NULL; + uint64_t confirmGeneration = 0; + bool wake = false; + + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (!audio.audioDev || !audio.audioDev->record.start || + !audio.audioDev->record.stop || + !audio.record.thread || audio.record.shuttingDown || + !atomic_load_explicit( + &audio.record.workerAlive, memory_order_acquire) || + !binding || !generation) + { + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } - const bool restart = audio.record.started; - if (audio.record.started) + if (!audioFormatValid(format)) { - if (audioFormatEqual(format, &audio.record.lastFormat)) - { - LG_UNLOCK(audio.record.lock); - return; - } - - realRecordStopLocked(); + oldConfirm = recordCancelConfirmLocked(); + audio.record.requested = false; + audio.record.enabled = false; + audio.record.requestedBinding = (AudioBinding) { 0 }; + audio.record.requestedGeneration = 0; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + app_msgBoxClose(oldConfirm); + recordWorkerWake(); + DEBUG_ERROR("Invalid recording format"); + return; } - MsgBoxHandle oldConfirm = recordCancelConfirmLocked(); - audio.record.requested = true; - audio.record.lastFormat = *format; + const bool sameRequest = audio.record.requested && + generation == audio.record.requestedGeneration && + audioBindingEqual(binding, &audio.record.requestedBinding); + if (sameRequest) + { + const bool formatChanged = !audioFormatEqual( + format, &audio.record.requestedFormat); + if (formatChanged) + { + audio.record.requestedFormat = *format; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + wake = true; + } - if (restart) - realRecordStartLocked(format); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (wake) + recordWorkerWake(); + return; + } + + const bool keepEnabled = audio.record.requested && + audio.record.enabled && + audioBindingEqual(binding, &audio.record.requestedBinding); + wake = true; + oldConfirm = recordCancelConfirmLocked(); + audio.record.requested = true; + audio.record.enabled = false; + audio.record.requestedBinding = *binding; + audio.record.requestedGeneration = generation; + audio.record.requestedFormat = *format; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + + if (keepEnabled) + { + audio.record.enabled = true; + wake = true; + } else if (g_state.micDefaultState == MIC_DEFAULT_DENY) DEBUG_INFO("Microphone access denied by default"); else if (g_state.micDefaultState == MIC_DEFAULT_ALLOW) { DEBUG_INFO("Microphone access granted by default"); - realRecordStartLocked(format); + audio.record.enabled = true; + wake = true; } else { - audio.record.confirmFormat = *format; - audio.record.confirmPending = true; - const uint64_t generation = ++audio.record.confirmGeneration; - LG_UNLOCK(audio.record.lock); + audio.record.confirmPending = true; + confirmGeneration = ++audio.record.confirmGeneration; + } + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (wake) + recordWorkerWake(); + if (confirmGeneration) + recordCreateConfirm(oldConfirm, confirmGeneration); + else app_msgBoxClose(oldConfirm); +} - LG_LOCK(audio.record.lock); - const bool current = - audio.record.confirmPending && - generation == audio.record.confirmGeneration && - audio.record.requested && - !audio.record.shuttingDown && - audio.audioDev; - if (current) - { - audio.record.confirmHandle = app_confirmMsgBox( - "Microphone", recordConfirm, (void *)(uintptr_t)generation, - "An application just opened the microphone!\n" - "Do you want it to access your microphone?"); - if (!audio.record.confirmHandle) - { - audio.record.confirmPending = false; - ++audio.record.confirmGeneration; - } - } - LG_UNLOCK(audio.record.lock); +static void recordStop( + const AudioBinding * binding, uint32_t generation) +{ + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (binding && + (!generation || !audio.record.requested || + generation != audio.record.requestedGeneration || + !audioBindingEqual(binding, &audio.record.requestedBinding))) + { + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } - LG_UNLOCK(audio.record.lock); - app_msgBoxClose(oldConfirm); -} - -static void recordReconfigure(const LG_AudioFormat * format) -{ - LG_LOCK(audio.record.lock); - if (!audio.audioDev || !audio.audioDev->record.start || - audio.record.shuttingDown || !audioFormatValid(format)) - { - LG_UNLOCK(audio.record.lock); - return; - } - - audio.record.lastFormat = *format; - if (audio.record.started && - !audioFormatEqual(format, &audio.record.format)) - { - realRecordStopLocked(); - realRecordStartLocked(format); - } - else if (audio.record.confirmPending) - audio.record.confirmFormat = *format; - - LG_UNLOCK(audio.record.lock); -} - -static void realRecordStopLocked(void) -{ - audio.audioDev->record.stop(); - audio.record.started = false; - - if (g_params.micShowIndicator) - app_showRecord(false); -} - -static void recordStop(void) -{ - LG_LOCK(audio.record.lock); - audio.record.requested = false; + const bool wasRequested = audio.record.requested; + audio.record.requested = false; + audio.record.enabled = false; + audio.record.requestedBinding = (AudioBinding) { 0 }; + audio.record.requestedGeneration = 0; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); MsgBoxHandle confirm = recordCancelConfirmLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); - if (audio.audioDev && audio.record.started) - { + if (wasRequested) DEBUG_INFO("Microphone recording stopped"); - realRecordStopLocked(); - } - LG_UNLOCK(audio.record.lock); - app_msgBoxClose(confirm); + recordWorkerWake(); } void lgAudio_recordToggleKeybind(int sc, void * opaque) { - LG_LOCK(audio.record.lock); - if (!audio.audioDev || audio.record.shuttingDown) + if (!atomic_load_explicit(&audio.ready, memory_order_acquire)) + return; + + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (!audio.audioDev || !audio.record.thread || + !atomic_load_explicit( + &audio.record.workerAlive, memory_order_acquire) || + audio.record.shuttingDown) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } if (!audio.record.requested) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); app_alert(LG_ALERT_WARNING, "No application is requesting microphone access."); return; } MsgBoxHandle confirm = recordCancelConfirmLocked(); - bool started; - if (audio.record.started) - { - DEBUG_INFO("Microphone recording stopped by user"); - realRecordStopLocked(); - started = false; - } - else - { - DEBUG_INFO("Microphone recording started by user"); - realRecordStartLocked(&audio.record.lastFormat); - started = true; - } - LG_UNLOCK(audio.record.lock); + audio.record.enabled = !audio.record.enabled; + const bool enabled = audio.record.enabled; + recordResetStartRetryLocked(); + recordAdvanceRequestLocked(); + recordDisarmLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + DEBUG_INFO("Microphone recording %s by user", + enabled ? "started" : "stopped"); app_msgBoxClose(confirm); + recordWorkerWake(); app_alert(LG_ALERT_INFO, - started ? "Microphone enabled" : "Microphone disabled"); + enabled ? "Microphone enabled" : "Microphone disabled"); } -static void recordVolume(int channels, const uint16_t volume[]) +static void recordVolume(const AudioBinding * binding, + uint32_t generation, int channels, const uint16_t volume[]) { - LG_LOCK(audio.record.lock); + LG_LOCK_EXCLUSIVE(audio.record.lock); if (!audio.audioDev || !audio.audioDev->record.volume || - !g_params.audioSyncVolume || audio.record.shuttingDown) + !g_params.audioSyncVolume || audio.record.shuttingDown || + !audio.record.requested || + generation != audio.record.requestedGeneration || + !audioBindingEqual(binding, &audio.record.requestedBinding)) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } - // store the values so we can restore the state if the stream is restarted channels = min(ARRAY_LENGTH(audio.record.volume), channels); memcpy(audio.record.volume, volume, sizeof(uint16_t) * channels); audio.record.volumeChannels = channels; + if (!++audio.record.controlSerial) + ++audio.record.controlSerial; + LG_UNLOCK_EXCLUSIVE(audio.record.lock); - if (!audio.record.started) - { - LG_UNLOCK(audio.record.lock); - return; - } - - audio.audioDev->record.volume(channels, volume); - LG_UNLOCK(audio.record.lock); + recordWorkerWake(); } -static void recordMute(bool mute) +static void recordMute( + const AudioBinding * binding, uint32_t generation, bool mute) { - LG_LOCK(audio.record.lock); + LG_LOCK_EXCLUSIVE(audio.record.lock); if (!audio.audioDev || !audio.audioDev->record.mute || - audio.record.shuttingDown) + audio.record.shuttingDown || !audio.record.requested || + generation != audio.record.requestedGeneration || + !audioBindingEqual(binding, &audio.record.requestedBinding)) { - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); return; } - // store the value so we can restore it if the stream is restarted audio.record.mute = mute; - if (!audio.record.started) + if (!++audio.record.controlSerial) + ++audio.record.controlSerial; + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + + recordWorkerWake(); +} + +static bool recordRequestCurrentLocked(uint64_t requestSerial, + const AudioBinding * binding, uint32_t generation, + const LG_AudioFormat * format) +{ + return !audio.record.shuttingDown && audio.record.requested && + audio.record.enabled && + requestSerial == audio.record.requestSerial && + generation == audio.record.requestedGeneration && + audioBindingEqual(binding, &audio.record.requestedBinding) && + audioFormatEqual(format, &audio.record.requestedFormat); +} + +static uint32_t recordArmLocked( + const AudioBinding * binding, uint32_t generation) +{ + if (!++audio.record.nextAttemptSerial) + ++audio.record.nextAttemptSerial; + + atomic_store_explicit( + &audio.record.failedAttemptSerial, 0, memory_order_release); + audio.record.deliveryBinding = *binding; + audio.record.deliveryGeneration = generation; + atomic_store_explicit(&audio.record.deliverySerial, + audio.record.nextAttemptSerial, memory_order_release); + return audio.record.nextAttemptSerial; +} + +static int recordThread(void * opaque) +{ + bool backendStarted = false; + bool controlsApplied = false; + uint64_t backendRequestSerial = 0; + uint32_t backendAttemptSerial = 0; + int64_t backendStartTime = 0; + uint64_t appliedControlSerial = 0; + + for (;;) { - LG_UNLOCK(audio.record.lock); - return; + const unsigned int timeout = recordWorkerWaitTimeout(); + if (!lgWaitEvent(audio.record.wake, timeout) && + timeout == TIMEOUT_INFINITE) + { + DEBUG_ERROR("Failed to wait for audio recording work"); + break; + } + + for (;;) + { + enum + { + RECORD_ACTION_WAIT, + RECORD_ACTION_START, + RECORD_ACTION_STOP, + RECORD_ACTION_CONTROLS, + RECORD_ACTION_EXIT, + } + action = RECORD_ACTION_WAIT; + + uint64_t requestSerial = 0; + uint32_t attemptSerial = 0; + uint64_t controlSerial = 0; + AudioBinding binding = { 0 }; + uint32_t generation = 0; + LG_AudioFormat format = { 0 }; + int volumeChannels = 0; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool mute = false; + bool reportStartFailure = false; + bool runtimeFailure = false; + bool resetStartRetry = false; + + LG_LOCK_EXCLUSIVE(audio.record.lock); + const bool desired = !audio.record.shuttingDown && + audio.record.requested && audio.record.enabled && + audio.record.requestedGeneration && audio.audioDev && + audio.audioDev->record.start && audio.audioDev->record.stop; + + const bool backendFailed = backendStarted && + backendAttemptSerial == atomic_load_explicit( + &audio.record.failedAttemptSerial, memory_order_acquire); + if (backendStarted && + (backendFailed || !desired || + backendRequestSerial != audio.record.requestSerial)) + { + recordDisarmLocked(); + if (backendFailed && desired && + backendRequestSerial == audio.record.requestSerial) + { + requestSerial = backendRequestSerial; + runtimeFailure = true; + resetStartRetry = + nanotime() - backendStartTime >= AUDIO_RETRY_RESET_NS; + } + action = RECORD_ACTION_STOP; + } + else if (!backendStarted && desired && + nanotime() >= audio.record.nextStartRetry) + { + requestSerial = audio.record.requestSerial; + binding = audio.record.requestedBinding; + generation = audio.record.requestedGeneration; + format = audio.record.requestedFormat; + attemptSerial = recordArmLocked(&binding, generation); + action = RECORD_ACTION_START; + } + else if (backendStarted && + (!controlsApplied || + appliedControlSerial != audio.record.controlSerial)) + { + requestSerial = backendRequestSerial; + controlSerial = audio.record.controlSerial; + volumeChannels = audio.record.volumeChannels; + if (volumeChannels) + memcpy(volume, audio.record.volume, + sizeof(*volume) * volumeChannels); + mute = audio.record.mute; + action = RECORD_ACTION_CONTROLS; + } + else if (audio.record.shuttingDown) + action = RECORD_ACTION_EXIT; + + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + + if (action == RECORD_ACTION_WAIT) + break; + + if (action == RECORD_ACTION_EXIT) + goto exit; + + if (action == RECORD_ACTION_STOP) + { + if (audio.audioDev && audio.audioDev->record.stop) + audio.audioDev->record.stop(); + + if (runtimeFailure) + { + LG_LOCK_EXCLUSIVE(audio.record.lock); + reportStartFailure = recordDelayRequestLocked( + requestSerial, resetStartRetry); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + } + + backendStarted = false; + controlsApplied = false; + backendRequestSerial = 0; + backendAttemptSerial = 0; + backendStartTime = 0; + + if (g_params.micShowIndicator) + app_showRecord(false); + if (runtimeFailure && reportStartFailure) + DEBUG_ERROR("Audio recording device failed; retrying"); + continue; + } + + if (action == RECORD_ACTION_START) + { + const bool started = audio.audioDev->record.start( + &format, recordPushFrames, + recordBackendFailed, attemptSerial); + + LG_LOCK_EXCLUSIVE(audio.record.lock); + const bool current = recordRequestCurrentLocked( + requestSerial, &binding, generation, &format); + const bool failed = attemptSerial == atomic_load_explicit( + &audio.record.failedAttemptSerial, memory_order_acquire); + const bool delayAfterStop = started && current && failed; + if (started && current && !failed) + { + backendStarted = true; + backendRequestSerial = requestSerial; + backendAttemptSerial = attemptSerial; + backendStartTime = nanotime(); + controlsApplied = false; + audio.record.nextStartRetry = 0; + } + else + { + if (attemptSerial == atomic_load_explicit( + &audio.record.deliverySerial, memory_order_relaxed)) + recordDisarmLocked(); + if (current && !started) + reportStartFailure = recordDelayStartLocked(); + } + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + + if (started && (!current || failed)) + audio.audioDev->record.stop(); + + if (delayAfterStop) + { + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (recordRequestCurrentLocked( + requestSerial, &binding, generation, &format)) + reportStartFailure = recordDelayStartLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + } + else if (started && current && g_params.micShowIndicator) + app_showRecord(true); + if (current && (!started || failed) && reportStartFailure) + DEBUG_ERROR("Failed to start audio recording device; retrying"); + continue; + } + + if (volumeChannels && audio.audioDev->record.volume) + audio.audioDev->record.volume(volumeChannels, volume); + if (audio.audioDev->record.mute) + audio.audioDev->record.mute(mute); + + LG_LOCK_EXCLUSIVE(audio.record.lock); + if (backendStarted && + requestSerial == backendRequestSerial && + requestSerial == audio.record.requestSerial) + { + controlsApplied = true; + appliedControlSerial = controlSerial; + } + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + } } - audio.audioDev->record.mute(mute); - LG_UNLOCK(audio.record.lock); +exit: + if (backendStarted && audio.audioDev && audio.audioDev->record.stop) + audio.audioDev->record.stop(); + + LG_LOCK_EXCLUSIVE(audio.record.lock); + recordDisarmLocked(); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (backendStarted && g_params.micShowIndicator) + app_showRecord(false); + atomic_store_explicit( + &audio.record.workerAlive, false, memory_order_release); + return 0; } static bool bindingActiveNL(const AudioBinding * binding) { - return binding->ops && - audio.active.ops == binding->ops && - audio.active.opaque == binding->opaque && - audio.active.generation == binding->generation; + return binding->ops && audioBindingEqual(&audio.active, binding); +} + +static bool eventBegin( + const AudioBinding * binding, AudioBinding * active) +{ + LG_LOCK_SHARED(audio.activeLock); + const bool current = bindingActiveNL(binding); + if (current) + { + *active = audio.active; + atomic_fetch_add_explicit( + &audio.activeCallbacks, 1, memory_order_acq_rel); + } + LG_UNLOCK_SHARED(audio.activeLock); + return current; +} + +static void eventEnd(void) +{ + const unsigned int previous = atomic_fetch_sub_explicit( + &audio.activeCallbacks, 1, memory_order_release); + if ((previous & ACTIVE_CALLBACK_WAITING) && + (previous & ACTIVE_CALLBACK_COUNT_MASK) == 1) + lgSignalEvent(audio.activeIdle); +} + +static void waitForActiveCallbacks(void) +{ + atomic_fetch_or_explicit( + &audio.activeCallbacks, ACTIVE_CALLBACK_WAITING, + memory_order_acq_rel); + while ((atomic_load_explicit( + &audio.activeCallbacks, memory_order_acquire) & + ACTIVE_CALLBACK_COUNT_MASK) != 0) + lgWaitEvent(audio.activeIdle, TIMEOUT_INFINITE); + atomic_fetch_and_explicit( + &audio.activeCallbacks, ~ACTIVE_CALLBACK_WAITING, + memory_order_release); } static void queueFeedback(const LG_AudioOps * ops, void * opaque, - uint32_t bindingGeneration, uint32_t generation, + uint32_t bindingEpoch, uint32_t bindingGeneration, + uint32_t generation, const LG_AudioClock * clock, double targetRate) { - if (!audio.feedback.event || !audio.feedback.thread || !clock) + if (!audio.feedback.wakeInitialized || !audio.feedback.thread || !clock) return; LG_LOCK(audio.feedback.lock); audio.feedback.ops = ops; audio.feedback.opaque = opaque; + audio.feedback.bindingEpoch = bindingEpoch; audio.feedback.bindingGeneration = bindingGeneration; audio.feedback.generation = generation; audio.feedback.clock = *clock; audio.feedback.targetRate = targetRate; audio.feedback.pending = true; LG_UNLOCK(audio.feedback.lock); - lgSignalEvent(audio.feedback.event); + feedbackWorkerWake(); } static void eventPlaybackStart(void * opaque, uint32_t generation, const LG_AudioFormat * format, const LG_AudioClock * sourceClock) { AudioBinding * binding = opaque; + AudioBinding active; + (void)sourceClock; - LG_LOCK_SHARED(audio.activeLock); - if (bindingActiveNL(binding)) + if (!eventBegin(binding, &active)) + return; + + const bool providerRateControl = + active.ops->clockFeedback && + audio.feedback.wakeInitialized && audio.feedback.thread; + + LG_LOCK(audio.playback.sourceLock); + ++audio.playback.requestSerial; + audio.playback.requestedGeneration = generation; + audio.playback.requestedFormatValid = + generation && audioFormatValid(format); + audio.playback.requestedProviderRateControl = providerRateControl; + audio.playback.forceSoftwareResampler = false; + audio.playback.startPending = false; + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + + if (audio.playback.requestedFormatValid) { - LG_LOCK(audio.playback.sourceLock); - atomic_store_explicit(&audio.playback.streamGeneration, - generation, memory_order_release); - playbackStart(format, sourceClock, - binding->ops->clockFeedback && - audio.feedback.event && audio.feedback.thread); - LG_UNLOCK(audio.playback.sourceLock); + audio.playback.requestedFormat = *format; + if (audio.playback.startInProgress) + { + audio.playback.startPending = true; + playbackWorkerWake(); + } + else + playbackScheduleStart(); } - LG_UNLOCK_SHARED(audio.activeLock); + else + { + if (generation) + DEBUG_ERROR("Invalid playback format"); + if (playbackGetState() != STREAM_STATE_STOP) + { + LG_LOCK(audio.playback.deviceLock); + playbackStop(); + LG_UNLOCK(audio.playback.deviceLock); + } + audio.playback.requestedGeneration = 0; + } + LG_UNLOCK(audio.playback.sourceLock); + eventEnd(); } static void eventPlaybackStop(void * opaque, uint32_t generation) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - if (bindingActiveNL(binding)) + if (!eventBegin(binding, &active)) + return; + + LG_LOCK(audio.playback.sourceLock); + if (audio.playback.requestedGeneration == generation) { - LG_LOCK(audio.playback.sourceLock); + ++audio.playback.requestSerial; if (atomic_load_explicit(&audio.playback.streamGeneration, memory_order_acquire) == generation) { + LG_LOCK(audio.playback.deviceLock); playbackSourceStop(); - atomic_store_explicit( - &audio.playback.streamGeneration, 0, memory_order_release); + LG_UNLOCK(audio.playback.deviceLock); } - LG_UNLOCK(audio.playback.sourceLock); + audio.playback.requestedGeneration = 0; + audio.playback.requestedFormatValid = false; + audio.playback.requestedProviderRateControl = false; + audio.playback.forceSoftwareResampler = false; + audio.playback.startPending = false; + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); } - LG_UNLOCK_SHARED(audio.activeLock); + LG_UNLOCK(audio.playback.sourceLock); + eventEnd(); } static void eventPlaybackVolume(void * opaque, uint32_t generation, uint8_t channels, const uint16_t volume[]) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - if (bindingActiveNL(binding)) + if (!eventBegin(binding, &active)) + return; + + bool wake = false; + LG_LOCK(audio.playback.sourceLock); + if (volume && + audio.playback.requestedGeneration == generation) { - LG_LOCK(audio.playback.sourceLock); - if (volume && - atomic_load_explicit(&audio.playback.streamGeneration, - memory_order_acquire) == generation) - playbackVolume(channels, volume); - LG_UNLOCK(audio.playback.sourceLock); + const int storedChannels = playbackStoreVolume(channels, volume); + if (storedChannels) + { + if (!++audio.playback.controlSerial) + ++audio.playback.controlSerial; + audio.playback.controlsPending = true; + wake = atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation; + } } - LG_UNLOCK_SHARED(audio.activeLock); + LG_UNLOCK(audio.playback.sourceLock); + if (wake) + playbackWorkerWake(); + eventEnd(); } static void eventPlaybackMute(void * opaque, uint32_t generation, bool mute) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - if (bindingActiveNL(binding)) + if (!eventBegin(binding, &active)) + return; + + bool wake = false; + LG_LOCK(audio.playback.sourceLock); + if (audio.playback.requestedGeneration == generation) { - LG_LOCK(audio.playback.sourceLock); - if (atomic_load_explicit(&audio.playback.streamGeneration, - memory_order_acquire) == generation) - playbackMute(mute); - LG_UNLOCK(audio.playback.sourceLock); + const bool stored = playbackStoreMute(mute); + if (stored) + { + if (!++audio.playback.controlSerial) + ++audio.playback.controlSerial; + audio.playback.controlsPending = true; + wake = atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation; + } } - LG_UNLOCK_SHARED(audio.activeLock); + LG_UNLOCK(audio.playback.sourceLock); + if (wake) + playbackWorkerWake(); + eventEnd(); } static void eventPlaybackData(void * opaque, uint32_t generation, const void * data, size_t frames, const LG_AudioClock * sourceClock) { + const int64_t arrivalTime = nanotime(); AudioBinding * binding = opaque; - const LG_AudioOps * ops; - void * providerOpaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding); - ops = active ? binding->ops : NULL; - providerOpaque = active ? binding->opaque : NULL; - if (active) + if (!eventBegin(binding, &active)) + return; + + LG_LOCK(audio.playback.sourceLock); + if (audio.playback.requestedGeneration == generation) { - LG_LOCK(audio.playback.sourceLock); - if (atomic_load_explicit(&audio.playback.streamGeneration, - memory_order_acquire) == generation) + const uint32_t liveGeneration = atomic_load_explicit( + &audio.playback.streamGeneration, memory_order_acquire); + PlaybackDataResult result = PLAYBACK_DATA_DROP; + if (liveGeneration == generation) { - playbackData(data, frames, sourceClock); + result = playbackData(data, frames, sourceClock, arrivalTime); - LG_AudioClock feedback; - double targetRate; - if (ops->clockFeedback && - playbackGetFeedback(&feedback, &targetRate)) - queueFeedback(ops, providerOpaque, binding->generation, - generation, &feedback, targetRate); + if (result == PLAYBACK_DATA_PROCESSED && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) != generation) + result = PLAYBACK_DATA_DROP; + + if (result == PLAYBACK_DATA_RETRY || + result == PLAYBACK_DATA_RETRY_NOW) + { + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + if (result == PLAYBACK_DATA_RETRY_NOW) + audio.playback.nextStartRetry = 0; + else + playbackDelayStart(); + } } - LG_UNLOCK(audio.playback.sourceLock); + + if (result != PLAYBACK_DATA_PROCESSED) + playbackScheduleStart(); + + LG_AudioClock feedback; + double targetRate; + if (result == PLAYBACK_DATA_PROCESSED && + active.ops->clockFeedback && + playbackGetFeedback(&feedback, &targetRate)) + queueFeedback(active.ops, active.opaque, active.epoch, + active.generation, generation, &feedback, targetRate); } - LG_UNLOCK_SHARED(audio.activeLock); + LG_UNLOCK(audio.playback.sourceLock); + eventEnd(); } static void eventRecordStart(void * opaque, uint32_t generation, const LG_AudioFormat * format) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding) && - binding->ops->recordData; - if (active) - { - const uint32_t previous = atomic_exchange_explicit( - &audio.record.streamGeneration, generation, memory_order_acq_rel); - if (previous == generation) - recordReconfigure(format); - else - recordStart(format); - } - LG_UNLOCK_SHARED(audio.activeLock); + if (!eventBegin(binding, &active)) + return; + + if (active.ops->recordData) + recordStart(&active, generation, format); + eventEnd(); } static void eventRecordStop(void * opaque, uint32_t generation) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding); - if (active && - atomic_load_explicit(&audio.record.streamGeneration, - memory_order_acquire) == generation) - { - atomic_store_explicit( - &audio.record.streamGeneration, 0, memory_order_release); - recordStop(); - } - LG_UNLOCK_SHARED(audio.activeLock); + if (!eventBegin(binding, &active)) + return; + + recordStop(&active, generation); + eventEnd(); } static void eventRecordVolume(void * opaque, uint32_t generation, uint8_t channels, const uint16_t volume[]) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding); - if (active && volume && - atomic_load_explicit(&audio.record.streamGeneration, - memory_order_acquire) == generation) - recordVolume(channels, volume); - LG_UNLOCK_SHARED(audio.activeLock); + if (!eventBegin(binding, &active)) + return; + + if (volume) + recordVolume(&active, generation, channels, volume); + eventEnd(); } static void eventRecordMute(void * opaque, uint32_t generation, bool mute) { AudioBinding * binding = opaque; + AudioBinding active; - LG_LOCK_SHARED(audio.activeLock); - const bool active = bindingActiveNL(binding); - if (active && - atomic_load_explicit(&audio.record.streamGeneration, - memory_order_acquire) == generation) - recordMute(mute); - LG_UNLOCK_SHARED(audio.activeLock); + if (!eventBegin(binding, &active)) + return; + + recordMute(&active, generation, mute); + eventEnd(); } static const LG_AudioEventOps eventOps = @@ -2620,11 +3767,20 @@ static bool validOps(const LG_AudioOps * ops) static AudioBinding makeBinding(const LG_AudioOps * ops, void * opaque) { + uint32_t epoch = 0; + if (ops) + { + epoch = ++audio.nextBindingEpoch; + if (!epoch) + epoch = ++audio.nextBindingEpoch; + } + return (AudioBinding) { .ops = ops, .opaque = opaque, .available = ops && !ops->setStatusListener, + .epoch = epoch, .generation = 0, }; } @@ -2642,14 +3798,24 @@ static void stopStreams(void) { LG_LOCK(audio.playback.sourceLock); if (audio.audioDev) + { + LG_LOCK(audio.playback.deviceLock); playbackStop(); + LG_UNLOCK(audio.playback.deviceLock); + } + ++audio.playback.requestSerial; + audio.playback.requestedGeneration = 0; + audio.playback.requestedFormatValid = false; + audio.playback.requestedProviderRateControl = false; + audio.playback.forceSoftwareResampler = false; + audio.playback.startPending = false; + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; atomic_store_explicit( &audio.playback.streamGeneration, 0, memory_order_release); LG_UNLOCK(audio.playback.sourceLock); - atomic_store_explicit( - &audio.record.streamGeneration, 0, memory_order_release); - recordStop(); + recordStop(NULL, 0); } /* providerLock must be held. dropActive suppresses calls into an endpoint @@ -2663,6 +3829,7 @@ static void updateActive(bool dropActive) AudioBinding next = slot ? *slot : (AudioBinding) { 0 }; const AudioBinding old = audio.active; if (old.ops == next.ops && old.opaque == next.opaque && + old.epoch == next.epoch && old.generation == next.generation) { audio.active = next; @@ -2672,6 +3839,8 @@ static void updateActive(bool dropActive) audio.active = (AudioBinding) { 0 }; LG_UNLOCK_EXCLUSIVE(audio.activeLock); + waitForActiveCallbacks(); + if (old.ops && !dropActive) old.ops->detach(old.opaque); dropActive = false; @@ -2695,17 +3864,20 @@ static void updateActive(bool dropActive) return; } - next.ops->detach(next.opaque); - stopStreams(); - LG_LOCK_EXCLUSIVE(audio.activeLock); if (audio.active.ops == next.ops && - audio.active.opaque == next.opaque) + audio.active.opaque == next.opaque && + audio.active.epoch == next.epoch && + audio.active.generation == next.generation) audio.active = (AudioBinding) { 0 }; if (slot->ops == next.ops && slot->opaque == next.opaque) slot->available = false; LG_UNLOCK_EXCLUSIVE(audio.activeLock); + waitForActiveCallbacks(); + next.ops->detach(next.opaque); + stopStreams(); + DEBUG_WARN("Failed to attach Audio provider: %s", next.ops->name); } } @@ -2752,16 +3924,10 @@ static void transportStatusChanged(void * opaque, LG_UNLOCK(audio.providerLock); } -static void setBinding(AudioBinding * target, const LG_AudioOps * ops, +/* bindingLock must be held. */ +static void setBindingLocked(AudioBinding * target, const LG_AudioOps * ops, void * opaque, LG_AudioStatusFn statusFn) { - if (ops && !validOps(ops)) - { - DEBUG_ERROR("Invalid audio operations"); - ops = NULL; - opaque = NULL; - } - LG_LOCK(audio.providerLock); const AudioBinding old = *target; LG_UNLOCK(audio.providerLock); @@ -2781,16 +3947,100 @@ static void setBinding(AudioBinding * target, const LG_AudioOps * ops, next.ops->setStatusListener(next.opaque, statusFn, next.opaque); } +static void setBinding(AudioBinding * target, const LG_AudioOps * ops, + void * opaque, LG_AudioStatusFn statusFn) +{ + if (ops && !validOps(ops)) + { + DEBUG_ERROR("Invalid audio operations"); + ops = NULL; + opaque = NULL; + } + + LG_LOCK(audio.bindingLock); + if (atomic_load_explicit(&audio.ready, memory_order_acquire)) + setBindingLocked(target, ops, opaque, statusFn); + LG_UNLOCK(audio.bindingLock); +} + +static int playbackThread(void * opaque) +{ + for (;;) + { + int result; + do + result = sem_wait(&audio.playback.worker.wake); + while (result < 0 && errno == EINTR); + if (result < 0) + break; + + atomic_store_explicit( + &audio.playback.worker.wakePending, false, memory_order_release); + if (atomic_load_explicit( + &audio.playback.worker.stop, memory_order_acquire)) + break; + + playbackProcessStopPending(); + playbackProcessStart(); + playbackProcessControls(); + playbackProcessDeviceStart(); + playbackProcessDiagnostics(); + } + + return 0; +} + +static void playbackFeedbackFailed(uint32_t generation) +{ + bool restart = false; + + LG_LOCK(audio.playback.sourceLock); + if (generation && + audio.playback.requestedGeneration == generation && + audio.playback.requestedProviderRateControl && + audio.playback.rateControl == PLAYBACK_RATE_PROVIDER && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + { + ++audio.playback.requestSerial; + audio.playback.requestedProviderRateControl = false; + audio.playback.forceSoftwareResampler = false; + audio.playback.startPending = true; + audio.playback.startFailures = 0; + audio.playback.nextStartRetry = 0; + playbackQueueSourceStop(); + restart = true; + } + LG_UNLOCK(audio.playback.sourceLock); + + if (restart) + DEBUG_WARN("Audio feedback stopped; using local rate control"); +} + static int feedbackThread(void * opaque) { - while (lgWaitEvent(audio.feedback.event, TIMEOUT_INFINITE)) + AudioBinding rejectedBinding = { 0 }; + uint32_t rejectedGeneration = 0; + int64_t rejectedSince = 0; + + for (;;) { + int result; + do + result = sem_wait(&audio.feedback.wake); + while (result < 0 && errno == EINTR); + if (result < 0) + break; + + atomic_store_explicit( + &audio.feedback.wakePending, false, memory_order_release); if (atomic_load_explicit( &audio.feedback.stop, memory_order_acquire)) break; const LG_AudioOps * ops; void * providerOpaque; + uint32_t bindingEpoch; uint32_t bindingGeneration; uint32_t generation; LG_AudioClock clock; @@ -2800,6 +4050,7 @@ static int feedbackThread(void * opaque) const bool pending = audio.feedback.pending; ops = audio.feedback.ops; providerOpaque = audio.feedback.opaque; + bindingEpoch = audio.feedback.bindingEpoch; bindingGeneration = audio.feedback.bindingGeneration; generation = audio.feedback.generation; clock = audio.feedback.clock; @@ -2807,18 +4058,59 @@ static int feedbackThread(void * opaque) audio.feedback.pending = false; LG_UNLOCK(audio.feedback.lock); - if (!pending || !ops || !ops->clockFeedback) + if (!pending) continue; - LG_LOCK_SHARED(audio.activeLock); - if (audio.active.ops == ops && - audio.active.opaque == providerOpaque && - audio.active.generation == bindingGeneration && + const AudioBinding binding = + { + .ops = ops, + .opaque = providerOpaque, + .epoch = bindingEpoch, + .generation = bindingGeneration, + }; + AudioBinding active; + if (!eventBegin(&binding, &active)) + { + rejectedSince = 0; + continue; + } + + bool attempted = false; + bool accepted = false; + if (active.ops->clockFeedback && atomic_load_explicit(&audio.playback.streamGeneration, memory_order_acquire) == generation) - ops->clockFeedback( - providerOpaque, generation, &clock, targetRate); - LG_UNLOCK_SHARED(audio.activeLock); + { + attempted = true; + accepted = active.ops->clockFeedback( + active.opaque, generation, &clock, targetRate); + } + + if (!attempted || accepted) + { + rejectedSince = 0; + eventEnd(); + continue; + } + + const int64_t now = nanotime(); + if (!rejectedSince || + !audioBindingEqual(&rejectedBinding, &binding) || + rejectedGeneration != generation) + { + rejectedBinding = binding; + rejectedGeneration = generation; + rejectedSince = now; + eventEnd(); + continue; + } + + if (now - rejectedSince >= PLAYBACK_FEEDBACK_FAILURE_NS) + { + rejectedSince = 0; + playbackFeedbackFailed(generation); + } + eventEnd(); } return 0; @@ -2826,75 +4118,215 @@ static int feedbackThread(void * opaque) void lgAudio_init(void) { + atomic_init(&audio.ready, false); + LG_LOCK_INIT(audio.bindingLock); LG_LOCK_INIT(audio.providerLock); LG_RWLOCK_INIT(audio.activeLock); + atomic_init(&audio.activeCallbacks, 0); LG_LOCK_INIT(audio.playback.sourceLock); - LG_LOCK_INIT(audio.record.lock); + LG_LOCK_INIT(audio.playback.deviceLock); + LG_RWLOCK_INIT(audio.record.lock); LG_LOCK_INIT(audio.feedback.lock); audio.record.shuttingDown = false; atomic_init(&audio.playback.streamGeneration, 0); - atomic_init(&audio.record.streamGeneration, 0); + atomic_init(&audio.playback.activeAttemptSerial, 0); + atomic_init(&audio.playback.failedAttemptSerial, 0); + atomic_init(&audio.playback.diagnosticsEpoch, 1); + atomic_init(&audio.playback.graphReady, false); + atomic_init(&audio.playback.worker.stop, false); + atomic_init(&audio.playback.worker.wakePending, false); + atomic_init(&audio.record.deliverySerial, 0); + atomic_init(&audio.record.failedAttemptSerial, 0); + atomic_init(&audio.record.workerAlive, false); atomic_init(&audio.feedback.stop, false); + atomic_init(&audio.feedback.wakePending, false); atomic_store_explicit( &audio.playback.callbackState, PLAYBACK_CALLBACK_DISABLED, memory_order_release); - audio.feedback.event = lgCreateEvent(true, 0); - if (audio.feedback.event && - !lgCreateThread("audioFeedback", feedbackThread, + if (sem_init(&audio.playback.callbackIdle, 0, 0) != 0) + { + DEBUG_ERROR("Failed to create the audio callback semaphore"); + return; + } + audio.playback.callbackIdleInitialized = true; + + audio.activeIdle = lgCreateEvent(true, 0); + if (!audio.activeIdle) + { + DEBUG_ERROR("Failed to create the audio provider event"); + goto err_callback; + } + + if (sem_init(&audio.playback.worker.wake, 0, 0) != 0) + { + DEBUG_ERROR("Failed to create the audio playback semaphore"); + goto err_active; + } + audio.playback.worker.wakeInitialized = true; + if (!lgCreateThread("audioPlayback", playbackThread, + NULL, &audio.playback.worker.thread)) + { + DEBUG_ERROR("Failed to create the audio playback thread"); + goto err_playbackWake; + } + + if (sem_init(&audio.feedback.wake, 0, 0) != 0) + { + DEBUG_ERROR("Failed to create the audio feedback semaphore"); + goto err_playbackThread; + } + audio.feedback.wakeInitialized = true; + if (!lgCreateThread("audioFeedback", feedbackThread, NULL, &audio.feedback.thread)) { - lgFreeEvent(audio.feedback.event); - audio.feedback.event = NULL; + DEBUG_ERROR("Failed to create the audio feedback thread"); + goto err_feedbackWake; + } + + audio.record.wake = lgCreateEvent(true, 0); + if (!audio.record.wake) + DEBUG_ERROR("Failed to create the audio recording event"); + else + { + atomic_store_explicit( + &audio.record.workerAlive, true, memory_order_release); + if (!lgCreateThread("audioRecord", recordThread, + NULL, &audio.record.thread)) + { + atomic_store_explicit( + &audio.record.workerAlive, false, memory_order_release); + DEBUG_ERROR("Failed to create the audio recording thread"); + lgFreeEvent(audio.record.wake); + audio.record.wake = NULL; + } } for (int i = 0; i < LG_AUDIODEV_COUNT; ++i) if (LG_AudioDevs[i]->init()) { audio.audioDev = LG_AudioDevs[i]; + atomic_store_explicit(&audio.ready, true, memory_order_release); DEBUG_INFO("Using AudioDev: %s", audio.audioDev->name); return; } DEBUG_WARN("Failed to initialize an audio backend"); + return; + +err_feedbackWake: + sem_destroy(&audio.feedback.wake); + audio.feedback.wakeInitialized = false; + +err_playbackThread: + atomic_store_explicit( + &audio.playback.worker.stop, true, memory_order_release); + playbackWorkerWake(); + lgJoinThread(audio.playback.worker.thread, NULL); + audio.playback.worker.thread = NULL; + +err_playbackWake: + sem_destroy(&audio.playback.worker.wake); + audio.playback.worker.wakeInitialized = false; + +err_active: + lgFreeEvent(audio.activeIdle); + audio.activeIdle = NULL; + +err_callback: + sem_destroy(&audio.playback.callbackIdle); + audio.playback.callbackIdleInitialized = false; } void lgAudio_free(void) { - lgAudio_setTransport(NULL, NULL); - lgAudio_setFallback(NULL, NULL); + LG_LOCK(audio.bindingLock); + const bool ready = atomic_exchange_explicit( + &audio.ready, false, memory_order_acq_rel); + if (ready) + { + setBindingLocked( + &audio.fallback, NULL, NULL, fallbackStatusChanged); + setBindingLocked( + &audio.transport, NULL, NULL, transportStatusChanged); + } + LG_UNLOCK(audio.bindingLock); stopStreams(); + if (audio.playback.worker.thread) + { + atomic_store_explicit( + &audio.playback.worker.stop, true, memory_order_release); + playbackWorkerWake(); + lgJoinThread(audio.playback.worker.thread, NULL); + audio.playback.worker.thread = NULL; + } + if (audio.playback.worker.wakeInitialized) + { + sem_destroy(&audio.playback.worker.wake); + audio.playback.worker.wakeInitialized = false; + } + if (audio.feedback.thread) { atomic_store_explicit( &audio.feedback.stop, true, memory_order_release); - lgSignalEvent(audio.feedback.event); + feedbackWorkerWake(); lgJoinThread(audio.feedback.thread, NULL); audio.feedback.thread = NULL; } - if (audio.feedback.event) + if (audio.feedback.wakeInitialized) { - lgFreeEvent(audio.feedback.event); - audio.feedback.event = NULL; + sem_destroy(&audio.feedback.wake); + audio.feedback.wakeInitialized = false; } - LG_LOCK(audio.record.lock); + LG_LOCK_EXCLUSIVE(audio.record.lock); audio.record.shuttingDown = true; - audio.record.requested = false; + audio.record.requested = false; + audio.record.enabled = false; + recordAdvanceRequestLocked(); + recordDisarmLocked(); MsgBoxHandle confirm = recordCancelConfirmLocked(); - struct LG_AudioDevOps * audioDev = audio.audioDev; - audio.audioDev = NULL; - LG_UNLOCK(audio.record.lock); + LG_UNLOCK_EXCLUSIVE(audio.record.lock); app_msgBoxClose(confirm); + if (audio.record.thread) + { + recordWorkerWake(); + lgJoinThread(audio.record.thread, NULL); + } + + LG_LOCK_EXCLUSIVE(audio.record.lock); + LGEvent * recordWake = audio.record.wake; + audio.record.thread = NULL; + audio.record.wake = NULL; + LG_UNLOCK_EXCLUSIVE(audio.record.lock); + if (recordWake) + lgFreeEvent(recordWake); + + if (audio.playback.callbackIdleInitialized) + { + sem_destroy(&audio.playback.callbackIdle); + audio.playback.callbackIdleInitialized = false; + } + if (audio.activeIdle) + { + lgFreeEvent(audio.activeIdle); + audio.activeIdle = NULL; + } + + struct LG_AudioDevOps * audioDev = audio.audioDev; + audio.audioDev = NULL; if (audioDev) audioDev->free(); LG_RWLOCK_FREE(audio.activeLock); LG_LOCK_FREE(audio.playback.sourceLock); + LG_LOCK_FREE(audio.playback.deviceLock); + LG_LOCK_FREE(audio.bindingLock); LG_LOCK_FREE(audio.providerLock); - LG_LOCK_FREE(audio.record.lock); + LG_RWLOCK_FREE(audio.record.lock); LG_LOCK_FREE(audio.feedback.lock); } @@ -2910,6 +4342,13 @@ void lgAudio_setTransport(const LG_AudioOps * ops, void * opaque) void lgAudio_dropTransport(void) { + LG_LOCK(audio.bindingLock); + if (!atomic_load_explicit(&audio.ready, memory_order_acquire)) + { + LG_UNLOCK(audio.bindingLock); + return; + } + LG_LOCK(audio.providerLock); LG_LOCK_EXCLUSIVE(audio.activeLock); const AudioBinding old = audio.transport; @@ -2921,6 +4360,7 @@ void lgAudio_dropTransport(void) if (old.ops && old.ops->setStatusListener) old.ops->setStatusListener(old.opaque, NULL, NULL); + LG_UNLOCK(audio.bindingLock); } #endif diff --git a/client/src/audio_spice.c b/client/src/audio_spice.c index 310bb88e..1a359e99 100644 --- a/client/src/audio_spice.c +++ b/client/src/audio_spice.c @@ -21,6 +21,7 @@ #include "audio_spice.h" #include "common/debug.h" +#include "common/event.h" #include "common/locking.h" #include @@ -50,6 +51,22 @@ typedef struct SpiceAudioEventTarget } SpiceAudioEventTarget; +typedef struct SpiceAudioCallbackWaiter +{ + struct SpiceAudioCallbackWaiter * next; + LGEvent * event; + unsigned int depth; +} +SpiceAudioCallbackWaiter; + +typedef struct SpiceAudioCallbackWaitQueue +{ + LG_Lock lock; + atomic_uint count; + SpiceAudioCallbackWaiter * waiters; +} +SpiceAudioCallbackWaitQueue; + static struct { LG_RWLock lock; @@ -59,11 +76,13 @@ static struct LG_AudioStatusFn statusCallback; void * statusOpaque; atomic_uint statusInFlight; + SpiceAudioCallbackWaitQueue statusWait; const LG_AudioEventOps * events; void * eventOpaque; uint32_t eventGeneration; atomic_uint inFlight; + SpiceAudioCallbackWaitQueue eventWait; SpiceAudioStream playback; SpiceAudioStream record; @@ -83,7 +102,17 @@ l_spice = .writer = ATOMIC_FLAG_INIT, }, .statusInFlight = ATOMIC_VAR_INIT(0), + .statusWait = + { + .lock = ATOMIC_FLAG_INIT, + .count = ATOMIC_VAR_INIT(0), + }, .inFlight = ATOMIC_VAR_INIT(0), + .eventWait = + { + .lock = ATOMIC_FLAG_INIT, + .count = ATOMIC_VAR_INIT(0), + }, }; static _Thread_local unsigned int l_eventDepth; @@ -96,6 +125,96 @@ static uint32_t nextGeneration(uint32_t generation) return generation; } +static LGEvent * createWaitEvent(void) +{ + LGEvent * event = lgCreateEvent(true, 0); + if (!event) + DEBUG_FATAL("Failed to create SPICE audio wait event"); + return event; +} + +static void waitEvent(LGEvent * event) +{ + if (!lgWaitEvent(event, TIMEOUT_INFINITE)) + DEBUG_FATAL("Failed to wait for SPICE audio event"); +} + +static void signalWaitEvent(LGEvent * event) +{ + if (!lgSignalEvent(event)) + DEBUG_FATAL("Failed to signal SPICE audio event"); +} + +static void signalCallbackWaiters( + SpiceAudioCallbackWaitQueue * queue, unsigned int remaining) +{ + if (!atomic_load_explicit(&queue->count, memory_order_acquire)) + return; + + LG_LOCK(queue->lock); + SpiceAudioCallbackWaiter ** link = &queue->waiters; + while (*link) + { + SpiceAudioCallbackWaiter * waiter = *link; + if (remaining > waiter->depth) + { + link = &waiter->next; + continue; + } + + *link = waiter->next; + atomic_fetch_sub_explicit( + &queue->count, 1, memory_order_release); + signalWaitEvent(waiter->event); + } + LG_UNLOCK(queue->lock); +} + +static void endCallback(atomic_uint * inFlight, + SpiceAudioCallbackWaitQueue * waitQueue) +{ + const unsigned int previous = atomic_fetch_sub_explicit( + inFlight, 1, memory_order_release); + DEBUG_ASSERT(previous > 0); + signalCallbackWaiters(waitQueue, previous - 1); +} + +static void waitCallbacks(atomic_uint * inFlight, + SpiceAudioCallbackWaitQueue * waitQueue, unsigned int depth) +{ + if (atomic_load_explicit(inFlight, memory_order_acquire) <= depth) + return; + + SpiceAudioCallbackWaiter waiter = + { + .event = createWaitEvent(), + .depth = depth, + }; + + bool queued = false; + LG_LOCK(waitQueue->lock); + if (atomic_load_explicit(inFlight, memory_order_acquire) > depth) + { + waiter.next = waitQueue->waiters; + waitQueue->waiters = &waiter; + atomic_fetch_add_explicit( + &waitQueue->count, 1, memory_order_release); + queued = true; + } + LG_UNLOCK(waitQueue->lock); + + if (queued) + { + waitEvent(waiter.event); + + /* The signaler owns the waiter until it releases the queue lock. */ + LG_LOCK(waitQueue->lock); + LG_UNLOCK(waitQueue->lock); + } + + lgFreeEvent(waiter.event); +} + /* l_spice.lock must be held exclusively while admitting a callback so detach * cannot invalidate the target between the snapshot and in-flight increment. */ static bool beginEventNL(SpiceAudioEventTarget * target) @@ -142,7 +261,7 @@ static bool recordEventCurrent(const SpiceAudioEventTarget * target, static void endEvent(void) { --l_eventDepth; - atomic_fetch_sub_explicit(&l_spice.inFlight, 1, memory_order_release); + endCallback(&l_spice.inFlight, &l_spice.eventWait); } static bool sampleFormat(PSAudioFormat source, @@ -318,13 +437,11 @@ static void spiceSetStatusListener(void * opaque, { callback(callbackOpaque, &status); --l_statusDepth; - atomic_fetch_sub_explicit( - &l_spice.statusInFlight, 1, memory_order_release); + endCallback(&l_spice.statusInFlight, &l_spice.statusWait); } else - while (atomic_load_explicit( - &l_spice.statusInFlight, memory_order_acquire) > l_statusDepth) - ; + waitCallbacks(&l_spice.statusInFlight, + &l_spice.statusWait, l_statusDepth); } static bool spiceAttach(void * opaque, const LG_AudioEventOps * events, @@ -413,9 +530,7 @@ static void spiceDetach(void * opaque) nextGeneration(l_spice.eventGeneration); LG_UNLOCK_EXCLUSIVE(l_spice.lock); - while (atomic_load_explicit( - &l_spice.inFlight, memory_order_acquire) > l_eventDepth) - ; + waitCallbacks(&l_spice.inFlight, &l_spice.eventWait, l_eventDepth); } static bool spiceRecordData(void * opaque, uint32_t generation, @@ -501,8 +616,7 @@ void lgaSpice_setAvailable(bool available) { callback(callbackOpaque, &status); --l_statusDepth; - atomic_fetch_sub_explicit( - &l_spice.statusInFlight, 1, memory_order_release); + endCallback(&l_spice.statusInFlight, &l_spice.statusWait); } } diff --git a/client/src/audio_usb.c b/client/src/audio_usb.c index bb4855ff..52d94096 100644 --- a/client/src/audio_usb.c +++ b/client/src/audio_usb.c @@ -22,11 +22,15 @@ #include "usb_audio.h" +#include "common/debug.h" +#include "common/event.h" #include "common/locking.h" #include "common/time.h" +#include #include #include +#include #define USB_AUDIO_NS_PER_SECOND INT64_C(1000000000) @@ -37,49 +41,94 @@ typedef struct USBAudioCallbackFrame } USBAudioCallbackFrame; +typedef struct USBAudioCallbackWaiter +{ + struct USBAudioCallbackWaiter * next; + LGEvent * event; + unsigned int depth; +} +USBAudioCallbackWaiter; + +typedef struct USBAudioCallbackWaitQueue +{ + LG_Lock lock; + atomic_uint count; + USBAudioCallbackWaiter * waiters; +} +USBAudioCallbackWaitQueue; + +typedef struct USBAudioCallbackGate +{ + atomic_uint inFlight; + USBAudioCallbackWaitQueue wait; +} +USBAudioCallbackGate; + typedef struct USBAudioEventTarget { - const LG_AudioEventOps * events; - void * opaque; - uint32_t attachmentGeneration; - USBAudioCallbackFrame frame; + const LG_AudioEventOps * events; + void * opaque; + uint32_t attachmentGeneration; + USBAudioCallbackFrame frame; + USBAudioCallbackFrame ** frames; + USBAudioCallbackGate * gate; } USBAudioEventTarget; +typedef struct USBAudioOperationWaiter +{ + struct USBAudioOperationWaiter * next; + LGEvent * event; +} +USBAudioOperationWaiter; + +typedef struct USBAudioOperationQueue +{ + LG_Lock lock; + bool active; + USBAudioOperationWaiter * head; + USBAudioOperationWaiter * tail; +} +USBAudioOperationQueue; + struct LGA_USBState { LG_USBAudio * device; LG_USBRedir * redir; - LG_Lock statusLock; - atomic_bool available; - uint32_t statusGeneration; - LG_AudioStatusFn statusCallback; - void * statusOpaque; - atomic_uint statusInFlight; - atomic_uint_fast64_t statusNextTicket; - atomic_uint_fast64_t statusServingTicket; + LG_Lock statusLock; + atomic_bool available; + uint32_t statusGeneration; + LG_AudioStatusFn statusCallback; + void * statusOpaque; + atomic_uint statusInFlight; + USBAudioCallbackWaitQueue statusWait; + USBAudioOperationQueue statusOperation; - LG_Lock stateLock; - bool attached; - bool detaching; - uint32_t attachmentGeneration; - const LG_AudioEventOps * events; - void * eventOpaque; + LG_Lock stateLock; + LGEvent * detachEvent; + bool attached; + bool detaching; + uint32_t attachmentGeneration; + const LG_AudioEventOps * events; + void * eventOpaque; - LG_AudioFormat playbackFormat; - uint32_t playbackGeneration; - LG_AudioFormat recordFormat; - uint32_t recordGeneration; - uint32_t generationSerial; - int64_t playbackClockOrigin; - atomic_uint_fast64_t playbackPosition; - atomic_uint playbackDeliveryGeneration; - atomic_uint recordDeliveryGeneration; - atomic_uint inFlight; + LG_AudioFormat playbackFormat; + uint32_t playbackGeneration; + LG_AudioFormat recordFormat; + uint32_t recordGeneration; + uint32_t generationSerial; + int64_t playbackClockOrigin; + atomic_uint_fast64_t playbackPosition; + atomic_uint playbackDeliveryGeneration; + atomic_uint recordDeliveryGeneration; + alignas(64) USBAudioCallbackGate playbackGate; + alignas(64) USBAudioCallbackGate recordGate; + USBAudioOperationQueue recordOperation; }; -static _Thread_local USBAudioCallbackFrame * l_eventFrames; +static _Thread_local USBAudioCallbackFrame * l_playbackFrames; +static _Thread_local USBAudioCallbackFrame * l_recordFrames; static _Thread_local USBAudioCallbackFrame * l_statusFrames; static const LG_AudioFormat l_formatTemplate = @@ -139,6 +188,51 @@ static unsigned int callbackDepth( return depth; } +static LGEvent * createWaitEvent(void) +{ + LGEvent * event = lgCreateEvent(true, 0); + if (!event) + DEBUG_FATAL("Failed to create USB audio wait event"); + return event; +} + +static void waitEvent(LGEvent * event) +{ + if (!lgWaitEvent(event, TIMEOUT_INFINITE)) + DEBUG_FATAL("Failed to wait for USB audio event"); +} + +static void signalWaitEvent(LGEvent * event) +{ + if (!lgSignalEvent(event)) + DEBUG_FATAL("Failed to signal USB audio event"); +} + +static void signalCallbackWaiters( + USBAudioCallbackWaitQueue * queue, unsigned int remaining) +{ + if (!atomic_load_explicit(&queue->count, memory_order_seq_cst)) + return; + + LG_LOCK(queue->lock); + USBAudioCallbackWaiter ** link = &queue->waiters; + while (*link) + { + USBAudioCallbackWaiter * waiter = *link; + if (remaining > waiter->depth) + { + link = &waiter->next; + continue; + } + + *link = waiter->next; + atomic_fetch_sub_explicit( + &queue->count, 1, memory_order_release); + signalWaitEvent(waiter->event); + } + LG_UNLOCK(queue->lock); +} + static void beginCallback(LGA_USBState * state, USBAudioCallbackFrame * frame, USBAudioCallbackFrame ** frames, atomic_uint * inFlight) @@ -150,10 +244,59 @@ static void beginCallback(LGA_USBState * state, } static void endCallback(USBAudioCallbackFrame * frame, - USBAudioCallbackFrame ** frames, atomic_uint * inFlight) + USBAudioCallbackFrame ** frames, atomic_uint * inFlight, + USBAudioCallbackWaitQueue * waitQueue) { *frames = frame->previous; - atomic_fetch_sub_explicit(inFlight, 1, memory_order_seq_cst); + const unsigned int previous = atomic_fetch_sub_explicit( + inFlight, 1, memory_order_seq_cst); + DEBUG_ASSERT(previous > 0); + signalCallbackWaiters(waitQueue, previous - 1); +} + +static void waitCallbacks(LGA_USBState * state, + USBAudioCallbackFrame * frames, atomic_uint * inFlight, + USBAudioCallbackWaitQueue * waitQueue) +{ + const unsigned int depth = callbackDepth(frames, state); + if (atomic_load_explicit(inFlight, memory_order_seq_cst) <= depth) + return; + + USBAudioCallbackWaiter waiter = + { + .event = createWaitEvent(), + .depth = depth, + }; + + bool queued = false; + LG_LOCK(waitQueue->lock); + + /* Publish the waiter before rechecking inFlight. This prevents the final + * callback from taking the lock-free path while a waiter is being queued. */ + atomic_fetch_add_explicit( + &waitQueue->count, 1, memory_order_seq_cst); + if (atomic_load_explicit(inFlight, memory_order_seq_cst) > depth) + { + waiter.next = waitQueue->waiters; + waitQueue->waiters = &waiter; + queued = true; + } + else + atomic_fetch_sub_explicit( + &waitQueue->count, 1, memory_order_seq_cst); + + LG_UNLOCK(waitQueue->lock); + + if (queued) + { + waitEvent(waiter.event); + + /* The signaler owns the waiter until it releases the queue lock. */ + LG_LOCK(waitQueue->lock); + LG_UNLOCK(waitQueue->lock); + } + + lgFreeEvent(waiter.event); } static LG_AudioClock makePlaybackClock( @@ -178,7 +321,8 @@ static LG_AudioClock makePlaybackClock( /* stateLock must be held while admitting a control event. The data event * path performs the equivalent admission using deliveryGeneration. */ static bool beginEventNL( - LGA_USBState * state, USBAudioEventTarget * target) + LGA_USBState * state, USBAudioEventTarget * target, + USBAudioCallbackFrame ** frames, USBAudioCallbackGate * gate) { if (!state->attached || !state->events) return false; @@ -186,11 +330,27 @@ static bool beginEventNL( target->events = state->events; target->opaque = state->eventOpaque; target->attachmentGeneration = state->attachmentGeneration; + target->frames = frames; + target->gate = gate; beginCallback( - state, &target->frame, &l_eventFrames, &state->inFlight); + state, &target->frame, frames, &gate->inFlight); return true; } +static bool beginPlaybackEventNL( + LGA_USBState * state, USBAudioEventTarget * target) +{ + return beginEventNL( + state, target, &l_playbackFrames, &state->playbackGate); +} + +static bool beginRecordEventNL( + LGA_USBState * state, USBAudioEventTarget * target) +{ + return beginEventNL( + state, target, &l_recordFrames, &state->recordGate); +} + static bool attachmentCurrentNL(const LGA_USBState * state, const USBAudioEventTarget * target) { @@ -214,37 +374,84 @@ static bool recordEventCurrentNL(const LGA_USBState * state, state->recordGeneration == streamGeneration; } -static void endEvent(LGA_USBState * state, USBAudioEventTarget * target) +static void endEvent(USBAudioEventTarget * target) { - endCallback(&target->frame, &l_eventFrames, &state->inFlight); + endCallback(&target->frame, target->frames, + &target->gate->inFlight, &target->gate->wait); } -static void waitEvents(const LGA_USBState * state) +static void waitPlaybackEvents(LGA_USBState * state) { - const unsigned int depth = callbackDepth(l_eventFrames, state); - while (atomic_load_explicit( - &state->inFlight, memory_order_seq_cst) > depth) - ; + waitCallbacks(state, l_playbackFrames, + &state->playbackGate.inFlight, &state->playbackGate.wait); } -static bool beginStatusOperation(LGA_USBState * state) +static void waitRecordEvents(LGA_USBState * state) { - if (callbackDepth(l_statusFrames, state)) + waitCallbacks(state, l_recordFrames, + &state->recordGate.inFlight, &state->recordGate.wait); +} + +static bool beginOperation(LGA_USBState * state, + USBAudioOperationQueue * operation, + const USBAudioCallbackFrame * callbackFrames) +{ + /* Reentrant control must run inline: the current owner may be waiting for + * this callback to finish before it can release the operation. */ + if (callbackDepth(callbackFrames, state)) return false; - const uint_fast64_t ticket = atomic_fetch_add_explicit( - &state->statusNextTicket, 1, memory_order_relaxed); - while (atomic_load_explicit( - &state->statusServingTicket, memory_order_acquire) != ticket) - ; + USBAudioOperationWaiter waiter = + { + .event = createWaitEvent(), + }; + + bool queued = false; + LG_LOCK(operation->lock); + if (operation->active) + { + if (operation->tail) + operation->tail->next = &waiter; + else + operation->head = &waiter; + operation->tail = &waiter; + queued = true; + } + else + operation->active = true; + LG_UNLOCK(operation->lock); + + if (queued) + { + waitEvent(waiter.event); + + /* The previous owner owns the waiter until it releases the queue lock. */ + LG_LOCK(operation->lock); + LG_UNLOCK(operation->lock); + } + + lgFreeEvent(waiter.event); return true; } -static void endStatusOperation(LGA_USBState * state, bool owner) +static void endOperation( + USBAudioOperationQueue * operation, bool owner) { - if (owner) - atomic_fetch_add_explicit( - &state->statusServingTicket, 1, memory_order_release); + if (!owner) + return; + + LG_LOCK(operation->lock); + USBAudioOperationWaiter * waiter = operation->head; + if (waiter) + { + operation->head = waiter->next; + if (!operation->head) + operation->tail = NULL; + signalWaitEvent(waiter->event); + } + else + operation->active = false; + LG_UNLOCK(operation->lock); } static void beginStatusCallback( @@ -257,15 +464,32 @@ static void beginStatusCallback( static void endStatusCallback( LGA_USBState * state, USBAudioCallbackFrame * frame) { - endCallback(frame, &l_statusFrames, &state->statusInFlight); + endCallback(frame, &l_statusFrames, + &state->statusInFlight, &state->statusWait); } -static void waitStatusCallbacks(const LGA_USBState * state) +static void waitStatusCallbacks(LGA_USBState * state) { - const unsigned int depth = callbackDepth(l_statusFrames, state); - while (atomic_load_explicit( - &state->statusInFlight, memory_order_acquire) > depth) - ; + waitCallbacks(state, l_statusFrames, + &state->statusInFlight, &state->statusWait); +} + +/* Returns with stateLock held. A callback must not wait for a detach which + * may itself be waiting for that callback to return. */ +static bool lockStateAfterDetach(LGA_USBState * state) +{ + for (;;) + { + LG_LOCK(state->stateLock); + if (!state->detaching) + return true; + LG_UNLOCK(state->stateLock); + + if (callbackDepth(l_playbackFrames, state) || + callbackDepth(l_recordFrames, state)) + return false; + waitEvent(state->detachEvent); + } } static void usbPlaybackStart( @@ -294,10 +518,10 @@ static void usbPlaybackStart( atomic_store_explicit( &state->playbackDeliveryGeneration, 0, memory_order_seq_cst); clock = makePlaybackClock(state, 0); - const bool admitted = beginEventNL(state, &target); + const bool admitted = beginPlaybackEventNL(state, &target); dispatch = admitted && target.events->playbackStart; if (admitted && !dispatch) - endEvent(state, &target); + endEvent(&target); LG_UNLOCK(state->stateLock); if (!dispatch) @@ -311,7 +535,7 @@ static void usbPlaybackStart( atomic_store_explicit(&state->playbackDeliveryGeneration, generation, memory_order_seq_cst); LG_UNLOCK(state->stateLock); - endEvent(state, &target); + endEvent(&target); } static void usbPlaybackStop(void * opaque) @@ -330,10 +554,10 @@ static void usbPlaybackStop(void * opaque) state->playbackGeneration = 0; atomic_store_explicit( &state->playbackDeliveryGeneration, 0, memory_order_seq_cst); - const bool admitted = beginEventNL(state, &target); + const bool admitted = beginPlaybackEventNL(state, &target); LG_UNLOCK(state->stateLock); - waitEvents(state); + waitPlaybackEvents(state); if (!admitted) return; @@ -345,7 +569,7 @@ static void usbPlaybackStop(void * opaque) if (dispatch) target.events->playbackStop(target.opaque, generation); - endEvent(state, &target); + endEvent(&target); } static void usbPlaybackData( @@ -353,7 +577,8 @@ static void usbPlaybackData( { LGA_USBState * state = opaque; USBAudioCallbackFrame frame; - beginCallback(state, &frame, &l_eventFrames, &state->inFlight); + beginCallback(state, &frame, &l_playbackFrames, + &state->playbackGate.inFlight); const uint64_t position = atomic_fetch_add_explicit( &state->playbackPosition, frames, memory_order_relaxed); @@ -370,7 +595,8 @@ static void usbPlaybackData( target, generation, data, frames, &clock); } } - endCallback(&frame, &l_eventFrames, &state->inFlight); + endCallback(&frame, &l_playbackFrames, + &state->playbackGate.inFlight, &state->playbackGate.wait); } static void usbRecordStart( @@ -378,11 +604,15 @@ static void usbRecordStart( { LGA_USBState * state = opaque; USBAudioEventTarget target; + LG_AudioFormat format; bool dispatch; uint32_t generation; + const bool recordOwner = beginOperation( + state, &state->recordOperation, l_recordFrames); LG_LOCK(state->stateLock); setStreamFormat(&state->recordFormat, sampleRate, channelMask); + format = state->recordFormat; generation = state->recordGeneration; if (!generation) { @@ -392,49 +622,58 @@ static void usbRecordStart( } atomic_store_explicit( &state->recordDeliveryGeneration, 0, memory_order_seq_cst); - const bool admitted = beginEventNL(state, &target); + const bool admitted = beginRecordEventNL(state, &target); dispatch = admitted && target.events->recordStart; + if (dispatch) + atomic_store_explicit(&state->recordDeliveryGeneration, + generation, memory_order_seq_cst); if (admitted && !dispatch) - endEvent(state, &target); + endEvent(&target); LG_UNLOCK(state->stateLock); if (!dispatch) + { + endOperation(&state->recordOperation, recordOwner); return; + } target.events->recordStart( - target.opaque, generation, &state->recordFormat); - - LG_LOCK(state->stateLock); - if (recordEventCurrentNL(state, &target, generation)) - atomic_store_explicit(&state->recordDeliveryGeneration, - generation, memory_order_seq_cst); - LG_UNLOCK(state->stateLock); - endEvent(state, &target); + target.opaque, generation, &format); + endEvent(&target); + endOperation(&state->recordOperation, recordOwner); } static void usbRecordStop(void * opaque) { LGA_USBState * state = opaque; USBAudioEventTarget target; + const bool recordOwner = beginOperation( + state, &state->recordOperation, l_recordFrames); LG_LOCK(state->stateLock); const uint32_t generation = state->recordGeneration; if (!generation) { LG_UNLOCK(state->stateLock); + endOperation(&state->recordOperation, recordOwner); return; } state->recordGeneration = 0; atomic_store_explicit( &state->recordDeliveryGeneration, 0, memory_order_seq_cst); - const bool admitted = beginEventNL(state, &target); + const bool admitted = beginRecordEventNL(state, &target); LG_UNLOCK(state->stateLock); - waitEvents(state); + /* A reentrant stop cannot quiesce callbacks for the same reason that it + * cannot wait for ownership above. */ + waitRecordEvents(state); if (!admitted) + { + endOperation(&state->recordOperation, recordOwner); return; + } LG_LOCK(state->stateLock); const bool dispatch = attachmentCurrentNL(state, &target) && @@ -443,7 +682,8 @@ static void usbRecordStop(void * opaque) if (dispatch) target.events->recordStop(target.opaque, generation); - endEvent(state, &target); + endEvent(&target); + endOperation(&state->recordOperation, recordOwner); } static const LG_USBAudioEventOps l_usbAudioEvents = @@ -458,7 +698,8 @@ static const LG_USBAudioEventOps l_usbAudioEvents = static void usbSetAvailable(void * opaque, bool available) { LGA_USBState * state = opaque; - const bool statusOwner = beginStatusOperation(state); + const bool statusOwner = beginOperation( + state, &state->statusOperation, l_statusFrames); LG_LOCK(state->statusLock); const bool changed = atomic_load_explicit( @@ -488,14 +729,15 @@ static void usbSetAvailable(void * opaque, bool available) callback(callbackOpaque, &status); endStatusCallback(state, &frame); } - endStatusOperation(state, statusOwner); + endOperation(&state->statusOperation, statusOwner); } static void usbSetStatusListener(void * opaque, LG_AudioStatusFn callback, void * callbackOpaque) { LGA_USBState * state = opaque; - const bool statusOwner = beginStatusOperation(state); + const bool statusOwner = beginOperation( + state, &state->statusOperation, l_statusFrames); LG_LOCK(state->statusLock); state->statusCallback = callback; @@ -518,7 +760,7 @@ static void usbSetStatusListener(void * opaque, } else waitStatusCallbacks(state); - endStatusOperation(state, statusOwner); + endOperation(&state->statusOperation, statusOwner); } static bool usbAttach(void * opaque, const LG_AudioEventOps * events, @@ -529,25 +771,21 @@ static bool usbAttach(void * opaque, const LG_AudioEventOps * events, &state->available, memory_order_acquire)) return false; - USBAudioEventTarget target; + USBAudioEventTarget playbackTarget; + USBAudioEventTarget recordTarget; LG_AudioFormat playbackFormat; LG_AudioFormat recordFormat; LG_AudioClock playbackClock; + uint32_t attachmentGeneration; uint32_t playbackGeneration; uint32_t recordGeneration; bool playbackDispatch; bool recordDispatch; - bool admitted; + bool playbackAdmitted; + bool recordAdmitted; - for (;;) - { - LG_LOCK(state->stateLock); - if (!state->detaching) - break; - LG_UNLOCK(state->stateLock); - if (callbackDepth(l_eventFrames, state)) - return false; - } + if (!lockStateAfterDetach(state)) + return false; if (state->attached || !atomic_load_explicit( &state->available, memory_order_acquire)) @@ -561,61 +799,72 @@ static bool usbAttach(void * opaque, const LG_AudioEventOps * events, nextGeneration(state->attachmentGeneration); state->events = events; state->eventOpaque = eventOpaque; + attachmentGeneration = state->attachmentGeneration; playbackGeneration = state->playbackGeneration; recordGeneration = state->recordGeneration; playbackDispatch = playbackGeneration && events->playbackStart; recordDispatch = recordGeneration && events->recordStart; - admitted = (playbackDispatch || recordDispatch) && - beginEventNL(state, &target); - playbackDispatch = playbackDispatch && admitted; - recordDispatch = recordDispatch && admitted; + playbackAdmitted = playbackDispatch && + beginPlaybackEventNL(state, &playbackTarget); + playbackDispatch = playbackDispatch && playbackAdmitted; + recordAdmitted = false; if (playbackDispatch) { playbackFormat = state->playbackFormat; playbackClock = makePlaybackClock(state, atomic_load_explicit( &state->playbackPosition, memory_order_relaxed)); } - if (recordDispatch) - recordFormat = state->recordFormat; lgUsbRedir_setPlugged(state->redir, true); LG_UNLOCK(state->stateLock); if (playbackDispatch) { - target.events->playbackStart( - target.opaque, playbackGeneration, + playbackTarget.events->playbackStart( + playbackTarget.opaque, playbackGeneration, &playbackFormat, &playbackClock); LG_LOCK(state->stateLock); if (playbackEventCurrentNL( - state, &target, playbackGeneration)) + state, &playbackTarget, playbackGeneration)) atomic_store_explicit(&state->playbackDeliveryGeneration, playbackGeneration, memory_order_seq_cst); LG_UNLOCK(state->stateLock); } + if (playbackAdmitted) + endEvent(&playbackTarget); if (recordDispatch) { + const bool recordOwner = beginOperation( + state, &state->recordOperation, l_recordFrames); + LG_LOCK(state->stateLock); - recordDispatch = recordEventCurrentNL( - state, &target, recordGeneration); + recordDispatch = state->attached && + state->attachmentGeneration == attachmentGeneration && + state->events == events && + state->eventOpaque == eventOpaque && + state->recordGeneration == recordGeneration; + recordAdmitted = recordDispatch && + beginRecordEventNL(state, &recordTarget); + recordDispatch = recordAdmitted && + recordEventCurrentNL(state, &recordTarget, recordGeneration); + if (recordDispatch) + { + recordFormat = state->recordFormat; + atomic_store_explicit(&state->recordDeliveryGeneration, + recordGeneration, memory_order_seq_cst); + } LG_UNLOCK(state->stateLock); if (recordDispatch) - target.events->recordStart( - target.opaque, recordGeneration, &recordFormat); + recordTarget.events->recordStart( + recordTarget.opaque, recordGeneration, &recordFormat); - LG_LOCK(state->stateLock); - if (recordDispatch && - recordEventCurrentNL(state, &target, recordGeneration)) - atomic_store_explicit(&state->recordDeliveryGeneration, - recordGeneration, memory_order_seq_cst); - LG_UNLOCK(state->stateLock); + if (recordAdmitted) + endEvent(&recordTarget); + endOperation(&state->recordOperation, recordOwner); } - if (admitted) - endEvent(state, &target); - return true; } @@ -623,17 +872,11 @@ static void usbDetach(void * opaque) { LGA_USBState * state = opaque; - for (;;) - { - LG_LOCK(state->stateLock); - if (!state->detaching) - break; - LG_UNLOCK(state->stateLock); - if (callbackDepth(l_eventFrames, state)) - return; - } + if (!lockStateAfterDetach(state)) + return; state->detaching = true; + lgResetEvent(state->detachEvent); state->attached = false; state->attachmentGeneration = nextGeneration(state->attachmentGeneration); @@ -646,7 +889,8 @@ static void usbDetach(void * opaque) lgUsbRedir_setPlugged(state->redir, false); LG_UNLOCK(state->stateLock); - waitEvents(state); + waitPlaybackEvents(state); + waitRecordEvents(state); LG_LOCK(state->stateLock); if (!state->attached && @@ -655,6 +899,7 @@ static void usbDetach(void * opaque) state->events = NULL; state->eventOpaque = NULL; state->detaching = false; + signalWaitEvent(state->detachEvent); } LG_UNLOCK(state->stateLock); } @@ -666,7 +911,8 @@ static bool usbClockFeedback(void * opaque, uint32_t generation, LG_LOCK(state->stateLock); if (!state->attached || - state->playbackGeneration != generation) + state->playbackGeneration != generation || + !lgUsbAudio_feedbackActive(state->device)) { LG_UNLOCK(state->stateLock); return false; @@ -686,17 +932,19 @@ static bool usbRecordData(void * opaque, uint32_t generation, const void * data, size_t frames, const LG_AudioClock * sourceClock) { - (void)sourceClock; LGA_USBState * state = opaque; USBAudioCallbackFrame frame; - beginCallback(state, &frame, &l_eventFrames, &state->inFlight); + beginCallback(state, &frame, &l_recordFrames, + &state->recordGate.inFlight); const bool valid = generation && generation == atomic_load_explicit( &state->recordDeliveryGeneration, memory_order_seq_cst); const bool result = valid && - lgUsbAudio_recordData(state->device, data, frames); + lgUsbAudio_recordData( + state->device, data, frames, sourceClock); - endCallback(&frame, &l_eventFrames, &state->inFlight); + endCallback(&frame, &l_recordFrames, + &state->recordGate.inFlight, &state->recordGate.wait); return result; } @@ -710,28 +958,47 @@ const LG_AudioOps LGA_USB = .clockFeedback = usbClockFeedback, }; -LGA_USBState * lgaUsb_create(void) +LGA_USBState * lgaUsb_create(bool debug) { - LGA_USBState * state = calloc(1, sizeof(*state)); + LGA_USBState * state = aligned_alloc( + alignof(LGA_USBState), sizeof(*state)); if (!state) return NULL; + memset(state, 0, sizeof(*state)); LG_LOCK_INIT(state->statusLock); + LG_LOCK_INIT(state->statusWait.lock); + LG_LOCK_INIT(state->statusOperation.lock); LG_LOCK_INIT(state->stateLock); + LG_LOCK_INIT(state->playbackGate.wait.lock); + LG_LOCK_INIT(state->recordGate.wait.lock); + LG_LOCK_INIT(state->recordOperation.lock); atomic_init(&state->available, false); atomic_init(&state->playbackPosition, 0); atomic_init(&state->playbackDeliveryGeneration, 0); atomic_init(&state->recordDeliveryGeneration, 0); - atomic_init(&state->inFlight, 0); + atomic_init(&state->playbackGate.inFlight, 0); + atomic_init(&state->recordGate.inFlight, 0); atomic_init(&state->statusInFlight, 0); - atomic_init(&state->statusNextTicket, 0); - atomic_init(&state->statusServingTicket, 0); + atomic_init(&state->playbackGate.wait.count, 0); + atomic_init(&state->recordGate.wait.count, 0); + atomic_init(&state->statusWait.count, 0); state->playbackFormat = l_formatTemplate; state->recordFormat = l_formatTemplate; - state->device = lgUsbAudio_create(&l_usbAudioEvents, state); + state->detachEvent = lgCreateEvent(false, 0); + if (!state->detachEvent || !lgSignalEvent(state->detachEvent)) + { + if (state->detachEvent) + lgFreeEvent(state->detachEvent); + free(state); + return NULL; + } + + state->device = lgUsbAudio_create(&l_usbAudioEvents, state, debug); if (!state->device) { + lgFreeEvent(state->detachEvent); free(state); return NULL; } @@ -741,6 +1008,7 @@ LGA_USBState * lgaUsb_create(void) if (!state->redir) { lgUsbAudio_destroy(state->device); + lgFreeEvent(state->detachEvent); free(state); return NULL; } @@ -757,7 +1025,13 @@ void lgaUsb_destroy(LGA_USBState * state) usbDetach(state); lgUsbRedir_destroy(state->redir); lgUsbAudio_destroy(state->device); + lgFreeEvent(state->detachEvent); + LG_LOCK_FREE(state->recordOperation.lock); + LG_LOCK_FREE(state->recordGate.wait.lock); + LG_LOCK_FREE(state->playbackGate.wait.lock); LG_LOCK_FREE(state->stateLock); + LG_LOCK_FREE(state->statusOperation.lock); + LG_LOCK_FREE(state->statusWait.lock); LG_LOCK_FREE(state->statusLock); free(state); } @@ -767,7 +1041,7 @@ LG_USBRedir * lgaUsb_redir(LGA_USBState * state) return state ? state->redir : NULL; } -bool lgaUsb_recording(const LGA_USBState * state) +uint64_t lgaUsb_processDelayNs(const LGA_USBState * state) { - return state && lgUsbAudio_recording(state->device); + return state ? lgUsbAudio_processDelayNs(state->device) : UINT64_MAX; } diff --git a/client/src/audio_usb.h b/client/src/audio_usb.h index b704a491..5f26ce87 100644 --- a/client/src/audio_usb.h +++ b/client/src/audio_usb.h @@ -24,14 +24,16 @@ #include "interface/audio.h" #include "usbredir.h" +#include + typedef struct LGA_USBState LGA_USBState; -LGA_USBState * lgaUsb_create(void); +LGA_USBState * lgaUsb_create(bool debug); /* Detach this provider and stop PureSpice before destroying its state. */ void lgaUsb_destroy(LGA_USBState * state); LG_USBRedir * lgaUsb_redir(LGA_USBState * state); -bool lgaUsb_recording(const LGA_USBState * state); +uint64_t lgaUsb_processDelayNs(const LGA_USBState * state); extern const LG_AudioOps LGA_USB; diff --git a/client/src/main.c b/client/src/main.c index 6aec5bc6..cf3515b7 100644 --- a/client/src/main.c +++ b/client/src/main.c @@ -1728,7 +1728,7 @@ int spiceThread(void * arg) DEBUG_WARN("USB audio requires a playback backend, using SPICE audio"); g_params.useSpiceUSBAudio = false; } - else if (!(usbAudio = lgaUsb_create())) + else if (!(usbAudio = lgaUsb_create(g_params.audioDebug))) { DEBUG_WARN("Failed to initialize USB audio, using SPICE audio"); g_params.useSpiceUSBAudio = false; @@ -1839,7 +1839,17 @@ int spiceThread(void * arg) if (usbRedir && !lgUsbRedir_process(usbRedir)) DEBUG_WARN("Failed to process USB audio redirection"); if (usbAudio) - processTimeout = lgaUsb_recording(usbAudio) ? 1 : 10; + { + const uint64_t delay = lgaUsb_processDelayNs(usbAudio); + if (delay == UINT64_MAX) + processTimeout = 10; + else + { + const uint64_t timeout = delay / UINT64_C(1000000) + + (delay % UINT64_C(1000000) != 0); + processTimeout = (int)min(timeout, UINT64_C(10)); + } + } #endif if ((status = purespice_process(processTimeout)) != PS_STATUS_RUN) diff --git a/client/src/overlay/status.c b/client/src/overlay/status.c index c9f83507..e35876a1 100644 --- a/client/src/overlay/status.c +++ b/client/src/overlay/status.c @@ -21,6 +21,7 @@ #include "interface/overlay.h" #include "math.h" #include "cimgui.h" +#include #include "../overlays.h" #include "../main.h" @@ -32,7 +33,7 @@ //TODO: Make this user configurable? #define ICON_SIZE 32 -static bool l_state[LG_USER_STATUS_MAX] = { 0 }; +static atomic_bool l_state[LG_USER_STATUS_MAX] = { 0 }; static OverlayImage l_image[LG_USER_STATUS_MAX] = { 0 }; static bool l_recordToggle; static double l_scale = 1.0; @@ -92,7 +93,8 @@ static int status_render(void * udata, bool interactive, struct Rect * windowRec for(int i = 0; i < LG_USER_STATUS_MAX; ++i) { OverlayImage * img = &l_image[i]; - if (!l_state[i] || !img->tex) + if (!atomic_load_explicit(&l_state[i], memory_order_relaxed) || + !img->tex) continue; // if the recording indicator is off, don't draw but reserve space @@ -147,9 +149,9 @@ struct LG_OverlayOps LGOverlayStatus = void overlayStatus_set(LGUserStatus status, bool value) { - if (l_state[status] == value) + if (atomic_exchange_explicit( + &l_state[status], value, memory_order_relaxed) == value) return; - l_state[status] = value; app_invalidateOverlay(true); }; diff --git a/client/src/usb_audio.c b/client/src/usb_audio.c index 2c65d8b3..a9bcde97 100644 --- a/client/src/usb_audio.c +++ b/client/src/usb_audio.c @@ -21,12 +21,13 @@ #include "usb_audio.h" #include "common/debug.h" -#include "common/locking.h" +#include "common/event.h" #include "common/ringbuffer.h" #include "common/time.h" #include +#include #include #include #include @@ -60,7 +61,15 @@ enum USB_AUDIO_FEEDBACK_SIZE = 4, USB_AUDIO_FEEDBACK_MAX_QUEUE = 512, USB_AUDIO_SAMPLE_SIZE = 3, + USB_AUDIO_MAX_SAMPLE_RATE = 192000, + USB_AUDIO_PLAYBACK_MAX_CHANNELS = 8, USB_AUDIO_PLAYBACK_HS_PACKET_FRAMES = 25, + USB_AUDIO_PLAYBACK_BATCH_PACKETS = USB_AUDIO_FEEDBACK_INTERVAL, + USB_AUDIO_PLAYBACK_BATCH_FRAMES = + USB_AUDIO_PLAYBACK_HS_PACKET_FRAMES * USB_AUDIO_PLAYBACK_BATCH_PACKETS, + USB_AUDIO_PLAYBACK_BATCH_SIZE = + USB_AUDIO_PLAYBACK_BATCH_FRAMES * + USB_AUDIO_PLAYBACK_MAX_CHANNELS * USB_AUDIO_SAMPLE_SIZE, USB_AUDIO_FS_PACKET_FRAMES = 97, USB_AUDIO_RECORD_PACKET_FRAMES = 97, USB_AUDIO_RECORD_PACKETS_PER_SECOND = 2000, @@ -70,8 +79,15 @@ enum USB_AUDIO_RECORD_RATE_GAP_NS = 100000000, USB_AUDIO_RECORD_RATE_DENOMINATOR = USB_AUDIO_RECORD_PACKETS_PER_SECOND * USB_AUDIO_RECORD_RATE_SCALE, - USB_AUDIO_RECORD_MAX_BATCH = 64, - USB_AUDIO_RECORD_QUEUE_FRAMES = 3840, + USB_AUDIO_RECORD_MAX_BATCH = 128, + USB_AUDIO_RECORD_MAX_SAFETY_PACKETS = + USB_AUDIO_RECORD_PACKETS_PER_SECOND / 10, + /* Hold the complete QEMU ISO-IN safety window in addition to the normal + * 20 ms queue target at the maximum supported sample rate. */ + USB_AUDIO_RECORD_QUEUE_FRAMES = + USB_AUDIO_RECORD_MAX_SAFETY_PACKETS * + USB_AUDIO_RECORD_PACKET_FRAMES + + USB_AUDIO_MAX_SAMPLE_RATE / 50, USB_AUDIO_RECORD_FRAME_SIZE = 2 * USB_AUDIO_SAMPLE_SIZE, USB_AUDIO_RECORD_PACKET_SIZE = @@ -82,6 +98,8 @@ enum USB_AUDIO_RECORD_STREAM_DESC_SIZE = 46, }; +#define USB_AUDIO_RECORD_DEBUG_INTERVAL_NS UINT64_C(5000000000) + enum { USB_AUDIO_LAYOUT_STEREO = 0x00000003, @@ -299,7 +317,7 @@ static const uint32_t l_sampleRates[] = 88200, 96000, 176400, - 192000, + USB_AUDIO_MAX_SAMPLE_RATE, }; static const uint8_t l_clockValid[] = { 0x01 }; @@ -327,33 +345,67 @@ _Static_assert(USB_AUDIO_FS_PACKET_FRAMES * 2 * USB_AUDIO_SAMPLE_SIZE <= _Static_assert(USB_AUDIO_RECORD_PACKET_SIZE <= 1024, "high-speed recording packet is too large"); +typedef enum +{ + RECORD_RATE_NONE, + RECORD_RATE_ARRIVAL, + RECORD_RATE_SOURCE +} +RecordRateMode; + struct LG_USBAudio { const LG_USBAudioEventOps * events; void * eventOpaque; struct usbredirparser * parser; + bool debug; - uint8_t configuration; - uint8_t playbackAlt; - uint8_t recordAlt; - uint32_t playbackSampleRate; - uint32_t recordSampleRate; - bool playbackStreaming; - bool feedbackStreaming; - uint64_t feedbackPacketId; - uint8_t feedbackDataPackets; - atomic_uint feedbackValue; + uint8_t configuration; + uint8_t playbackAlt; + uint8_t recordAlt; + uint32_t playbackSampleRate; + atomic_uint recordSampleRate; + bool playbackStreaming; + uint8_t playbackBatchPackets; + uint8_t playbackBatchFrameSize; + uint16_t playbackBatchFrames; + atomic_bool feedbackStreaming; + uint64_t feedbackPacketId; + uint8_t feedbackDataPackets; + atomic_uint feedbackValue; - atomic_bool recordStreaming; - uint64_t recordPacketId; - uint64_t recordPacketPhase; - uint64_t recordNextPacketTime; - LG_Lock recordLock; - RingBuffer recordBuffer; - uint64_t recordRateQ16; - uint64_t recordRateStartTime; - uint64_t recordRateLastTime; - uint64_t recordRateFrames; + atomic_bool recordStreaming; + atomic_bool recordAccepting; + atomic_bool recordOverflowPending; + atomic_uint recordGeneration; + atomic_uint recordInFlight; + LGEvent * recordIdle; + uint64_t recordPacketId; + uint64_t recordPacketPhase; + uint64_t recordNextPacketTime; + uint32_t recordBatchPackets; + uint32_t recordSafetyPackets; + uint32_t recordRefillPackets; + RingBuffer recordBuffer; + atomic_uint_fast64_t recordRateQ16; + RecordRateMode recordRateMode; + uint64_t recordArrivalStartTime; + uint64_t recordArrivalLastTime; + uint64_t recordArrivalFrames; + uint64_t recordSourceStartPosition; + uint64_t recordSourceLastPosition; + int64_t recordSourceStartTime; + int64_t recordSourceLastTime; + + atomic_uint_fast64_t recordOverflowFrames; + atomic_uint_fast64_t recordUnderflowFrames; + atomic_uint_fast64_t recordTrimmedFrames; + atomic_uint_fast64_t recordLatePackets; + atomic_uint_fast64_t recordMaxLateness; + atomic_uint_fast64_t recordDiscontinuities; + uint64_t recordNextDebugTime; + + uint8_t playbackBatch[USB_AUDIO_PLAYBACK_BATCH_SIZE]; }; static LG_USBAudio * getAudio(void * opaque) @@ -368,6 +420,43 @@ static uint64_t recordTime(void) return microtime() * UINT64_C(1000); } +static void recordCounterMax( + atomic_uint_fast64_t * counter, uint64_t value) +{ + uint_fast64_t current = atomic_load_explicit( + counter, memory_order_relaxed); + while (current < value && !atomic_compare_exchange_weak_explicit( + counter, ¤t, value, + memory_order_relaxed, memory_order_relaxed)) + ; +} + +static void recordDisableInput(LG_USBAudio * audio) +{ + /* A producer takes the generation before entering recordInFlight. Changing + * both values prevents a callback which raced the gate from entering after + * the consumer has reset the SPSC queue. */ + atomic_store_explicit( + &audio->recordAccepting, false, memory_order_seq_cst); + atomic_fetch_add_explicit( + &audio->recordGeneration, 1, memory_order_seq_cst); +} + +static void recordWaitForInput(LG_USBAudio * audio) +{ + while (atomic_load_explicit( + &audio->recordInFlight, memory_order_seq_cst)) + if (!lgWaitEvent(audio->recordIdle, TIMEOUT_INFINITE)) + DEBUG_FATAL("Failed to wait for USB capture callbacks"); +} + +static int recordDiscardQueued(LG_USBAudio * audio) +{ + const int queued = ringbuffer_getCount(audio->recordBuffer); + return queued > 0 ? + ringbuffer_consume(audio->recordBuffer, NULL, queued) : 0; +} + static uint32_t readLE32(const uint8_t * data) { return @@ -454,8 +543,43 @@ static size_t writeSampleRateRange(uint8_t * buffer) return 2 + count * 12; } +static void flushPlayback(LG_USBAudio * audio) +{ + const size_t frames = audio->playbackBatchFrames; + audio->playbackBatchPackets = 0; + audio->playbackBatchFrameSize = 0; + audio->playbackBatchFrames = 0; + + if (frames && audio->playbackStreaming && audio->events && + audio->events->playbackData) + audio->events->playbackData( + audio->eventOpaque, audio->playbackBatch, frames); +} + +static void queuePlaybackPacket(LG_USBAudio * audio, + const uint8_t * data, size_t frames, uint8_t frameSize) +{ + if (audio->playbackBatchPackets == USB_AUDIO_PLAYBACK_BATCH_PACKETS || + audio->playbackBatchFrames + frames > + USB_AUDIO_PLAYBACK_BATCH_FRAMES || + (audio->playbackBatchFrames && + audio->playbackBatchFrameSize != frameSize)) + flushPlayback(audio); + + if (frames) + { + memcpy(audio->playbackBatch + + (size_t)audio->playbackBatchFrames * frameSize, + data, frames * frameSize); + audio->playbackBatchFrameSize = frameSize; + audio->playbackBatchFrames += (uint16_t)frames; + } + ++audio->playbackBatchPackets; +} + static void stopPlayback(LG_USBAudio * audio) { + flushPlayback(audio); if (!audio->playbackStreaming) return; @@ -467,7 +591,8 @@ static void stopPlayback(LG_USBAudio * audio) static void stopFeedback(LG_USBAudio * audio) { - audio->feedbackStreaming = false; + atomic_store_explicit( + &audio->feedbackStreaming, false, memory_order_release); audio->feedbackDataPackets = 0; } @@ -493,29 +618,184 @@ static void startPlayback(LG_USBAudio * audio) static void stopRecord(LG_USBAudio * audio) { - if (!atomic_exchange_explicit( - &audio->recordStreaming, false, memory_order_acq_rel)) + if (!atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire)) return; + recordDisableInput(audio); + atomic_store_explicit( + &audio->recordStreaming, false, memory_order_release); + if (audio->events && audio->events->recordStop) audio->events->recordStop(audio->eventOpaque); - LG_LOCK(audio->recordLock); + recordWaitForInput(audio); ringbuffer_reset(audio->recordBuffer); - LG_UNLOCK(audio->recordLock); + atomic_store_explicit( + &audio->recordOverflowPending, false, memory_order_relaxed); + audio->recordRefillPackets = 0; +} + +static void clearRecordRateWindows(LG_USBAudio * audio) +{ + audio->recordRateMode = RECORD_RATE_NONE; + audio->recordArrivalStartTime = 0; + audio->recordArrivalLastTime = 0; + audio->recordArrivalFrames = 0; + audio->recordSourceStartPosition = 0; + audio->recordSourceLastPosition = 0; + audio->recordSourceStartTime = 0; + audio->recordSourceLastTime = 0; +} + +static void resetRecordRate( + LG_USBAudio * audio, uint32_t sampleRate) +{ + atomic_store_explicit(&audio->recordRateQ16, + (uint64_t)sampleRate * USB_AUDIO_RECORD_RATE_SCALE, + memory_order_release); + clearRecordRateWindows(audio); +} + +static bool recordRateInRange( + uint32_t sampleRate, double measured) +{ + return measured >= sampleRate * 0.995 && + measured <= sampleRate * 1.005; +} + +static void updateMeasuredRecordRate( + LG_USBAudio * audio, double measured) +{ + const uint64_t measuredQ16 = (uint64_t)( + measured * USB_AUDIO_RECORD_RATE_SCALE + 0.5); + const uint64_t currentQ16 = atomic_load_explicit( + &audio->recordRateQ16, memory_order_relaxed); + atomic_store_explicit(&audio->recordRateQ16, + (currentQ16 * 3 + measuredQ16) / 4, + memory_order_release); +} + +static void startRecordArrivalWindow( + LG_USBAudio * audio, uint64_t now) +{ + clearRecordRateWindows(audio); + audio->recordRateMode = RECORD_RATE_ARRIVAL; + audio->recordArrivalStartTime = now; + audio->recordArrivalLastTime = now; +} + +static void updateRecordArrivalRate( + LG_USBAudio * audio, uint32_t sampleRate, size_t frames) +{ + const uint64_t now = recordTime(); + if (audio->recordRateMode != RECORD_RATE_ARRIVAL) + { + /* Preserve the current rate while a temporarily absent source clock + * starts a fresh fallback window. */ + startRecordArrivalWindow(audio, now); + return; + } + + if (now < audio->recordArrivalLastTime || + now - audio->recordArrivalLastTime > USB_AUDIO_RECORD_RATE_GAP_NS) + { + atomic_store_explicit(&audio->recordRateQ16, + (uint64_t)sampleRate * USB_AUDIO_RECORD_RATE_SCALE, + memory_order_release); + startRecordArrivalWindow(audio, now); + return; + } + + audio->recordArrivalFrames += frames; + audio->recordArrivalLastTime = now; + const uint64_t elapsed = now - audio->recordArrivalStartTime; + if (elapsed < USB_AUDIO_RECORD_RATE_SETTLE_NS) + return; + + const double measured = + (double)audio->recordArrivalFrames * 1000000000.0 / elapsed; + if (recordRateInRange(sampleRate, measured)) + updateMeasuredRecordRate(audio, measured); + + audio->recordArrivalStartTime = now; + audio->recordArrivalFrames = 0; +} + +static void startRecordSourceWindow( + LG_USBAudio * audio, const LG_AudioClock * sourceClock) +{ + clearRecordRateWindows(audio); + audio->recordRateMode = RECORD_RATE_SOURCE; + audio->recordSourceStartPosition = sourceClock->position; + audio->recordSourceLastPosition = sourceClock->position; + audio->recordSourceStartTime = sourceClock->time; + audio->recordSourceLastTime = sourceClock->time; +} + +static bool updateRecordSourceRate(LG_USBAudio * audio, + uint32_t sampleRate, const LG_AudioClock * sourceClock) +{ + if (sourceClock->discontinuity) + { + resetRecordRate(audio, sampleRate); + return false; + } + + if (audio->recordRateMode != RECORD_RATE_SOURCE) + { + /* Clock stability is not needed to establish the first position/time + * baseline. */ + startRecordSourceWindow(audio, sourceClock); + return true; + } + + if (sourceClock->position <= audio->recordSourceLastPosition || + sourceClock->time <= audio->recordSourceLastTime) + { + resetRecordRate(audio, sampleRate); + return false; + } + + if (!sourceClock->stable) + { + /* Keep an unstable clock as the next baseline, but do not let it + * contribute to the measured device rate. */ + startRecordSourceWindow(audio, sourceClock); + return true; + } + + audio->recordSourceLastPosition = sourceClock->position; + audio->recordSourceLastTime = sourceClock->time; + const int64_t elapsed = + sourceClock->time - audio->recordSourceStartTime; + if (elapsed < USB_AUDIO_RECORD_RATE_SETTLE_NS) + return true; + + const double measured = + (double)(sourceClock->position - + audio->recordSourceStartPosition) * 1000000000.0 / elapsed; + if (!recordRateInRange(sampleRate, measured)) + { + resetRecordRate(audio, sampleRate); + return false; + } + + updateMeasuredRecordRate(audio, measured); + audio->recordSourceStartPosition = sourceClock->position; + audio->recordSourceStartTime = sourceClock->time; + return true; } static void resetRecordData(LG_USBAudio * audio) { - LG_LOCK(audio->recordLock); ringbuffer_reset(audio->recordBuffer); - audio->recordRateQ16 = - (uint64_t)audio->recordSampleRate * USB_AUDIO_RECORD_RATE_SCALE; - audio->recordRateStartTime = 0; - audio->recordRateLastTime = 0; - audio->recordRateFrames = 0; - LG_UNLOCK(audio->recordLock); - audio->recordPacketPhase = 0; + resetRecordRate(audio, atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed)); + audio->recordPacketPhase = 0; + audio->recordRefillPackets = 0; + atomic_store_explicit( + &audio->recordOverflowPending, false, memory_order_relaxed); } static void startRecord(LG_USBAudio * audio) @@ -524,6 +804,8 @@ static void startRecord(LG_USBAudio * audio) &audio->recordStreaming, memory_order_acquire)) return; + recordDisableInput(audio); + recordWaitForInput(audio); resetRecordData(audio); audio->recordPacketId = 0; atomic_store_explicit( @@ -531,27 +813,39 @@ static void startRecord(LG_USBAudio * audio) if (audio->events && audio->events->recordStart) audio->events->recordStart(audio->eventOpaque, - audio->recordSampleRate, USB_AUDIO_LAYOUT_STEREO); + atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed), + USB_AUDIO_LAYOUT_STEREO); /* Backend startup may block while the capture graph is configured. Do not * turn that setup time into a burst of stale USB packets. */ + recordWaitForInput(audio); audio->recordNextPacketTime = recordTime(); + atomic_store_explicit( + &audio->recordAccepting, true, memory_order_seq_cst); } -static void reconfigureRecord(LG_USBAudio * audio) +static void reconfigureRecord( + LG_USBAudio * audio, uint32_t sampleRate) { if (!atomic_load_explicit( &audio->recordStreaming, memory_order_acquire)) return; + recordDisableInput(audio); + atomic_store_explicit( + &audio->recordSampleRate, sampleRate, memory_order_relaxed); if (audio->events && audio->events->recordStart) audio->events->recordStart(audio->eventOpaque, - audio->recordSampleRate, USB_AUDIO_LAYOUT_STEREO); + sampleRate, USB_AUDIO_LAYOUT_STEREO); /* The provider restart has quiesced the old producer. Drop any frames * queued around the format transition and restart the USB sample phase. */ + recordWaitForInput(audio); resetRecordData(audio); audio->recordNextPacketTime = recordTime(); + atomic_store_explicit( + &audio->recordAccepting, true, memory_order_seq_cst); } static void stopStreams(LG_USBAudio * audio) @@ -577,17 +871,21 @@ static void setPlaybackSampleRate( static void setRecordSampleRate( LG_USBAudio * audio, uint32_t sampleRate) { - LG_LOCK(audio->recordLock); - const bool changed = audio->recordSampleRate != sampleRate; - audio->recordSampleRate = sampleRate; - LG_UNLOCK(audio->recordLock); - if (!changed) + if (atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed) == sampleRate) return; - const bool restartRecord = atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire); - if (restartRecord) - reconfigureRecord(audio); + if (atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire)) + reconfigureRecord(audio, sampleRate); + else + { + atomic_store_explicit( + &audio->recordSampleRate, sampleRate, memory_order_relaxed); + atomic_store_explicit(&audio->recordRateQ16, + (uint64_t)sampleRate * USB_AUDIO_RECORD_RATE_SCALE, + memory_order_relaxed); + } } static void resetDevice(LG_USBAudio * audio) @@ -597,9 +895,14 @@ static void resetDevice(LG_USBAudio * audio) audio->playbackAlt = 0; audio->recordAlt = 0; audio->playbackSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; - LG_LOCK(audio->recordLock); - audio->recordSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; - LG_UNLOCK(audio->recordLock); + audio->recordBatchPackets = 1; + audio->recordSafetyPackets = 1; + atomic_store_explicit(&audio->recordSampleRate, + LG_USB_AUDIO_DEFAULT_SAMPLE_RATE, memory_order_relaxed); + atomic_store_explicit(&audio->recordRateQ16, + (uint64_t)LG_USB_AUDIO_DEFAULT_SAMPLE_RATE * + USB_AUDIO_RECORD_RATE_SCALE, + memory_order_relaxed); resetFeedbackRate(audio); } @@ -718,6 +1021,7 @@ static void setAltSetting(void * opaque, uint64_t id, struct usb_redir_set_alt_setting_header * request) { LG_USBAudio * audio = getAudio(opaque); + flushPlayback(audio); struct usb_redir_alt_setting_status_header status = { .status = usb_redir_stall, @@ -812,40 +1116,19 @@ static void sendFeedbackPackets(LG_USBAudio * audio, uint32_t count) &packet, data, USB_AUDIO_FEEDBACK_SIZE); } -static void sendRecordPackets(LG_USBAudio * audio) +static void sendRecordPacketBatch(LG_USBAudio * audio, uint32_t count, + uint64_t rateQ16, bool forceSilence, uint64_t leadingSilenceFrames, + uint64_t availableFrames) { - if (!atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire)) - return; - - const uint64_t now = recordTime(); - if (audio->recordNextPacketTime > now) - return; - - uint64_t count = - (now - audio->recordNextPacketTime) / - USB_AUDIO_RECORD_PACKET_INTERVAL_NS + 1; - if (count > USB_AUDIO_RECORD_MAX_BATCH) - { - count = USB_AUDIO_RECORD_MAX_BATCH; - audio->recordNextPacketTime = now - - (count - 1) * USB_AUDIO_RECORD_PACKET_INTERVAL_NS; - } - uint8_t data[USB_AUDIO_RECORD_PACKET_SIZE]; - const int target = max( - (int)(audio->recordSampleRate / 50), - USB_AUDIO_RECORD_PACKET_FRAMES); - LG_LOCK(audio->recordLock); - const uint64_t rateQ16 = audio->recordRateQ16; - LG_UNLOCK(audio->recordLock); + uint64_t underflowFrames = 0; struct usb_redir_iso_packet_header packet = { .endpoint = USB_AUDIO_RECORD_DATA_ENDPOINT, .status = usb_redir_success, }; - for (uint64_t i = 0; i < count; ++i) + for (uint32_t i = 0; i < count; ++i) { audio->recordPacketPhase += rateQ16; const uint32_t frames = (uint32_t)(audio->recordPacketPhase / @@ -856,25 +1139,131 @@ static void sendRecordPackets(LG_USBAudio * audio) const size_t size = frames * USB_AUDIO_RECORD_FRAME_SIZE; memset(data, 0, size); - LG_LOCK(audio->recordLock); - const int queued = ringbuffer_getCount(audio->recordBuffer); - if (queued > target) - ringbuffer_consume(audio->recordBuffer, NULL, queued - target); - ringbuffer_consume(audio->recordBuffer, data, frames); - LG_UNLOCK(audio->recordLock); + if (!forceSilence) + { + const uint32_t leading = (uint32_t)min( + leadingSilenceFrames, (uint64_t)frames); + leadingSilenceFrames -= leading; + underflowFrames += leading; + + const uint32_t requested = frames - leading; + const uint32_t available = (uint32_t)min( + availableFrames, (uint64_t)requested); + const int consumed = ringbuffer_consume(audio->recordBuffer, + data + (size_t)leading * USB_AUDIO_RECORD_FRAME_SIZE, + available); + availableFrames -= consumed; + underflowFrames += requested - consumed; + } packet.length = (uint16_t)size; usbredirparser_send_iso_packet(audio->parser, audio->recordPacketId++, &packet, data, (int)size); - audio->recordNextPacketTime += - USB_AUDIO_RECORD_PACKET_INTERVAL_NS; } + + if (audio->debug && underflowFrames) + atomic_fetch_add_explicit(&audio->recordUnderflowFrames, + underflowFrames, memory_order_relaxed); +} + +static void sendRecordRefill(LG_USBAudio * audio, uint64_t rateQ16) +{ + const uint32_t count = min( + audio->recordRefillPackets, USB_AUDIO_RECORD_MAX_BATCH); + sendRecordPacketBatch(audio, count, rateQ16, true, 0, 0); + audio->recordRefillPackets -= count; + if (!audio->recordRefillPackets) + audio->recordNextPacketTime = + recordTime() + USB_AUDIO_RECORD_PACKET_INTERVAL_NS; +} + +static void sendRecordPackets(LG_USBAudio * audio) +{ + if (!atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire)) + return; + + const uint64_t rateQ16 = atomic_load_explicit( + &audio->recordRateQ16, memory_order_acquire); + if (audio->recordRefillPackets) + { + sendRecordRefill(audio, rateQ16); + return; + } + + const uint64_t now = recordTime(); + if (audio->recordNextPacketTime > now) + return; + + const bool overflow = atomic_exchange_explicit( + &audio->recordOverflowPending, false, memory_order_acq_rel); + const uint64_t lateness = now - audio->recordNextPacketTime; + uint64_t count = lateness / USB_AUDIO_RECORD_PACKET_INTERVAL_NS + 1; + if (audio->debug) + { + if (count > 1) + atomic_fetch_add_explicit(&audio->recordLatePackets, + count - 1, memory_order_relaxed); + recordCounterMax(&audio->recordMaxLateness, lateness); + } + + if (overflow || count > audio->recordSafetyPackets) + { + const int discarded = recordDiscardQueued(audio); + if (audio->debug) + { + if (discarded) + atomic_fetch_add_explicit(&audio->recordTrimmedFrames, + discarded, memory_order_relaxed); + atomic_fetch_add_explicit(&audio->recordDiscontinuities, + 1, memory_order_relaxed); + } + + audio->recordPacketPhase = 0; + audio->recordRefillPackets = audio->recordSafetyPackets; + sendRecordRefill(audio, rateQ16); + return; + } + + const uint32_t sendCount = (uint32_t)min( + count, (uint64_t)USB_AUDIO_RECORD_MAX_BATCH); + const uint64_t catchupFrames = + (audio->recordPacketPhase + rateQ16 * count) / + USB_AUDIO_RECORD_RATE_DENOMINATOR; + const uint64_t totalFrames = + (audio->recordPacketPhase + rateQ16 * sendCount) / + USB_AUDIO_RECORD_RATE_DENOMINATOR; + const uint32_t sampleRate = atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed); + /* Preserve the samples needed by every overdue packet. Trimming to the + * normal target first would replace valid catch-up audio with silence. */ + const int retainFrames = max( + (int)catchupFrames, + max((int)(sampleRate / 50), USB_AUDIO_RECORD_PACKET_FRAMES)); + int queued = ringbuffer_getCount(audio->recordBuffer); + if (queued > retainFrames) + { + const int discarded = ringbuffer_consume( + audio->recordBuffer, NULL, queued - retainFrames); + queued -= discarded; + if (audio->debug && discarded) + atomic_fetch_add_explicit(&audio->recordTrimmedFrames, + discarded, memory_order_relaxed); + } + + const uint64_t leadingSilenceFrames = + totalFrames > (uint64_t)queued ? totalFrames - queued : 0; + sendRecordPacketBatch(audio, sendCount, rateQ16, false, + leadingSilenceFrames, queued); + audio->recordNextPacketTime += + (uint64_t)sendCount * USB_AUDIO_RECORD_PACKET_INTERVAL_NS; } static void startISOStream(void * opaque, uint64_t id, struct usb_redir_start_iso_stream_header * request) { LG_USBAudio * audio = getAudio(opaque); + flushPlayback(audio); uint8_t result = usb_redir_stall; uint32_t feedbackPrefill = 0; @@ -893,7 +1282,8 @@ static void startISOStream(void * opaque, uint64_t id, if (!getLayout(audio->playbackAlt)) break; resetFeedbackRate(audio); - audio->feedbackStreaming = true; + atomic_store_explicit( + &audio->feedbackStreaming, true, memory_order_release); audio->feedbackDataPackets = 0; audio->feedbackPacketId = 0; feedbackPrefill = @@ -909,6 +1299,12 @@ static void startISOStream(void * opaque, uint64_t id, case USB_AUDIO_RECORD_DATA_ENDPOINT: if (audio->recordAlt != 1) break; + audio->recordBatchPackets = max(UINT32_C(1), + min((uint32_t)request->pkts_per_urb, + (uint32_t)USB_AUDIO_RECORD_MAX_BATCH)); + audio->recordSafetyPackets = max(UINT32_C(1), + min((uint32_t)request->pkts_per_urb * request->no_urbs, + (uint32_t)USB_AUDIO_RECORD_MAX_SAFETY_PACKETS)); result = usb_redir_success; startRecord(audio); break; @@ -924,6 +1320,7 @@ static void stopISOStream(void * opaque, uint64_t id, struct usb_redir_stop_iso_stream_header * request) { LG_USBAudio * audio = getAudio(opaque); + flushPlayback(audio); uint8_t result = usb_redir_stall; switch (request->endpoint) { @@ -1021,6 +1418,7 @@ static void controlPacket(void * opaque, uint64_t id, uint8_t * data, int dataLength) { LG_USBAudio * audio = getAudio(opaque); + flushPlayback(audio); uint8_t buffer[sizeof(l_configurationDescriptor)]; const uint8_t * response = NULL; size_t responseSize = 0; @@ -1151,7 +1549,8 @@ static void controlPacket(void * opaque, uint64_t id, control == USB_AUDIO_CONTROL_FREQUENCY) { writeLE32(buffer, playbackClock ? - audio->playbackSampleRate : audio->recordSampleRate); + audio->playbackSampleRate : atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed)); response = buffer; responseSize = 4; status = usb_redir_success; @@ -1220,6 +1619,7 @@ static void isoPacket(void * opaque, uint64_t id, if (packet->endpoint != USB_AUDIO_PLAYBACK_DATA_ENDPOINT || packet->status != usb_redir_success || dataLength < 0 || packet->length != dataLength || + (dataLength && !data) || !layout || dataLength > layoutPacketSize(layout) || dataLength % frameSize != 0) { @@ -1228,13 +1628,14 @@ static void isoPacket(void * opaque, uint64_t id, } else { - if (dataLength && audio->events && audio->events->playbackData) - audio->events->playbackData(audio->eventOpaque, data, - dataLength / frameSize); + queuePlaybackPacket( + audio, data, dataLength / frameSize, frameSize); - if (audio->feedbackStreaming && + if (atomic_load_explicit( + &audio->feedbackStreaming, memory_order_acquire) && ++audio->feedbackDataPackets == USB_AUDIO_FEEDBACK_INTERVAL) { + flushPlayback(audio); audio->feedbackDataPackets = 0; sendFeedbackPackets(audio, 1); } @@ -1287,9 +1688,54 @@ static void unplugDevice(void * opaque) resetDevice(opaque); } +static void reportRecordDebug(LG_USBAudio * audio) +{ + if (!audio->debug) + return; + + const uint64_t now = recordTime(); + if (!audio->recordNextDebugTime) + { + audio->recordNextDebugTime = + now + USB_AUDIO_RECORD_DEBUG_INTERVAL_NS; + return; + } + + if (now < audio->recordNextDebugTime) + return; + audio->recordNextDebugTime = now + USB_AUDIO_RECORD_DEBUG_INTERVAL_NS; + + const uint_fast64_t overflow = atomic_exchange_explicit( + &audio->recordOverflowFrames, 0, memory_order_relaxed); + const uint_fast64_t underflow = atomic_exchange_explicit( + &audio->recordUnderflowFrames, 0, memory_order_relaxed); + const uint_fast64_t trimmed = atomic_exchange_explicit( + &audio->recordTrimmedFrames, 0, memory_order_relaxed); + const uint_fast64_t late = atomic_exchange_explicit( + &audio->recordLatePackets, 0, memory_order_relaxed); + const uint_fast64_t maxLateness = atomic_exchange_explicit( + &audio->recordMaxLateness, 0, memory_order_relaxed); + const uint_fast64_t discontinuities = atomic_exchange_explicit( + &audio->recordDiscontinuities, 0, memory_order_relaxed); + if (!overflow && !underflow && !trimmed && !late && !discontinuities) + return; + + DEBUG_INFO( + "USB capture: overflow %" PRIuFAST64 + ", underflow %" PRIuFAST64 + ", trimmed %" PRIuFAST64 + " frames; late %" PRIuFAST64 + " packets, max %.3f ms, discontinuities %" PRIuFAST64, + overflow, underflow, trimmed, late, maxLateness / 1000000.0, + discontinuities); +} + static void processDevice(void * opaque) { - sendRecordPackets(opaque); + LG_USBAudio * audio = opaque; + flushPlayback(audio); + sendRecordPackets(audio); + reportRecordDebug(audio); } static const LG_USBRedirDeviceOps l_deviceOps = @@ -1300,29 +1746,53 @@ static const LG_USBRedirDeviceOps l_deviceOps = .unplug = unplugDevice, }; -LG_USBAudio * lgUsbAudio_create( - const LG_USBAudioEventOps * events, void * eventOpaque) +LG_USBAudio * lgUsbAudio_create(const LG_USBAudioEventOps * events, + void * eventOpaque, bool debug) { LG_USBAudio * audio = calloc(1, sizeof(*audio)); if (!audio) return NULL; - audio->events = events; - audio->eventOpaque = eventOpaque; - audio->playbackSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; - audio->recordSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; - LG_LOCK_INIT(audio->recordLock); - audio->recordBuffer = ringbuffer_new( + audio->events = events; + audio->eventOpaque = eventOpaque; + audio->debug = debug; + audio->playbackSampleRate = LG_USB_AUDIO_DEFAULT_SAMPLE_RATE; + audio->recordBatchPackets = 1; + audio->recordSafetyPackets = 1; + audio->recordIdle = lgCreateEvent(true, 0); + if (!audio->recordIdle) + { + free(audio); + return NULL; + } + audio->recordBuffer = ringbuffer_new( USB_AUDIO_RECORD_QUEUE_FRAMES, USB_AUDIO_RECORD_FRAME_SIZE); if (!audio->recordBuffer) { + lgFreeEvent(audio->recordIdle); free(audio); return NULL; } atomic_init(&audio->feedbackValue, encodeFeedbackRate(audio->playbackSampleRate)); + atomic_init(&audio->feedbackStreaming, false); + atomic_init( + &audio->recordSampleRate, LG_USB_AUDIO_DEFAULT_SAMPLE_RATE); atomic_init(&audio->recordStreaming, false); + atomic_init(&audio->recordAccepting, false); + atomic_init(&audio->recordOverflowPending, false); + atomic_init(&audio->recordGeneration, 0); + atomic_init(&audio->recordInFlight, 0); + atomic_init(&audio->recordRateQ16, + (uint64_t)LG_USB_AUDIO_DEFAULT_SAMPLE_RATE * + USB_AUDIO_RECORD_RATE_SCALE); + atomic_init(&audio->recordOverflowFrames, 0); + atomic_init(&audio->recordUnderflowFrames, 0); + atomic_init(&audio->recordTrimmedFrames, 0); + atomic_init(&audio->recordLatePackets, 0); + atomic_init(&audio->recordMaxLateness, 0); + atomic_init(&audio->recordDiscontinuities, 0); return audio; } @@ -1335,89 +1805,101 @@ void lgUsbAudio_setFeedbackRate(LG_USBAudio * audio, double sampleRate) encodeFeedbackRate(sampleRate), memory_order_release); } -bool lgUsbAudio_recordData( - LG_USBAudio * audio, const void * data, size_t frames) +bool lgUsbAudio_feedbackActive(const LG_USBAudio * audio) { - if (!audio || frames > INT_MAX || (frames && !data) || - !atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire)) + return audio && atomic_load_explicit( + &audio->feedbackStreaming, memory_order_acquire); +} + +bool lgUsbAudio_recordData( + LG_USBAudio * audio, const void * data, size_t frames, + const LG_AudioClock * sourceClock) +{ + if (!audio || frames > INT_MAX || (frames && !data)) return false; - LG_LOCK(audio->recordLock); + const unsigned int generation = atomic_load_explicit( + &audio->recordGeneration, memory_order_seq_cst); if (!atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire)) + &audio->recordStreaming, memory_order_acquire) || + !atomic_load_explicit( + &audio->recordAccepting, memory_order_seq_cst)) + return false; + + atomic_fetch_add_explicit( + &audio->recordInFlight, 1, memory_order_seq_cst); + if (!atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire) || + !atomic_load_explicit( + &audio->recordAccepting, memory_order_seq_cst) || + atomic_load_explicit( + &audio->recordGeneration, memory_order_seq_cst) != generation) { - LG_UNLOCK(audio->recordLock); + const unsigned int previous = atomic_fetch_sub_explicit( + &audio->recordInFlight, 1, memory_order_seq_cst); + if (previous == 1 && !atomic_load_explicit( + &audio->recordAccepting, memory_order_seq_cst)) + lgSignalEvent(audio->recordIdle); return false; } const size_t receivedFrames = frames; - const uint64_t now = recordTime(); - /* Average complete capture batches over time so callback jitter does not - * become USB packet jitter. recordRateQ16 is frames/second in Q16. */ - const bool resetRate = - !audio->recordRateLastTime || - now - audio->recordRateLastTime > USB_AUDIO_RECORD_RATE_GAP_NS; - if (resetRate) - { - audio->recordRateQ16 = - (uint64_t)audio->recordSampleRate * USB_AUDIO_RECORD_RATE_SCALE; - audio->recordRateStartTime = now; - audio->recordRateFrames = 0; - } + const uint32_t sampleRate = atomic_load_explicit( + &audio->recordSampleRate, memory_order_relaxed); + bool sourceContinuous = true; + if (sourceClock) + sourceContinuous = updateRecordSourceRate( + audio, sampleRate, sourceClock); else - audio->recordRateFrames += receivedFrames; - audio->recordRateLastTime = now; + updateRecordArrivalRate(audio, sampleRate, receivedFrames); - const uint64_t elapsed = now - audio->recordRateStartTime; - if (elapsed >= USB_AUDIO_RECORD_RATE_SETTLE_NS) + size_t overflowFrames = 0; + if (sourceContinuous) { - const double measured = - (double)audio->recordRateFrames * 1000000000.0 / elapsed; - if (measured >= audio->recordSampleRate * 0.995 && - measured <= audio->recordSampleRate * 1.005) + const int length = ringbuffer_getLength(audio->recordBuffer); + const uint8_t * input = data; + if (frames > (size_t)length) { - const uint64_t measuredQ16 = (uint64_t)( - measured * USB_AUDIO_RECORD_RATE_SCALE + 0.5); - audio->recordRateQ16 = - (audio->recordRateQ16 * 3 + measuredQ16) / 4; + input += (frames - length) * USB_AUDIO_RECORD_FRAME_SIZE; + frames = length; } - audio->recordRateStartTime = now; - audio->recordRateFrames = 0; + const int appended = ringbuffer_append( + audio->recordBuffer, input, (int)frames); + overflowFrames = receivedFrames - appended; } - const int length = ringbuffer_getLength(audio->recordBuffer); - const uint8_t * input = data; - bool dropped = false; - if (frames > (size_t)length) + if (!sourceContinuous || overflowFrames) { - input += (frames - length) * USB_AUDIO_RECORD_FRAME_SIZE; - frames = length; - ringbuffer_reset(audio->recordBuffer); - dropped = true; - } - else - { - const int overflow = - ringbuffer_getCount(audio->recordBuffer) + (int)frames - length; - if (overflow > 0) - { - ringbuffer_consume(audio->recordBuffer, NULL, overflow); - dropped = true; - } + atomic_store_explicit( + &audio->recordOverflowPending, true, memory_order_release); + if (audio->debug) + atomic_fetch_add_explicit(&audio->recordOverflowFrames, + overflowFrames, memory_order_relaxed); } - const int appended = ringbuffer_append( - audio->recordBuffer, input, (int)frames); - LG_UNLOCK(audio->recordLock); - return !dropped && appended == (int)frames; + const unsigned int previous = atomic_fetch_sub_explicit( + &audio->recordInFlight, 1, memory_order_seq_cst); + if (previous == 1 && !atomic_load_explicit( + &audio->recordAccepting, memory_order_seq_cst)) + lgSignalEvent(audio->recordIdle); + return !sourceContinuous || !overflowFrames; } -bool lgUsbAudio_recording(const LG_USBAudio * audio) +uint64_t lgUsbAudio_processDelayNs(const LG_USBAudio * audio) { - return audio && atomic_load_explicit( - &audio->recordStreaming, memory_order_acquire); + if (!audio || !atomic_load_explicit( + &audio->recordStreaming, memory_order_acquire)) + return UINT64_MAX; + + if (audio->recordRefillPackets) + return 0; + + const uint64_t now = recordTime(); + const uint64_t deadline = audio->recordNextPacketTime + + (uint64_t)(audio->recordBatchPackets - 1) * + USB_AUDIO_RECORD_PACKET_INTERVAL_NS; + return deadline > now ? deadline - now : 0; } void lgUsbAudio_destroy(LG_USBAudio * audio) @@ -1425,8 +1907,10 @@ void lgUsbAudio_destroy(LG_USBAudio * audio) if (!audio) return; + recordDisableInput(audio); + recordWaitForInput(audio); ringbuffer_free(&audio->recordBuffer); - LG_LOCK_FREE(audio->recordLock); + lgFreeEvent(audio->recordIdle); free(audio); } diff --git a/client/src/usb_audio.h b/client/src/usb_audio.h index 71d435b7..dc852bfd 100644 --- a/client/src/usb_audio.h +++ b/client/src/usb_audio.h @@ -21,6 +21,7 @@ #ifndef _H_LG_CLIENT_USB_AUDIO_ #define _H_LG_CLIENT_USB_AUDIO_ +#include "interface/audio.h" #include "usbredir.h" #include @@ -49,21 +50,25 @@ typedef struct LG_USBAudioEventOps LG_USBAudioEventOps; LG_USBAudio * lgUsbAudio_create( - const LG_USBAudioEventOps * events, void * eventOpaque); + const LG_USBAudioEventOps * events, void * eventOpaque, bool debug); /* Destroy the LG_USBRedir using this device before destroying the device. */ void lgUsbAudio_destroy(LG_USBAudio * audio); /* Publish the requested source rate without touching usbredir from the audio * feedback thread. */ void lgUsbAudio_setFeedbackRate(LG_USBAudio * audio, double sampleRate); +bool lgUsbAudio_feedbackActive(const LG_USBAudio * audio); -/* Queue interleaved packed signed 24-bit microphone frames. This may be - * called from the audio recording thread. */ +/* Queue interleaved packed signed 24-bit microphone frames. sourceClock is + * borrowed for the call and identifies the first frame when present. This may + * be called from the audio recording thread. */ bool lgUsbAudio_recordData( - LG_USBAudio * audio, const void * data, size_t frames); + LG_USBAudio * audio, const void * data, size_t frames, + const LG_AudioClock * sourceClock); -/* This must be queried on the PureSpice processing thread. */ -bool lgUsbAudio_recording(const LG_USBAudio * audio); +/* Return the time until ISO-IN processing is needed. This must be queried on + * the PureSpice processing thread. */ +uint64_t lgUsbAudio_processDelayNs(const LG_USBAudio * audio); const LG_USBRedirDeviceOps * lgUsbAudio_deviceOps(void); diff --git a/client/src/usbredir.c b/client/src/usbredir.c index 7925a4a3..9734f6dc 100644 --- a/client/src/usbredir.c +++ b/client/src/usbredir.c @@ -21,6 +21,7 @@ #include "usbredir.h" #include "common/debug.h" +#include "common/time.h" #include @@ -29,6 +30,9 @@ #include #define USB_REDIR_CHANNEL_COUNT 256 +#define USB_REDIR_DISCONNECT_TIMEOUT_NS INT64_C(500000000) +#define USB_REDIR_RECONNECT_DELAY_NS INT64_C(250000000) +#define USB_REDIR_MAX_OUTPUT_BYTES UINT64_C(1048576) struct LG_USBRedir { @@ -49,6 +53,8 @@ struct LG_USBRedir atomic_bool available; bool plugged; bool disconnectPending; + int64_t disconnectDeadline; + int64_t reconnectDeadline; }; static void setAvailable(LG_USBRedir * usbredir, bool available) @@ -123,6 +129,7 @@ static void deviceDisconnectAck(void * opaque) { LG_USBRedir * usbredir = opaque; usbredir->disconnectPending = false; + usbredir->disconnectDeadline = 0; } static void unplugDevice(LG_USBRedir * usbredir) @@ -139,6 +146,7 @@ static void destroyParser(LG_USBRedir * usbredir) setAvailable(usbredir, false); unplugDevice(usbredir); usbredir->disconnectPending = false; + usbredir->disconnectDeadline = 0; if (!usbredir->parser) return; @@ -149,8 +157,20 @@ static void destroyParser(LG_USBRedir * usbredir) static bool flushUSBRedir(LG_USBRedir * usbredir) { - return !usbredir->parser || - usbredirparser_do_write(usbredir->parser) == 0; + if (!usbredir->parser) + return true; + + if (usbredirparser_do_write(usbredir->parser) != 0) + return false; + + const uint64_t buffered = + usbredirparser_get_bufferered_output_size(usbredir->parser); + if (buffered <= USB_REDIR_MAX_OUTPUT_BYTES) + return true; + + DEBUG_ERROR("USB redirection output queue exceeded %u bytes", + (unsigned int)USB_REDIR_MAX_OUTPUT_BYTES); + return false; } static bool connectChannel(LG_USBRedir * usbredir, @@ -166,10 +186,10 @@ static bool connectChannel(LG_USBRedir * usbredir, return false; } -static void selectChannel(LG_USBRedir * usbredir) +static bool selectChannel(LG_USBRedir * usbredir) { if (usbredir->channel) - return; + return true; for (unsigned int i = 0; i < USB_REDIR_CHANNEL_COUNT; ++i) { @@ -178,8 +198,28 @@ static void selectChannel(LG_USBRedir * usbredir) continue; if (connectChannel(usbredir, channel)) - return; + { + usbredir->reconnectDeadline = 0; + return true; + } } + + usbredir->reconnectDeadline = + (int64_t)nanotime() + USB_REDIR_RECONNECT_DELAY_NS; + return false; +} + +static void resetChannel(LG_USBRedir * usbredir) +{ + PSUSBRedirChannel * channel = usbredir->channel; + usbredir->channel = NULL; + destroyParser(usbredir); + + if (channel && purespice_usbRedirConnected(channel)) + purespice_usbRedirDisconnect(channel); + + usbredir->reconnectDeadline = + (int64_t)nanotime() + USB_REDIR_RECONNECT_DELAY_NS; } static bool createParser(LG_USBRedir * usbredir) @@ -268,14 +308,37 @@ bool lgUsbRedir_disconnectPending(const LG_USBRedir * usbredir) return usbredir->disconnectPending; } -bool lgUsbRedir_process(LG_USBRedir * usbredir) +static bool processUSBRedir(LG_USBRedir * usbredir, bool recover) { + const int64_t now = (int64_t)nanotime(); + if (!usbredir->channel && + now >= usbredir->reconnectDeadline) + selectChannel(usbredir); + if (!usbredir->parser || !atomic_load_explicit(&usbredir->available, memory_order_acquire)) - return flushUSBRedir(usbredir); + { + const bool result = flushUSBRedir(usbredir); + if (!result && recover) + resetChannel(usbredir); + return result; + } if (usbredir->disconnectPending) - return flushUSBRedir(usbredir); + { + if (usbredir->disconnectDeadline && + now >= usbredir->disconnectDeadline) + { + DEBUG_WARN("USB redirection disconnect acknowledgement timed out"); + resetChannel(usbredir); + return true; + } + + const bool result = flushUSBRedir(usbredir); + if (!result && recover) + resetChannel(usbredir); + return result; + } const bool desired = atomic_load_explicit(&usbredir->desiredPlugged, memory_order_acquire); @@ -292,6 +355,8 @@ bool lgUsbRedir_process(LG_USBRedir * usbredir) unplugDevice(usbredir); usbredir->disconnectPending = usbredirparser_peer_has_cap( usbredir->parser, usb_redir_cap_device_disconnect_ack); + usbredir->disconnectDeadline = usbredir->disconnectPending ? + now + USB_REDIR_DISCONNECT_TIMEOUT_NS : 0; usbredirparser_send_device_disconnect(usbredir->parser); } } @@ -299,7 +364,15 @@ bool lgUsbRedir_process(LG_USBRedir * usbredir) if (usbredir->plugged && usbredir->deviceOps->process) usbredir->deviceOps->process(usbredir->deviceOpaque); - return flushUSBRedir(usbredir); + const bool result = flushUSBRedir(usbredir); + if (!result && recover) + resetChannel(usbredir); + return result; +} + +bool lgUsbRedir_process(LG_USBRedir * usbredir) +{ + return processUSBRedir(usbredir, true); } void lgUsbRedir_state(PSUSBRedirChannel * channel, @@ -328,7 +401,13 @@ void lgUsbRedir_state(PSUSBRedirChannel * channel, case PS_USB_REDIR_DISCONNECTED: if (channel == usbredir->channel) + { destroyParser(usbredir); + usbredir->channel = NULL; + if (purespice_usbRedirAvailable(channel)) + usbredir->reconnectDeadline = + (int64_t)nanotime() + USB_REDIR_RECONNECT_DELAY_NS; + } break; case PS_USB_REDIR_UNAVAILABLE: @@ -375,7 +454,9 @@ bool lgUsbRedir_data(PSUSBRedirChannel * channel, return false; } - return lgUsbRedir_process(usbredir); + /* Returning false lets PureSpice disconnect after this callback unwinds; + * resetting it here would destroy the channel during its own dispatch. */ + return processUSBRedir(usbredir, false); } void * lgUsbRedir_device(void * parserOpaque)