From 0da4bfa8dd72ad0e1820608aac55e6e677f287e3 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Sun, 9 Aug 2026 16:37:39 +1000 Subject: [PATCH] [client] audio: add transport provider interface Route audio through a provider interface. Connected transports take priority while SPICE remains the fallback. Carry explicit sample formats, rates, channel layouts, and source clocks through playback and recording. Add optional presentation clock feedback for active synchronization. --- client/CMakeLists.txt | 5 +- client/audiodevs/PipeWire/pipewire.c | 158 ++- client/audiodevs/PulseAudio/pulseaudio.c | 89 +- client/include/interface/audio.h | 166 +++ client/include/interface/audiodev.h | 14 +- client/include/interface/transport.h | 5 + client/src/audio.c | 1485 ++++++++++++++++------ client/src/audio.h | 45 +- client/src/audio_spice.c | 842 ++++++++++++ client/src/audio_spice.h | 45 + client/src/keybind.c | 4 +- client/src/main.c | 68 +- 12 files changed, 2441 insertions(+), 485 deletions(-) create mode 100644 client/include/interface/audio.h create mode 100644 client/src/audio_spice.c create mode 100644 client/src/audio_spice.h diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index f4b8960f..8749d81f 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -188,7 +188,10 @@ if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") endif() if(ENABLE_AUDIO) - list(APPEND SOURCES src/audio.c) + list(APPEND SOURCES + src/audio.c + src/audio_spice.c + ) endif() add_subdirectory("${PROJECT_TOP}/resources" "${CMAKE_BINARY_DIR}/resources") diff --git a/client/audiodevs/PipeWire/pipewire.c b/client/audiodevs/PipeWire/pipewire.c index 8b4404f9..92791086 100644 --- a/client/audiodevs/PipeWire/pipewire.c +++ b/client/audiodevs/PipeWire/pipewire.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "common/debug.h" #include "common/stringutils.h" @@ -67,8 +68,7 @@ struct PipeWire enum pw_stream_state connectionState; bool resamplerEnabled; - int channels; - int sampleRate; + LG_AudioFormat format; int stride; LG_AudioPullFn pullFn; int maxPeriodFrames; @@ -82,8 +82,7 @@ struct PipeWire { struct pw_stream * stream; - int channels; - int sampleRate; + LG_AudioFormat format; int stride; LG_AudioPushFn pushFn; @@ -105,6 +104,94 @@ struct PipeWire static struct PipeWire pw = {0}; +static bool pipewire_audioFormatEqual(const LG_AudioFormat * a, + const LG_AudioFormat * b) +{ + return + a->sampleFormat == b->sampleFormat && + a->sampleRate == b->sampleRate && + a->channelCount == b->channelCount && + memcmp(a->channels, b->channels, + a->channelCount * sizeof(*a->channels)) == 0; +} + +static enum spa_audio_channel pipewire_channel( + LG_AudioChannel channel, uint8_t index) +{ + switch (channel) + { + case LG_AUDIO_CH_UNKNOWN: + return (enum spa_audio_channel)(SPA_AUDIO_CHANNEL_AUX0 + index); + case LG_AUDIO_CH_MONO : return SPA_AUDIO_CHANNEL_MONO; + case LG_AUDIO_CH_FRONT_LEFT : return SPA_AUDIO_CHANNEL_FL; + case LG_AUDIO_CH_FRONT_RIGHT : return SPA_AUDIO_CHANNEL_FR; + case LG_AUDIO_CH_FRONT_CENTER : return SPA_AUDIO_CHANNEL_FC; + case LG_AUDIO_CH_LFE : return SPA_AUDIO_CHANNEL_LFE; + case LG_AUDIO_CH_REAR_LEFT : return SPA_AUDIO_CHANNEL_RL; + case LG_AUDIO_CH_REAR_RIGHT : return SPA_AUDIO_CHANNEL_RR; + case LG_AUDIO_CH_FRONT_LEFT_CENTER : return SPA_AUDIO_CHANNEL_FLC; + case LG_AUDIO_CH_FRONT_RIGHT_CENTER : return SPA_AUDIO_CHANNEL_FRC; + case LG_AUDIO_CH_REAR_CENTER : return SPA_AUDIO_CHANNEL_RC; + case LG_AUDIO_CH_SIDE_LEFT : return SPA_AUDIO_CHANNEL_SL; + case LG_AUDIO_CH_SIDE_RIGHT : return SPA_AUDIO_CHANNEL_SR; + case LG_AUDIO_CH_TOP_CENTER : return SPA_AUDIO_CHANNEL_TC; + case LG_AUDIO_CH_TOP_FRONT_LEFT : return SPA_AUDIO_CHANNEL_TFL; + case LG_AUDIO_CH_TOP_FRONT_CENTER : return SPA_AUDIO_CHANNEL_TFC; + case LG_AUDIO_CH_TOP_FRONT_RIGHT : return SPA_AUDIO_CHANNEL_TFR; + case LG_AUDIO_CH_TOP_REAR_LEFT : return SPA_AUDIO_CHANNEL_TRL; + case LG_AUDIO_CH_TOP_REAR_CENTER : return SPA_AUDIO_CHANNEL_TRC; + case LG_AUDIO_CH_TOP_REAR_RIGHT : return SPA_AUDIO_CHANNEL_TRR; + } + + return (enum spa_audio_channel)(SPA_AUDIO_CHANNEL_AUX0 + index); +} + +static enum spa_audio_format pipewire_sampleFormat( + LG_AudioSampleFormat format) +{ + switch (format) + { + case LG_AUDIO_FMT_U8 : return SPA_AUDIO_FORMAT_U8; + case LG_AUDIO_FMT_S16_LE : return SPA_AUDIO_FORMAT_S16_LE; + case LG_AUDIO_FMT_S24_LE : return SPA_AUDIO_FORMAT_S24_LE; + case LG_AUDIO_FMT_S32_LE : return SPA_AUDIO_FORMAT_S32_LE; + case LG_AUDIO_FMT_F32_LE : return SPA_AUDIO_FORMAT_F32_LE; + case LG_AUDIO_FMT_F64_LE : return SPA_AUDIO_FORMAT_F64_LE; + } + + return SPA_AUDIO_FORMAT_UNKNOWN; +} + +static int pipewire_sampleSize(LG_AudioSampleFormat format) +{ + switch (format) + { + case LG_AUDIO_FMT_U8 : return 1; + case LG_AUDIO_FMT_S16_LE : return 2; + case LG_AUDIO_FMT_S24_LE : return 3; + case LG_AUDIO_FMT_S32_LE : + case LG_AUDIO_FMT_F32_LE : return 4; + case LG_AUDIO_FMT_F64_LE : return 8; + } + + return 0; +} + +static struct spa_audio_info_raw pipewire_audioInfo( + const LG_AudioFormat * format, enum spa_audio_format sampleFormat) +{ + struct spa_audio_info_raw info = SPA_AUDIO_INFO_RAW_INIT( + .format = sampleFormat, + .channels = format->channelCount, + .rate = format->sampleRate + ); + + for (uint8_t i = 0; i < format->channelCount; ++i) + info.position[i] = pipewire_channel(format->channels[i], i); + + return info; +} + static void pipewire_reportPlaybackErrors(void) { const unsigned int bufferErrors = atomic_exchange_explicit( @@ -411,13 +498,16 @@ static void pipewire_playbackStopStream(void) pipewire_reportPlaybackErrors(); } -static bool pipewire_playbackSetup(int channels, int sampleRate, +static bool pipewire_playbackSetup(const LG_AudioFormat * format, int requestedPeriodFrames, bool requestResampler, bool * resamplerEnabled, int * maxPeriodFrames, int * startFrames, LG_AudioPullFn pullFn) { *resamplerEnabled = false; + const int channels = format->channelCount; + const int sampleRate = format->sampleRate; + const struct spa_pod * params[1]; uint8_t buffer[1024]; struct spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer)); @@ -431,8 +521,7 @@ static bool pipewire_playbackSetup(int channels, int sampleRate, }; if (pw.playback.stream && - pw.playback.channels == channels && - pw.playback.sampleRate == sampleRate) + pipewire_audioFormatEqual(&pw.playback.format, format)) { #if PW_CHECK_VERSION(1, 4, 0) atomic_store_explicit( @@ -451,10 +540,9 @@ static bool pipewire_playbackSetup(int channels, int sampleRate, snprintf(requestedNodeLatency, sizeof(requestedNodeLatency), "%d/%d", requestedPeriodFrames, sampleRate); - pw.playback.channels = channels; - pw.playback.sampleRate = sampleRate; - pw.playback.stride = sizeof(float) * channels; - pw.playback.pullFn = pullFn; + pw.playback.format = *format; + pw.playback.stride = sizeof(float) * channels; + pw.playback.pullFn = pullFn; pw_thread_loop_lock(pw.thread); @@ -538,12 +626,10 @@ static bool pipewire_playbackSetup(int channels, int sampleRate, *maxPeriodFrames = pw.playback.maxPeriodFrames; *startFrames = pw.playback.startFrames; - params[0] = spa_format_audio_raw_build(&b, SPA_PARAM_EnumFormat, - &SPA_AUDIO_INFO_RAW_INIT( - .format = SPA_AUDIO_FORMAT_F32, - .channels = channels, - .rate = sampleRate - )); + struct spa_audio_info_raw info = + pipewire_audioInfo(format, SPA_AUDIO_FORMAT_F32); + params[0] = spa_format_audio_raw_build( + &b, SPA_PARAM_EnumFormat, &info); pw.playback.connectionState = PW_STREAM_STATE_CONNECTING; const int result = pw_stream_connect( @@ -654,7 +740,7 @@ done: static void pipewire_playbackVolume(int channels, const uint16_t volume[]) { - if (channels != pw.playback.channels) + if (channels != pw.playback.format.channelCount) return; float param[channels]; @@ -864,9 +950,12 @@ static void pipewire_onRecordProcess(void * userdata) pw_stream_queue_buffer(pw.record.stream, pbuf); } -static void pipewire_recordStart(int channels, int sampleRate, +static void pipewire_recordStart(const LG_AudioFormat * format, LG_AudioPushFn pushFn) { + const int channels = format->channelCount; + const int sampleRate = format->sampleRate; + const int sampleSize = pipewire_sampleSize(format->sampleFormat); const struct spa_pod * params[1]; uint8_t buffer[1024]; struct spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer)); @@ -877,8 +966,7 @@ static void pipewire_recordStart(int channels, int sampleRate, }; if (pw.record.stream && - pw.record.channels == channels && - pw.record.sampleRate == sampleRate) + pipewire_audioFormatEqual(&pw.record.format, format)) { if (!pw.record.active) { @@ -892,10 +980,9 @@ static void pipewire_recordStart(int channels, int sampleRate, pipewire_recordStopStream(); - pw.record.channels = channels; - pw.record.sampleRate = sampleRate; - pw.record.stride = sizeof(uint16_t) * channels; - pw.record.pushFn = pushFn; + pw.record.format = *format; + pw.record.stride = sampleSize * channels; + pw.record.pushFn = pushFn; if (!pipewire_recordStartSender(sampleRate)) return; @@ -941,12 +1028,11 @@ static void pipewire_recordStart(int channels, int sampleRate, return; } - params[0] = spa_format_audio_raw_build(&b, SPA_PARAM_EnumFormat, - &SPA_AUDIO_INFO_RAW_INIT( - .format = SPA_AUDIO_FORMAT_S16, - .channels = channels, - .rate = sampleRate - )); + struct spa_audio_info_raw info = + pipewire_audioInfo(format, + pipewire_sampleFormat(format->sampleFormat)); + params[0] = spa_format_audio_raw_build( + &b, SPA_PARAM_EnumFormat, &info); const int result = pw_stream_connect( pw.record.stream, @@ -974,18 +1060,12 @@ static void pipewire_recordStart(int channels, int sampleRate, static void pipewire_recordStop(void) { - if (!pw.record.active) - return; - - pw_thread_loop_lock(pw.thread); - pw_stream_set_active(pw.record.stream, false); - pw.record.active = false; - pw_thread_loop_unlock(pw.thread); + pipewire_recordStopStream(); } static void pipewire_recordVolume(int channels, const uint16_t volume[]) { - if (channels != pw.record.channels) + if (channels != pw.record.format.channelCount) return; float param[channels]; diff --git a/client/audiodevs/PulseAudio/pulseaudio.c b/client/audiodevs/PulseAudio/pulseaudio.c index 01da6648..59c1c3d6 100644 --- a/client/audiodevs/PulseAudio/pulseaudio.c +++ b/client/audiodevs/PulseAudio/pulseaudio.c @@ -42,8 +42,7 @@ struct PulseAudio bool sinkStarting; int sinkMaxPeriodFrames; int sinkStartFrames; - int sinkSampleRate; - int sinkChannels; + LG_AudioFormat sinkFormat; int sinkStride; LG_AudioPullFn sinkPullFn; _Atomic(int64_t) sinkPresentationDeadline; @@ -51,6 +50,68 @@ struct PulseAudio static struct PulseAudio pa = {0}; +static bool pulseaudio_audioFormatEqual(const LG_AudioFormat * a, + const LG_AudioFormat * b) +{ + return + a->sampleFormat == b->sampleFormat && + a->sampleRate == b->sampleRate && + a->channelCount == b->channelCount && + memcmp(a->channels, b->channels, + a->channelCount * sizeof(*a->channels)) == 0; +} + +static pa_channel_position_t pulseaudio_channel( + LG_AudioChannel channel, uint8_t index) +{ + switch (channel) + { + case LG_AUDIO_CH_UNKNOWN: + return (pa_channel_position_t)(PA_CHANNEL_POSITION_AUX0 + index); + + case LG_AUDIO_CH_MONO : + return PA_CHANNEL_POSITION_MONO; + case LG_AUDIO_CH_FRONT_LEFT : + return PA_CHANNEL_POSITION_FRONT_LEFT; + case LG_AUDIO_CH_FRONT_RIGHT : + return PA_CHANNEL_POSITION_FRONT_RIGHT; + case LG_AUDIO_CH_FRONT_CENTER : + return PA_CHANNEL_POSITION_FRONT_CENTER; + case LG_AUDIO_CH_LFE : + return PA_CHANNEL_POSITION_LFE; + case LG_AUDIO_CH_REAR_LEFT : + return PA_CHANNEL_POSITION_REAR_LEFT; + case LG_AUDIO_CH_REAR_RIGHT : + return PA_CHANNEL_POSITION_REAR_RIGHT; + case LG_AUDIO_CH_FRONT_LEFT_CENTER : + return PA_CHANNEL_POSITION_FRONT_LEFT_OF_CENTER; + case LG_AUDIO_CH_FRONT_RIGHT_CENTER : + return PA_CHANNEL_POSITION_FRONT_RIGHT_OF_CENTER; + case LG_AUDIO_CH_REAR_CENTER : + return PA_CHANNEL_POSITION_REAR_CENTER; + case LG_AUDIO_CH_SIDE_LEFT : + return PA_CHANNEL_POSITION_SIDE_LEFT; + case LG_AUDIO_CH_SIDE_RIGHT : + return PA_CHANNEL_POSITION_SIDE_RIGHT; + case LG_AUDIO_CH_TOP_CENTER : + return PA_CHANNEL_POSITION_TOP_CENTER; + case LG_AUDIO_CH_TOP_FRONT_LEFT : + return PA_CHANNEL_POSITION_TOP_FRONT_LEFT; + case LG_AUDIO_CH_TOP_FRONT_CENTER : + return PA_CHANNEL_POSITION_TOP_FRONT_CENTER; + case LG_AUDIO_CH_TOP_FRONT_RIGHT : + return PA_CHANNEL_POSITION_TOP_FRONT_RIGHT; + case LG_AUDIO_CH_TOP_REAR_LEFT : + return PA_CHANNEL_POSITION_TOP_REAR_LEFT; + case LG_AUDIO_CH_TOP_REAR_CENTER : + return PA_CHANNEL_POSITION_TOP_REAR_CENTER; + case LG_AUDIO_CH_TOP_REAR_RIGHT : + return PA_CHANNEL_POSITION_TOP_REAR_RIGHT; + } + + return (pa_channel_position_t)(PA_CHANNEL_POSITION_AUX0 + index); +} + static void pulseaudio_unrefOperation(pa_operation * operation) { if (operation) @@ -299,14 +360,17 @@ static void pulseaudio_overflow_cb(pa_stream * p, void * userdata) DEBUG_WARN("Overflow"); } -static bool pulseaudio_setup(int channels, int sampleRate, +static bool pulseaudio_setup(const LG_AudioFormat * format, int requestedPeriodFrames, bool requestResampler, bool * resamplerEnabled, int * maxPeriodFrames, int * startFrames, LG_AudioPullFn pullFn) { *resamplerEnabled = false; - if (pa.sink && pa.sinkChannels == channels && pa.sinkSampleRate == sampleRate) + const int channels = format->channelCount; + const int sampleRate = format->sampleRate; + + if (pa.sink && pulseaudio_audioFormatEqual(&pa.sinkFormat, format)) { *maxPeriodFrames = pa.sinkMaxPeriodFrames; *startFrames = pa.sinkStartFrames; @@ -318,6 +382,9 @@ static bool pulseaudio_setup(int channels, int sampleRate, .rate = sampleRate, .channels = channels }; + pa_channel_map channelMap = { .channels = channels }; + for (uint8_t i = 0; i < format->channelCount; ++i) + channelMap.map[i] = pulseaudio_channel(format->channels[i], i); int stride = channels * sizeof(float); int bufferSize = requestedPeriodFrames * 2 * stride; @@ -332,14 +399,14 @@ static bool pulseaudio_setup(int channels, int sampleRate, pa_threaded_mainloop_lock(pa.loop); pulseaudio_sink_close_nl(); - pa.sinkChannels = channels; - pa.sinkSampleRate = sampleRate; - pa.sinkStride = stride; - pa.sinkPullFn = pullFn; - pa.sinkCorked = true; - pa.sinkStarting = false; + pa.sinkFormat = *format; + pa.sinkStride = stride; + pa.sinkPullFn = pullFn; + pa.sinkCorked = true; + pa.sinkStarting = false; - pa.sink = pa_stream_new(pa.context, "Looking Glass", &spec, NULL); + pa.sink = pa_stream_new( + pa.context, "Looking Glass", &spec, &channelMap); if (!pa.sink) { DEBUG_ERROR("Failed to create PulseAudio stream: %s", diff --git a/client/include/interface/audio.h b/client/include/interface/audio.h new file mode 100644 index 00000000..189de240 --- /dev/null +++ b/client/include/interface/audio.h @@ -0,0 +1,166 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef _H_LG_CLIENT_AUDIO_INTERFACE_ +#define _H_LG_CLIENT_AUDIO_INTERFACE_ + +#include +#include +#include + +#define LG_AUDIO_MAX_CHANNELS 32 + +typedef enum LG_AudioSampleFormat +{ + LG_AUDIO_FMT_U8, + LG_AUDIO_FMT_S16_LE, + /* Signed 24-bit little-endian samples packed into three bytes. */ + LG_AUDIO_FMT_S24_LE, + LG_AUDIO_FMT_S32_LE, + /* IEEE 754 little-endian floating-point samples. */ + LG_AUDIO_FMT_F32_LE, + LG_AUDIO_FMT_F64_LE, +} +LG_AudioSampleFormat; + +typedef enum LG_AudioChannel +{ + LG_AUDIO_CH_UNKNOWN, + LG_AUDIO_CH_MONO, + LG_AUDIO_CH_FRONT_LEFT, + LG_AUDIO_CH_FRONT_RIGHT, + LG_AUDIO_CH_FRONT_CENTER, + LG_AUDIO_CH_LFE, + LG_AUDIO_CH_REAR_LEFT, + LG_AUDIO_CH_REAR_RIGHT, + LG_AUDIO_CH_FRONT_LEFT_CENTER, + LG_AUDIO_CH_FRONT_RIGHT_CENTER, + LG_AUDIO_CH_REAR_CENTER, + LG_AUDIO_CH_SIDE_LEFT, + LG_AUDIO_CH_SIDE_RIGHT, + LG_AUDIO_CH_TOP_CENTER, + LG_AUDIO_CH_TOP_FRONT_LEFT, + LG_AUDIO_CH_TOP_FRONT_CENTER, + LG_AUDIO_CH_TOP_FRONT_RIGHT, + LG_AUDIO_CH_TOP_REAR_LEFT, + LG_AUDIO_CH_TOP_REAR_CENTER, + LG_AUDIO_CH_TOP_REAR_RIGHT, +} +LG_AudioChannel; + +typedef struct LG_AudioFormat +{ + LG_AudioSampleFormat sampleFormat; + uint32_t sampleRate; + uint8_t channelCount; + + /* Samples are interleaved in this order. Unknown positions are explicit. */ + LG_AudioChannel channels[LG_AUDIO_MAX_CHANNELS]; +} +LG_AudioFormat; + +typedef struct LG_AudioClock +{ + /* Frame position and time describe the same point on the stream timeline. + * Time is monotonic nanoseconds in the publisher's clock domain; consumers + * must use differences unless they know that they share that domain. */ + uint64_t position; + int64_t time; + double rate; /* measured frames per second, or zero when unavailable */ + bool stable; + bool discontinuity; +} +LG_AudioClock; + +typedef struct LG_AudioStatus +{ + bool available; + /* Changes whenever the provider endpoint is replaced or restarted. */ + uint32_t generation; +} +LG_AudioStatus; + +typedef void (*LG_AudioStatusFn)(void * opaque, + const LG_AudioStatus * status); + +typedef struct LG_AudioEventOps +{ + /* Format and clock pointers are borrowed for the duration of each call. + * A NULL source clock indicates that the provider has no usable clock. + * Providers must serialize event delivery for an attachment. Stream + * generations are nonzero and uniquely identify each stream instance. */ + void (*playbackStart)(void * opaque, uint32_t generation, + const LG_AudioFormat * format, const LG_AudioClock * sourceClock); + void (*playbackStop)(void * opaque, uint32_t generation); + void (*playbackVolume)(void * opaque, uint32_t generation, + uint8_t channels, const uint16_t volume[]); + void (*playbackMute)(void * opaque, uint32_t generation, bool mute); + /* Sample data is borrowed for the duration of the call. When supplied, the + * source clock position and time identify the first frame in this packet. */ + void (*playbackData)(void * opaque, uint32_t generation, + const void * data, size_t frames, + const LG_AudioClock * sourceClock); + + void (*recordStart)(void * opaque, uint32_t generation, + const LG_AudioFormat * format); + void (*recordStop)(void * opaque, uint32_t generation); + void (*recordVolume)(void * opaque, uint32_t generation, + uint8_t channels, const uint16_t volume[]); + void (*recordMute)(void * opaque, uint32_t generation, bool mute); +} +LG_AudioEventOps; + +typedef struct LG_AudioOps +{ + const char * name; + + /* Registration must synchronously report the current status after releasing + * any backend locks. Passing NULL unregisters the listener. Providers with + * no status listener are assumed to remain available while their transport + * session is connected. Status callbacks must not be invoked synchronously + * from event delivery or from another operation in this interface. Passing + * NULL must synchronously quiesce any in-flight status callback. */ + void (*setStatusListener)(void * opaque, LG_AudioStatusFn callback, + void * callbackOpaque); + + /* Attach begins event delivery and replays any active streams. Detach must + * synchronously quiesce event callbacks before returning. The owner must + * stop event delivery before dropping an endpoint that can no longer be + * detached. */ + bool (*attach)(void * opaque, const LG_AudioEventOps * events, + void * eventOpaque); + void (*detach)(void * opaque); + + /* Send microphone frames to the guest. Data is borrowed for the duration of + * the call and uses the format supplied with the matching recordStart. The + * source clock, when present, identifies the first frame in the buffer. */ + bool (*recordData)(void * opaque, uint32_t generation, + const void * data, size_t frames, const LG_AudioClock * sourceClock); + + /* Optional active synchronization feedback. This is called outside the + * realtime audio callback with the measured playback device clock. Its + * position uses the device's independent output-frame timeline and its time + * includes the backend's estimated presentation latency. */ + bool (*clockFeedback)(void * opaque, uint32_t generation, + const LG_AudioClock * playbackClock); +} +LG_AudioOps; + +#endif diff --git a/client/include/interface/audiodev.h b/client/include/interface/audiodev.h index 14d38bf8..1d30c816 100644 --- a/client/include/interface/audiodev.h +++ b/client/include/interface/audiodev.h @@ -25,6 +25,8 @@ #include #include +#include "interface/audio.h" + typedef int (*LG_AudioPullFn)(uint8_t * dst, int frames); typedef void (*LG_AudioPushFn)(uint8_t * src, int frames); @@ -50,14 +52,14 @@ struct LG_AudioDevOps * stream. * Note: the pull function returns f32 samples */ - bool (*setup)(int channels, int sampleRate, int requestedPeriodFrames, + bool (*setup)(const LG_AudioFormat * format, int requestedPeriodFrames, bool requestResampler, bool * resamplerEnabled, int * maxPeriodFrames, int * startFrames, LG_AudioPullFn pullFn); /* called when there is data available to start playback */ void (*start)(void); - /* called when SPICE reports the audio stream has stopped */ + /* called when the source reports the audio stream has stopped */ void (*stop)(void); /* [optional] called to set the volume of the channels */ @@ -77,12 +79,10 @@ struct LG_AudioDevOps struct { - /* start the record stream - * Note: currently SPICE only supports S16 samples so always assume so - */ - void (*start)(int channels, int sampleRate, LG_AudioPushFn pushFn); + /* start the record stream using the requested interleaved format */ + void (*start)(const LG_AudioFormat * format, LG_AudioPushFn pushFn); - /* called when SPICE reports the audio stream has stopped */ + /* called when the source reports the audio stream has stopped */ void (*stop)(void); /* [optional] called to set the volume of the channels */ diff --git a/client/include/interface/transport.h b/client/include/interface/transport.h index 174fdbe6..6fae07fa 100644 --- a/client/include/interface/transport.h +++ b/client/include/interface/transport.h @@ -27,6 +27,7 @@ #include "common/framebuffer.h" #include "common/types.h" +#include "interface/audio.h" #include "interface/input.h" #define LG_TRANSPORT_MAX_DAMAGE_RECTS LG_MAX_FRAME_DAMAGE_RECTS @@ -250,6 +251,10 @@ typedef struct LG_TransportOps * valid until disconnect; NULL indicates that this session has no input. */ const LG_InputOps *(*getInputOps)(LG_Transport * transport, void ** opaque); + /* Queried after connect. The returned operations and opaque value remain + * valid until disconnect; NULL indicates that this session has no audio. */ + const LG_AudioOps *(*getAudioOps)(LG_Transport * transport, + void ** opaque); bool (*attachRenderer)(LG_Transport * transport, const LG_RendererInterop * interop); void (*detachRenderer)(LG_Transport * transport); diff --git a/client/src/audio.c b/client/src/audio.c index c758f633..690a826c 100644 --- a/client/src/audio.c +++ b/client/src/audio.c @@ -23,6 +23,10 @@ #include "audio.h" #include "main.h" #include "common/array.h" +#include "common/debug.h" +#include "common/event.h" +#include "common/locking.h" +#include "common/thread.h" #include "common/util.h" #include "common/ringbuffer.h" @@ -51,7 +55,7 @@ * retains 20 frames at unity; the bounded ratio range changes this by less * than one frame. Keep it in the latency model, but not in the safety buffer. */ #define PLAYBACK_RESAMPLER_DELAY_FRAMES 20 -#define PLAYBACK_TIMESTAMP_DISCONTINUITY_MS 2000 +#define PLAYBACK_TIMESTAMP_DISCONTINUITY_NS INT64_C(2000000000) #define PLAYBACK_RATE_WINDOW_MS 60000 #define PLAYBACK_RATE_MIN_SPAN_MS 45000 #define PLAYBACK_RATE_SAMPLE_INTERVAL_MS 100 @@ -65,7 +69,7 @@ typedef enum { STREAM_STATE_STOP, - STREAM_STATE_SETUP_SPICE, + STREAM_STATE_SETUP_SOURCE, STREAM_STATE_SETUP_DEVICE, STREAM_STATE_RUN, STREAM_STATE_KEEP_ALIVE, @@ -119,9 +123,11 @@ typedef struct int64_t inputPosition; int64_t outputPosition; - uint32_t mediaTime; + int64_t mediaTime; + int64_t mediaElapsed; int64_t mediaTimeMs; int64_t mediaLocalOrigin; + uint64_t mediaPosition; int64_t lastPacketTime; int64_t lastArrivalTime; double arrivalJitterSec; @@ -130,6 +136,8 @@ typedef struct double sourcePacketDurationSec; bool sourcePhaseBaselineValid; bool mediaClockValid; + bool mediaPositionValid; + bool mediaClockFromSource; PlaybackRateSample rateSamples[PLAYBACK_RATE_MAX_SAMPLES]; unsigned int rateSampleStart; @@ -163,7 +171,7 @@ typedef struct PlaybackClock outputClock; SRC_STATE * src; } -PlaybackSpiceData; +PlaybackSourceData; typedef struct { @@ -175,17 +183,37 @@ typedef struct } PlaybackDeviceTiming; +typedef struct +{ + const LG_AudioOps * ops; + void * opaque; + bool available; + uint32_t generation; +} +AudioBinding; + typedef struct { struct LG_AudioDevOps * audioDev; + LG_Lock providerLock; + LG_RWLock activeLock; + AudioBinding fallback; + AudioBinding transport; + AudioBinding active; + struct { + LG_Lock sourceLock; _Atomic(StreamState) state; atomic_uint callbackState; + atomic_uint streamGeneration; int volumeChannels; - uint16_t volume[8]; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; bool mute; + LG_AudioFormat format; + LG_AudioFormat lastFormat; + bool lastFormatValid; int channels; int sampleRate; int stride; @@ -206,40 +234,195 @@ typedef struct GraphHandle graph; /* These two structs contain data specifically for use in the device and - * Spice data threads respectively. Keep them on separate cache lines to + * source data threads respectively. Keep them on separate cache lines to * avoid false sharing. */ alignas(64) PlaybackDeviceData deviceData; - alignas(64) PlaybackSpiceData spiceData; + alignas(64) PlaybackSourceData sourceData; } playback; struct { LG_Lock lock; + atomic_uint streamGeneration; bool shuttingDown; bool requested; bool started; int volumeChannels; - uint16_t volume[8]; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; bool mute; - atomic_int stride; - uint32_t time; - int lastChannels; - int lastSampleRate; - PSAudioFormat lastFormat; + LG_AudioFormat format; + LG_AudioFormat lastFormat; MsgBoxHandle confirmHandle; uint64_t confirmGeneration; bool confirmPending; - int confirmChannels; - int confirmSampleRate; - PSAudioFormat confirmFormat; + LG_AudioFormat confirmFormat; } record; + + struct + { + LG_Lock lock; + LGEvent * event; + LGThread * thread; + atomic_bool stop; + bool pending; + + const LG_AudioOps * ops; + void * opaque; + uint32_t bindingGeneration; + uint32_t generation; + LG_AudioClock clock; + } + feedback; } AudioState; static AudioState audio = { 0 }; +static size_t audioSampleSize(LG_AudioSampleFormat format) +{ + switch (format) + { + case LG_AUDIO_FMT_U8: return 1; + case LG_AUDIO_FMT_S16_LE: return 2; + case LG_AUDIO_FMT_S24_LE: return 3; + case LG_AUDIO_FMT_S32_LE: + case LG_AUDIO_FMT_F32_LE: return 4; + case LG_AUDIO_FMT_F64_LE: return 8; + } + + return 0; +} + +static bool audioFormatValid(const LG_AudioFormat * format) +{ + if (!format || format->channelCount < 1 || + format->channelCount > LG_AUDIO_MAX_CHANNELS || + format->sampleRate < 8000 || format->sampleRate > 384000 || + audioSampleSize(format->sampleFormat) == 0) + return false; + + for (unsigned int i = 0; i < format->channelCount; ++i) + if (format->channels[i] > LG_AUDIO_CH_TOP_REAR_RIGHT) + return false; + + return true; +} + +static bool audioFormatEqual(const LG_AudioFormat * a, + const LG_AudioFormat * b) +{ + return a->sampleFormat == b->sampleFormat && + a->sampleRate == b->sampleRate && + a->channelCount == b->channelCount && + memcmp(a->channels, b->channels, + sizeof(*a->channels) * a->channelCount) == 0; +} + +static bool audioConvertToFloat(float * dst, const void * src, + size_t samples, LG_AudioSampleFormat format) +{ + if (!dst || !src) + return false; + + switch (format) + { + case LG_AUDIO_FMT_U8: + { + const uint8_t * in = src; + for (size_t i = 0; i < samples; ++i) + dst[i] = ((int)in[i] - 128) / 128.0f; + return true; + } + + case LG_AUDIO_FMT_S16_LE: + { + const uint8_t * in = src; + for (size_t i = 0; i < samples; ++i, in += 2) + { + const int16_t value = (int16_t)( + (uint16_t)in[0] | (uint16_t)in[1] << 8); + dst[i] = value / 32768.0f; + } + return true; + } + + case LG_AUDIO_FMT_S24_LE: + { + const uint8_t * in = src; + for (size_t i = 0; i < samples; ++i, in += 3) + { + int32_t value = + (int32_t)((uint32_t)in[0] | + (uint32_t)in[1] << 8 | + (uint32_t)in[2] << 16); + if (value & 0x800000) + value |= (int32_t)0xff000000; + dst[i] = value / 8388608.0f; + } + return true; + } + + case LG_AUDIO_FMT_S32_LE: + { + const uint8_t * in = src; + for (size_t i = 0; i < samples; ++i, in += 4) + { + const int32_t value = (int32_t)( + (uint32_t)in[0] | + (uint32_t)in[1] << 8 | + (uint32_t)in[2] << 16 | + (uint32_t)in[3] << 24); + dst[i] = value / 2147483648.0f; + } + return true; + } + + case LG_AUDIO_FMT_F32_LE: + { +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + memcpy(dst, src, samples * sizeof(*dst)); +#else + const uint8_t * in = src; + for (size_t i = 0; i < samples; ++i, in += 4) + { + const uint32_t bits = + (uint32_t)in[0] | + (uint32_t)in[1] << 8 | + (uint32_t)in[2] << 16 | + (uint32_t)in[3] << 24; + memcpy(&dst[i], &bits, sizeof(bits)); + } +#endif + return true; + } + + case LG_AUDIO_FMT_F64_LE: + { + const uint8_t * in = src; + for (size_t i = 0; i < samples; ++i, in += 8) + { + const uint64_t bits = + (uint64_t)in[0] | + (uint64_t)in[1] << 8 | + (uint64_t)in[2] << 16 | + (uint64_t)in[3] << 24 | + (uint64_t)in[4] << 32 | + (uint64_t)in[5] << 40 | + (uint64_t)in[6] << 48 | + (uint64_t)in[7] << 56; + double value; + memcpy(&value, &bits, sizeof(value)); + dst[i] = value; + } + return true; + } + } + + return false; +} + typedef struct { int periodFrames; @@ -321,7 +504,7 @@ static bool playbackSourceClockUpdate(PlaybackClock * clock, int64_t time, return false; } - /* SPICE multimedia time periodically changes phase by several + /* source media time periodically changes phase by several * milliseconds. Preserve it as a diagnostic and discontinuity signal, but * advance the source clock solely from decoded sample position. Short-term * timestamp corrections must not move the playback buffer. */ @@ -332,57 +515,57 @@ static bool playbackSourceClockUpdate(PlaybackClock * clock, int64_t time, return true; } -static void playbackDeviceClockAcquireReset(PlaybackSpiceData * spiceData) +static void playbackDeviceClockAcquireReset(PlaybackSourceData * sourceData) { - spiceData->deviceClockAcquireStart = INT64_MIN; - spiceData->deviceClockCheckTime = INT64_MIN; - spiceData->deviceClockStableSec = 0.0; - spiceData->devicePositionOffsetFrames = 0.0; - spiceData->deviceClockStable = false; - spiceData->ratioIntegral = 0.0; - spiceData->lastClockRatio = 1.0; + sourceData->deviceClockAcquireStart = INT64_MIN; + sourceData->deviceClockCheckTime = INT64_MIN; + sourceData->deviceClockStableSec = 0.0; + sourceData->devicePositionOffsetFrames = 0.0; + sourceData->deviceClockStable = false; + sourceData->ratioIntegral = 0.0; + sourceData->lastClockRatio = 1.0; } static bool playbackDeviceClockAcquire( - PlaybackSpiceData * spiceData, int64_t time) + PlaybackSourceData * sourceData, int64_t time) { - if (spiceData->deviceClockStable) + if (sourceData->deviceClockStable) return false; - if (spiceData->deviceClockAcquireStart == INT64_MIN) + if (sourceData->deviceClockAcquireStart == INT64_MIN) { - spiceData->deviceClockAcquireStart = time; - spiceData->deviceClockCheckTime = time; - spiceData->deviceClockCheckFrameSec = - spiceData->deviceClock.frameSec; + sourceData->deviceClockAcquireStart = time; + sourceData->deviceClockCheckTime = time; + sourceData->deviceClockCheckFrameSec = + sourceData->deviceClock.frameSec; return false; } const double checkSec = - (time - spiceData->deviceClockCheckTime) * 1.0e-9; + (time - sourceData->deviceClockCheckTime) * 1.0e-9; if (checkSec < PLAYBACK_DEVICE_RATE_CHECK_SEC) return false; const double rateDeltaPpm = fabs( - spiceData->deviceClock.frameSec / - spiceData->deviceClockCheckFrameSec - 1.0) * 1.0e6; + sourceData->deviceClock.frameSec / + sourceData->deviceClockCheckFrameSec - 1.0) * 1.0e6; if (rateDeltaPpm <= PLAYBACK_DEVICE_RATE_STABLE_DELTA_PPM) - spiceData->deviceClockStableSec += checkSec; + sourceData->deviceClockStableSec += checkSec; else - spiceData->deviceClockStableSec = 0.0; + sourceData->deviceClockStableSec = 0.0; - spiceData->deviceClockCheckTime = time; - spiceData->deviceClockCheckFrameSec = - spiceData->deviceClock.frameSec; + sourceData->deviceClockCheckTime = time; + sourceData->deviceClockCheckFrameSec = + sourceData->deviceClock.frameSec; const double acquireSec = - (time - spiceData->deviceClockAcquireStart) * 1.0e-9; - if (spiceData->deviceClockStableSec < + (time - sourceData->deviceClockAcquireStart) * 1.0e-9; + if (sourceData->deviceClockStableSec < PLAYBACK_DEVICE_RATE_STABLE_SEC && acquireSec < PLAYBACK_DEVICE_RATE_MAX_ACQUIRE_SEC) return false; - spiceData->deviceClockStable = true; + sourceData->deviceClockStable = true; return true; } @@ -392,58 +575,58 @@ static double playbackClockPosition(const PlaybackClock * clock, int64_t time) (time - clock->time) * 1.0e-9 / clock->frameSec; } -static void playbackSourceRateReset(PlaybackSpiceData * spiceData) +static void playbackSourceRateReset(PlaybackSourceData * sourceData) { - spiceData->rateSampleStart = 0; - spiceData->rateSampleCount = 0; - spiceData->rateLastSampleTimeMs = INT64_MIN; - spiceData->rateFilterTimeMs = INT64_MIN; - spiceData->sourceRateValid = false; + sourceData->rateSampleStart = 0; + sourceData->rateSampleCount = 0; + sourceData->rateLastSampleTimeMs = INT64_MIN; + sourceData->rateFilterTimeMs = INT64_MIN; + sourceData->sourceRateValid = false; } static void playbackSourceRateAdd( - PlaybackSpiceData * spiceData, double nominalFrameSec) + PlaybackSourceData * sourceData, double nominalFrameSec) { - const int64_t timeMs = spiceData->mediaTimeMs; - if (spiceData->rateLastSampleTimeMs != INT64_MIN && - timeMs - spiceData->rateLastSampleTimeMs < + const int64_t timeMs = sourceData->mediaTimeMs; + if (sourceData->rateLastSampleTimeMs != INT64_MIN && + timeMs - sourceData->rateLastSampleTimeMs < PLAYBACK_RATE_SAMPLE_INTERVAL_MS) return; - spiceData->rateLastSampleTimeMs = timeMs; + sourceData->rateLastSampleTimeMs = timeMs; - while (spiceData->rateSampleCount > 0) + while (sourceData->rateSampleCount > 0) { const PlaybackRateSample * oldest = - &spiceData->rateSamples[spiceData->rateSampleStart]; + &sourceData->rateSamples[sourceData->rateSampleStart]; if (timeMs - oldest->timeMs <= PLAYBACK_RATE_WINDOW_MS) break; - spiceData->rateSampleStart = - (spiceData->rateSampleStart + 1) % PLAYBACK_RATE_MAX_SAMPLES; - --spiceData->rateSampleCount; + sourceData->rateSampleStart = + (sourceData->rateSampleStart + 1) % PLAYBACK_RATE_MAX_SAMPLES; + --sourceData->rateSampleCount; } - if (spiceData->rateSampleCount == PLAYBACK_RATE_MAX_SAMPLES) + if (sourceData->rateSampleCount == PLAYBACK_RATE_MAX_SAMPLES) { - spiceData->rateSampleStart = - (spiceData->rateSampleStart + 1) % PLAYBACK_RATE_MAX_SAMPLES; - --spiceData->rateSampleCount; + sourceData->rateSampleStart = + (sourceData->rateSampleStart + 1) % PLAYBACK_RATE_MAX_SAMPLES; + --sourceData->rateSampleCount; } const unsigned int index = - (spiceData->rateSampleStart + spiceData->rateSampleCount) % + (sourceData->rateSampleStart + sourceData->rateSampleCount) % PLAYBACK_RATE_MAX_SAMPLES; - spiceData->rateSamples[index] = (PlaybackRateSample) + sourceData->rateSamples[index] = (PlaybackRateSample) { .timeMs = timeMs, - .position = spiceData->inputPosition + .position = sourceData->inputPosition }; - ++spiceData->rateSampleCount; + ++sourceData->rateSampleCount; const PlaybackRateSample * first = - &spiceData->rateSamples[spiceData->rateSampleStart]; - if (spiceData->rateSampleCount < 2 || + &sourceData->rateSamples[sourceData->rateSampleStart]; + if (sourceData->rateSampleCount < 2 || timeMs - first->timeMs < PLAYBACK_RATE_MIN_SPAN_MS) return; @@ -451,11 +634,11 @@ static void playbackSourceRateAdd( double sumTime = 0.0; double sumPosition2 = 0.0; double sumPositionTime = 0.0; - for (unsigned int i = 0; i < spiceData->rateSampleCount; ++i) + for (unsigned int i = 0; i < sourceData->rateSampleCount; ++i) { const PlaybackRateSample * sample = - &spiceData->rateSamples[ - (spiceData->rateSampleStart + i) % PLAYBACK_RATE_MAX_SAMPLES]; + &sourceData->rateSamples[ + (sourceData->rateSampleStart + i) % PLAYBACK_RATE_MAX_SAMPLES]; const double position = sample->position - first->position; const double timeSec = (sample->timeMs - first->timeMs) / 1000.0; @@ -465,7 +648,7 @@ static void playbackSourceRateAdd( sumPositionTime += position * timeSec; } - const double count = spiceData->rateSampleCount; + const double count = sourceData->rateSampleCount; const double denominator = sumPosition2 - sumPosition * sumPosition / count; if (denominator <= 0.0) @@ -479,21 +662,21 @@ static void playbackSourceRateAdd( (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) return; - if (!spiceData->sourceRateValid) + if (!sourceData->sourceRateValid) { - spiceData->sourceRateFrameSec = frameSec; - spiceData->sourceRateValid = true; + sourceData->sourceRateFrameSec = frameSec; + sourceData->sourceRateValid = true; } else { const double elapsedSec = - (timeMs - spiceData->rateFilterTimeMs) / 1000.0; + (timeMs - sourceData->rateFilterTimeMs) / 1000.0; const double alpha = -expm1(-elapsedSec / PLAYBACK_RATE_FILTER_TIME_SEC); - spiceData->sourceRateFrameSec += - alpha * (frameSec - spiceData->sourceRateFrameSec); + sourceData->sourceRateFrameSec += + alpha * (frameSec - sourceData->sourceRateFrameSec); } - spiceData->rateFilterTimeMs = timeMs; + sourceData->rateFilterTimeMs = timeMs; } static void playbackPublishDeviceTiming( @@ -543,57 +726,89 @@ static bool playbackReadDeviceTiming( return true; } -static void playbackResetMediaClock( - PlaybackSpiceData * spiceData, uint32_t time, int64_t now) +static void playbackResetMediaClock(PlaybackSourceData * sourceData, + int64_t time, uint64_t position, bool fromSource, int64_t now) { - spiceData->mediaTime = time; - spiceData->mediaTimeMs = 0; - spiceData->mediaLocalOrigin = now; - spiceData->lastPacketTime = INT64_MIN; - spiceData->lastArrivalTime = INT64_MIN; - spiceData->mediaClockValid = true; - spiceData->inputPosition = 0; - spiceData->sourceClock.valid = false; - playbackSourceRateReset(spiceData); + sourceData->mediaTime = time; + sourceData->mediaElapsed = 0; + sourceData->mediaTimeMs = 0; + sourceData->mediaLocalOrigin = now; + sourceData->mediaPosition = position; + sourceData->lastPacketTime = INT64_MIN; + sourceData->lastArrivalTime = INT64_MIN; + sourceData->mediaClockValid = true; + sourceData->mediaPositionValid = true; + sourceData->mediaClockFromSource = fromSource; + sourceData->sourceClock.valid = false; + playbackSourceRateReset(sourceData); } static void playbackPrepareMediaClock( - PlaybackSpiceData * spiceData, uint32_t time) + PlaybackSourceData * sourceData, const LG_AudioClock * sourceClock) { - spiceData->mediaTime = time; - spiceData->mediaClockValid = false; - spiceData->inputPosition = 0; - spiceData->sourceClock.valid = false; - playbackSourceRateReset(spiceData); + sourceData->mediaTime = 0; + sourceData->mediaClockValid = false; + sourceData->mediaPositionValid = sourceClock != NULL; + sourceData->mediaClockFromSource = sourceClock != NULL; + sourceData->mediaPosition = sourceClock ? sourceClock->position : 0; + sourceData->inputPosition = 0; + sourceData->sourceClock.valid = false; + playbackSourceRateReset(sourceData); } -static int64_t playbackMapMediaTime(PlaybackSpiceData * spiceData, - uint32_t time, int64_t now, bool * discontinuity) +static int64_t playbackMapMediaTime(PlaybackSourceData * sourceData, + const LG_AudioClock * clock, int frames, int sampleRate, + int64_t now, bool * discontinuity) { - if (!spiceData->mediaClockValid) + const bool fromSource = clock != NULL; + const uint64_t position = clock ? clock->position : + (uint64_t)sourceData->inputPosition; + const int64_t time = clock ? clock->time : + llrint(sourceData->inputPosition * (1.0e9 / sampleRate)); + + if (clock && clock->discontinuity) + *discontinuity = true; + + if (!sourceData->mediaClockValid) { - playbackResetMediaClock(spiceData, time, now); + if (sourceData->mediaPositionValid && + (sourceData->mediaClockFromSource != fromSource || + sourceData->mediaPosition != position)) + *discontinuity = true; + + playbackResetMediaClock( + sourceData, time, position, fromSource, now); + sourceData->mediaPosition = position + frames; return now; } - const int32_t deltaMs = (int32_t)(time - spiceData->mediaTime); - if (deltaMs < 0 || deltaMs > PLAYBACK_TIMESTAMP_DISCONTINUITY_MS) + const int64_t delta = time - sourceData->mediaTime; + if (*discontinuity || + sourceData->mediaClockFromSource != fromSource || + !sourceData->mediaPositionValid || + sourceData->mediaPosition != position || delta < 0 || + delta > PLAYBACK_TIMESTAMP_DISCONTINUITY_NS) { - playbackResetMediaClock(spiceData, time, now); + playbackResetMediaClock( + sourceData, time, position, fromSource, now); + sourceData->mediaPosition = position + frames; *discontinuity = true; return now; } - spiceData->mediaTime = time; - spiceData->mediaTimeMs += deltaMs; - return spiceData->mediaLocalOrigin + spiceData->mediaTimeMs * 1000000; + sourceData->mediaTime = time; + sourceData->mediaPosition = position + frames; + sourceData->mediaElapsed += delta; + sourceData->mediaTimeMs = + sourceData->mediaElapsed / INT64_C(1000000); + return sourceData->mediaLocalOrigin + sourceData->mediaElapsed; } static void playbackStop(void); static MsgBoxHandle recordCancelConfirmLocked(void); -static void realRecordStartLocked( - int channels, int sampleRate, PSAudioFormat format); +static void realRecordStartLocked(const LG_AudioFormat * format); static void realRecordStopLocked(void); +static void recordStop(void); static StreamState playbackGetState(void) { @@ -646,51 +861,7 @@ static void playbackWaitForCallbacks(void) ; } -void audio_init(void) -{ - LG_LOCK_INIT(audio.record.lock); - audio.record.shuttingDown = false; - atomic_store_explicit( - &audio.playback.callbackState, PLAYBACK_CALLBACK_DISABLED, - memory_order_release); - - // search for the best audiodev to use - for(int i = 0; i < LG_AUDIODEV_COUNT; ++i) - if (LG_AudioDevs[i]->init()) - { - audio.audioDev = LG_AudioDevs[i]; - DEBUG_INFO("Using AudioDev: %s", audio.audioDev->name); - return; - } - - DEBUG_WARN("Failed to initialize an audio backend"); -} - -void audio_free(void) -{ - // immediate stop of the stream, do not wait for drain - if (audio.audioDev) - playbackStop(); - - LG_LOCK(audio.record.lock); - audio.record.shuttingDown = true; - audio.record.requested = false; - MsgBoxHandle confirm = recordCancelConfirmLocked(); - - if (audio.audioDev && audio.record.started) - realRecordStopLocked(); - - struct LG_AudioDevOps * audioDev = audio.audioDev; - audio.audioDev = NULL; - LG_UNLOCK(audio.record.lock); - - app_msgBoxClose(confirm); - - if (audioDev) - audioDev->free(); -} - -bool audio_supportsPlayback(void) +bool lgAudio_supportsPlayback(void) { return audio.audioDev && audio.audioDev->playback.start; } @@ -716,16 +887,16 @@ static void playbackStop(void) playbackSetState(STREAM_STATE_STOP); ringbuffer_free(&audio.playback.buffer); - audio.playback.spiceData.src = src_delete(audio.playback.spiceData.src); + audio.playback.sourceData.src = src_delete(audio.playback.sourceData.src); - if (audio.playback.spiceData.framesIn) + if (audio.playback.sourceData.framesIn) { - free(audio.playback.spiceData.framesIn); - free(audio.playback.spiceData.framesOut); - audio.playback.spiceData.framesIn = NULL; - audio.playback.spiceData.framesOut = NULL; - audio.playback.spiceData.framesInSize = 0; - audio.playback.spiceData.framesOutSize = 0; + free(audio.playback.sourceData.framesIn); + free(audio.playback.sourceData.framesOut); + audio.playback.sourceData.framesIn = NULL; + audio.playback.sourceData.framesOut = NULL; + audio.playback.sourceData.framesInSize = 0; + audio.playback.sourceData.framesOutSize = 0; } if (audio.playback.timings) @@ -867,37 +1038,35 @@ static int playbackPullFrames(uint8_t * dst, int frames) return frames; } -void audio_playbackStart(int channels, int sampleRate, PSAudioFormat format, - uint32_t time) +static void playbackStart(const LG_AudioFormat * format, + const LG_AudioClock * sourceClock) { if (!audio.audioDev) return; - if (channels < 1 || channels > 8 || sampleRate < 8000 || - sampleRate > 384000 || format != PS_AUDIO_FMT_S16) + if (!audioFormatValid(format)) { - DEBUG_ERROR("Invalid playback format: %d channels, %d Hz, format %d", - channels, sampleRate, format); + DEBUG_ERROR("Invalid playback format"); if (playbackGetState() != STREAM_STATE_STOP) playbackStop(); return; } - static int lastChannels = 0; - static int lastSampleRate = 0; - static PSAudioFormat lastFormat = PS_AUDIO_FMT_INVALID; + const int channels = format->channelCount; + const int sampleRate = format->sampleRate; StreamState state = playbackGetState(); if (state == STREAM_STATE_KEEP_ALIVE && - channels == lastChannels && sampleRate == lastSampleRate && - format == lastFormat) + audio.playback.lastFormatValid && + audioFormatEqual(format, &audio.playback.lastFormat)) { StreamState expected = STREAM_STATE_KEEP_ALIVE; if (atomic_compare_exchange_strong_explicit( &audio.playback.state, &expected, STREAM_STATE_RESUMING, memory_order_acq_rel, memory_order_acquire)) { - playbackPrepareMediaClock(&audio.playback.spiceData, time); + playbackPrepareMediaClock( + &audio.playback.sourceData, sourceClock); return; } @@ -913,43 +1082,43 @@ void audio_playbackStart(int channels, int sampleRate, PSAudioFormat format, if (!audio.playback.buffer) return; - lastChannels = channels; - lastSampleRate = sampleRate; - lastFormat = format; + audio.playback.format = *format; + audio.playback.lastFormat = *format; + audio.playback.lastFormatValid = true; audio.playback.channels = channels; audio.playback.sampleRate = sampleRate; audio.playback.stride = channels * sizeof(float); - playbackSetState(STREAM_STATE_SETUP_SPICE); + playbackSetState(STREAM_STATE_SETUP_SOURCE); audio.playback.deviceData.nextPosition = 0; audio.playback.deviceData.outputPosition = 0.0; audio.playback.deviceData.appliedRatio = 1.0; audio.playback.deviceData.startupSilenceFrames = 0; - audio.playback.spiceData.inputPosition = 0; - audio.playback.spiceData.outputPosition = 0; - audio.playback.spiceData.devPeriodFrames = 0; - audio.playback.spiceData.devReadPosition = 0; - audio.playback.spiceData.deviceTimingSequence = 0; - playbackDeviceClockAcquireReset(&audio.playback.spiceData); - audio.playback.spiceData.offsetError = 0.0; - audio.playback.spiceData.offsetErrorIntegral = 0.0; - audio.playback.spiceData.ratioIntegral = 0.0; - audio.playback.spiceData.lastRatio = 1.0; - audio.playback.spiceData.lastClockRatio = 1.0; - audio.playback.spiceData.bufferOverrunPending = false; - audio.playback.spiceData.bufferOverruns = 0; - audio.playback.spiceData.nextLogTime = + audio.playback.sourceData.inputPosition = 0; + audio.playback.sourceData.outputPosition = 0; + audio.playback.sourceData.devPeriodFrames = 0; + audio.playback.sourceData.devReadPosition = 0; + audio.playback.sourceData.deviceTimingSequence = 0; + playbackDeviceClockAcquireReset(&audio.playback.sourceData); + audio.playback.sourceData.offsetError = 0.0; + audio.playback.sourceData.offsetErrorIntegral = 0.0; + audio.playback.sourceData.ratioIntegral = 0.0; + audio.playback.sourceData.lastRatio = 1.0; + audio.playback.sourceData.lastClockRatio = 1.0; + audio.playback.sourceData.bufferOverrunPending = false; + audio.playback.sourceData.bufferOverruns = 0; + audio.playback.sourceData.nextLogTime = nanotime() + INT64_C(5000000000); - audio.playback.spiceData.arrivalJitterSec = 0.0; - audio.playback.spiceData.sourcePhaseBaselineSec = 0.0; - audio.playback.spiceData.sourcePhaseReserveSec = 0.0; - audio.playback.spiceData.sourcePacketDurationSec = 0.0; - audio.playback.spiceData.sourcePhaseBaselineValid = false; - audio.playback.spiceData.deviceClock.valid = false; - audio.playback.spiceData.outputClock.valid = false; - playbackPrepareMediaClock(&audio.playback.spiceData, time); + audio.playback.sourceData.arrivalJitterSec = 0.0; + audio.playback.sourceData.sourcePhaseBaselineSec = 0.0; + audio.playback.sourceData.sourcePhaseReserveSec = 0.0; + audio.playback.sourceData.sourcePacketDurationSec = 0.0; + audio.playback.sourceData.sourcePhaseBaselineValid = false; + audio.playback.sourceData.deviceClock.valid = false; + audio.playback.sourceData.outputClock.valid = false; + playbackPrepareMediaClock(&audio.playback.sourceData, sourceClock); atomic_store_explicit( &audio.playback.deviceTiming.sequence, 0, memory_order_relaxed); @@ -981,7 +1150,9 @@ void audio_playbackStart(int channels, int sampleRate, PSAudioFormat format, audio.playback.startupPacketPeriod = 0; const bool requestBackendResampler = g_params.audioResampler != AUDIO_RESAMPLER_LIBSAMPLERATE; - if (!audio.audioDev->playback.setup(channels, sampleRate, + LG_AudioFormat deviceFormat = *format; + deviceFormat.sampleFormat = LG_AUDIO_FMT_F32_LE; + if (!audio.audioDev->playback.setup(&deviceFormat, requestedPeriodFrames, requestBackendResampler, &audio.playback.backendResampler, &audio.playback.deviceMaxPeriodFrames, @@ -1002,9 +1173,9 @@ void audio_playbackStart(int channels, int sampleRate, PSAudioFormat format, if (!audio.playback.backendResampler) { int srcError; - audio.playback.spiceData.src = + audio.playback.sourceData.src = src_new(SRC_SINC_FASTEST, channels, &srcError); - if (!audio.playback.spiceData.src) + if (!audio.playback.sourceData.src) { DEBUG_ERROR("Failed to create resampler: %s", src_strerror(srcError)); playbackStop(); @@ -1012,7 +1183,7 @@ void audio_playbackStart(int channels, int sampleRate, PSAudioFormat format, } } else - audio.playback.spiceData.src = NULL; + audio.playback.sourceData.src = NULL; DEBUG_INFO("Using audio resampler: %s", audio.playback.backendResampler ? @@ -1036,7 +1207,7 @@ void audio_playbackStart(int channels, int sampleRate, PSAudioFormat format, &audio.playback.callbackState, 0, memory_order_release); } -void audio_playbackStop(void) +static void playbackSourceStop(void) { if (!audio.audioDev) return; @@ -1051,9 +1222,9 @@ void audio_playbackStop(void) playbackSetState(STREAM_STATE_KEEP_ALIVE); // Reset the software resampler so it is safe for the next playback - if (audio.playback.spiceData.src) + if (audio.playback.sourceData.src) { - int error = src_reset(audio.playback.spiceData.src); + int error = src_reset(audio.playback.sourceData.src); if (error) { DEBUG_ERROR("Failed to reset resampler: %s", src_strerror(error)); @@ -1064,7 +1235,7 @@ void audio_playbackStop(void) break; } - case STREAM_STATE_SETUP_SPICE: + case STREAM_STATE_SETUP_SOURCE: case STREAM_STATE_SETUP_DEVICE: case STREAM_STATE_STOP_PENDING: // Playback hasn't actually started yet so just clean up @@ -1078,7 +1249,7 @@ void audio_playbackStop(void) } } -void audio_playbackVolume(int channels, const uint16_t volume[]) +static void playbackVolume(int channels, const uint16_t volume[]) { if (!audio.audioDev || !audio.audioDev->playback.volume || !g_params.audioSyncVolume) @@ -1095,7 +1266,7 @@ void audio_playbackVolume(int channels, const uint16_t volume[]) audio.audioDev->playback.volume(channels, volume); } -void audio_playbackMute(bool mute) +static void playbackMute(bool mute) { if (!audio.audioDev || !audio.audioDev->playback.mute) return; @@ -1110,19 +1281,19 @@ void audio_playbackMute(bool mute) static double computeDevicePosition(int64_t curTime) { - const PlaybackSpiceData * spiceData = - &audio.playback.spiceData; + const PlaybackSourceData * sourceData = + &audio.playback.sourceData; return playbackClockPosition( - &spiceData->deviceClock, curTime) + - spiceData->devicePositionOffsetFrames; + &sourceData->deviceClock, curTime) + + sourceData->devicePositionOffsetFrames; } static bool playbackEnsureConversionBuffers( - PlaybackSpiceData * spiceData, int frames) + PlaybackSourceData * sourceData, int frames) { - if (frames > spiceData->framesInSize) + if (frames > sourceData->framesInSize) { - float * framesIn = realloc(spiceData->framesIn, + float * framesIn = realloc(sourceData->framesIn, (size_t)frames * audio.playback.stride); if (!framesIn) { @@ -1130,17 +1301,17 @@ static bool playbackEnsureConversionBuffers( return false; } - spiceData->framesIn = framesIn; - spiceData->framesInSize = frames; + sourceData->framesIn = framesIn; + sourceData->framesInSize = frames; } if (!audio.playback.backendResampler) { const int framesOut = (int)ceil(frames * (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) + 64; - if (framesOut > spiceData->framesOutSize) + if (framesOut > sourceData->framesOutSize) { - float * output = realloc(spiceData->framesOut, + float * output = realloc(sourceData->framesOut, (size_t)framesOut * audio.playback.stride); if (!output) { @@ -1148,8 +1319,8 @@ static bool playbackEnsureConversionBuffers( return false; } - spiceData->framesOut = output; - spiceData->framesOutSize = framesOut; + sourceData->framesOut = output; + sourceData->framesOutSize = framesOut; } } @@ -1157,7 +1328,7 @@ static bool playbackEnsureConversionBuffers( } static int playbackAppendFrames( - PlaybackSpiceData * spiceData, const void * frames, int count) + PlaybackSourceData * sourceData, const void * frames, int count) { const int occupancy = ringbuffer_getCount(audio.playback.buffer); const int length = ringbuffer_getLength(audio.playback.buffer); @@ -1175,15 +1346,15 @@ static int playbackAppendFrames( * Doing so makes a positive buffer count refer to overwritten samples and * sounds like corrupted PCM rather than an underrun. Resynchronize on the * next packet after dropping the excess output. */ - spiceData->bufferOverrunPending = true; - ++spiceData->bufferOverruns; + sourceData->bufferOverrunPending = true; + ++sourceData->bufferOverruns; } return advanced; } static int playbackSlewBuffer( - PlaybackSpiceData * spiceData, int requested) + PlaybackSourceData * sourceData, int requested) { const int occupancy = ringbuffer_getCount(audio.playback.buffer); const int length = ringbuffer_getLength(audio.playback.buffer); @@ -1196,12 +1367,13 @@ static int playbackSlewBuffer( DEBUG_ASSERT(advanced == slew); if (slew != requested) - spiceData->bufferOverrunPending = true; + sourceData->bufferOverrunPending = true; return advanced; } -void audio_playbackData(uint8_t * data, size_t size, uint32_t time) +static void playbackData(const void * data, size_t frameCount, + const LG_AudioClock * sourceClock) { StreamState state = playbackGetState(); if (state == STREAM_STATE_STOP_PENDING) @@ -1210,7 +1382,7 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) return; } - if (state == STREAM_STATE_STOP || !audio.audioDev || size == 0) + if (state == STREAM_STATE_STOP || !audio.audioDev || frameCount == 0) return; if (audio.playback.backendResampler && @@ -1223,151 +1395,162 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) return; } - PlaybackSpiceData * spiceData = &audio.playback.spiceData; + PlaybackSourceData * sourceData = &audio.playback.sourceData; /* Backend resampling changes how many source frames PipeWire requests per * device period. Use the command-normalized output clock for rate matching, * while deviceClock remains in the ring's source-frame domain for latency. */ const PlaybackClock * rateClock = audio.playback.backendResampler ? - &spiceData->outputClock : &spiceData->deviceClock; + &sourceData->outputClock : &sourceData->deviceClock; const int64_t now = nanotime(); const double nominalFrameSec = 1.0 / audio.playback.sampleRate; - const int spiceStride = audio.playback.channels * sizeof(int16_t); - if (size % spiceStride != 0 || size / spiceStride > INT_MAX) + if (!data || frameCount > INT_MAX || + frameCount > (size_t)audio.playback.sampleRate * 2) { - DEBUG_ERROR("Invalid playback packet size: %zu bytes for stride %d", - size, spiceStride); + DEBUG_ERROR("Invalid playback packet length: %zu frames", frameCount); playbackStop(); return; } + const int frames = frameCount; - const int frames = size / spiceStride; - if (frames == 0 || frames > audio.playback.sampleRate * 2) - { - DEBUG_ERROR("Invalid playback packet length: %d frames", frames); - playbackStop(); - return; - } - - if (!playbackEnsureConversionBuffers(spiceData, frames)) + if (!playbackEnsureConversionBuffers(sourceData, frames)) { playbackStop(); return; } - src_short_to_float_array((int16_t *) data, spiceData->framesIn, - frames * audio.playback.channels); + if (!audioConvertToFloat(sourceData->framesIn, data, + (size_t)frames * audio.playback.channels, + audio.playback.format.sampleFormat)) + { + DEBUG_ERROR("Failed to convert playback samples"); + playbackStop(); + return; + } - bool discontinuity = false; + bool discontinuity = sourceClock && sourceClock->discontinuity; const int64_t packetTime = - playbackMapMediaTime(spiceData, time, now, &discontinuity); - if (spiceData->bufferOverrunPending) + playbackMapMediaTime(sourceData, sourceClock, frames, + audio.playback.sampleRate, now, &discontinuity); + if (sourceData->bufferOverrunPending) { discontinuity = true; - spiceData->bufferOverrunPending = false; + sourceData->bufferOverrunPending = false; } - if (spiceData->lastPacketTime != INT64_MIN && - spiceData->lastArrivalTime != INT64_MIN) + if (sourceData->lastPacketTime != INT64_MIN && + sourceData->lastArrivalTime != INT64_MIN) { const double mediaDelta = - (packetTime - spiceData->lastPacketTime) * 1.0e-9; + (packetTime - sourceData->lastPacketTime) * 1.0e-9; const double arrivalDelta = - (now - spiceData->lastArrivalTime) * 1.0e-9; + (now - sourceData->lastArrivalTime) * 1.0e-9; const double jitter = fabs(arrivalDelta - mediaDelta); /* Keep a slowly decaying peak rather than feeding arrival jitter into the * virtual clock. This lets the buffer absorb real delivery jitter while - * the rate controller follows only the SPICE multimedia clock. */ - spiceData->arrivalJitterSec = + * the rate controller follows only the source media clock. */ + sourceData->arrivalJitterSec = min(PLAYBACK_MAX_JITTER_SEC, - max(jitter, spiceData->arrivalJitterSec * 0.999)); + max(jitter, sourceData->arrivalJitterSec * 0.999)); } - spiceData->lastPacketTime = packetTime; - spiceData->lastArrivalTime = now; + sourceData->lastPacketTime = packetTime; + sourceData->lastArrivalTime = now; const bool sourceRateWasValid = - spiceData->sourceRateValid; - playbackSourceRateAdd(spiceData, nominalFrameSec); + sourceData->sourceRateValid; + playbackSourceRateAdd(sourceData, nominalFrameSec); + if (sourceClock && sourceClock->stable && sourceClock->rate > 0.0) + { + const double frameSec = 1.0 / sourceClock->rate; + if (frameSec >= nominalFrameSec * + (1.0 - PLAYBACK_MAX_RATE_CORRECTION) && + frameSec <= nominalFrameSec * + (1.0 + PLAYBACK_MAX_RATE_CORRECTION)) + { + sourceData->sourceRateFrameSec = frameSec; + sourceData->sourceRateValid = true; + } + } const bool sourceRateBecameValid = - !sourceRateWasValid && spiceData->sourceRateValid; - if (!playbackSourceClockUpdate(&spiceData->sourceClock, - packetTime, spiceData->inputPosition, nominalFrameSec)) + !sourceRateWasValid && sourceData->sourceRateValid; + if (!playbackSourceClockUpdate(&sourceData->sourceClock, + packetTime, sourceData->inputPosition, nominalFrameSec)) discontinuity = true; - if (spiceData->sourceRateValid) - spiceData->sourceClock.frameSec = spiceData->sourceRateFrameSec; + if (sourceData->sourceRateValid) + sourceData->sourceClock.frameSec = sourceData->sourceRateFrameSec; /* Track phase variation around its local baseline, not its absolute value. * The absolute phase depends on the arbitrary local origin assigned to the - * SPICE multimedia clock and must not become buffer reserve. Positive + * source media clock and must not become buffer reserve. Positive * deviation means the latency model temporarily overstates how much audio * remains in the ring. */ const double sourcePhaseSec = - spiceData->sourceClock.phaseResidualSec; + sourceData->sourceClock.phaseResidualSec; const double packetSec = frames * nominalFrameSec; - spiceData->sourcePacketDurationSec = - max(packetSec, spiceData->sourcePacketDurationSec * + sourceData->sourcePacketDurationSec = + max(packetSec, sourceData->sourcePacketDurationSec * exp(-packetSec / PLAYBACK_PHASE_RESERVE_DECAY_SEC)); - if (!spiceData->sourcePhaseBaselineValid || - spiceData->sourceClock.updates == 1) + if (!sourceData->sourcePhaseBaselineValid || + sourceData->sourceClock.updates == 1) { - spiceData->sourcePhaseBaselineSec = sourcePhaseSec; - spiceData->sourcePhaseBaselineValid = true; + sourceData->sourcePhaseBaselineSec = sourcePhaseSec; + sourceData->sourcePhaseBaselineValid = true; } else { const double alpha = -expm1(-packetSec / PLAYBACK_PHASE_BASELINE_TIME_SEC); - spiceData->sourcePhaseBaselineSec += + sourceData->sourcePhaseBaselineSec += alpha * (sourcePhaseSec - - spiceData->sourcePhaseBaselineSec); + sourceData->sourcePhaseBaselineSec); } const double sourcePhaseDeviationSec = max(0.0, sourcePhaseSec - - spiceData->sourcePhaseBaselineSec); - spiceData->sourcePhaseReserveSec = + sourceData->sourcePhaseBaselineSec); + sourceData->sourcePhaseReserveSec = min(PLAYBACK_MAX_JITTER_SEC, max(sourcePhaseDeviationSec, - spiceData->sourcePhaseReserveSec * + sourceData->sourcePhaseReserveSec * exp(-packetSec / PLAYBACK_PHASE_RESERVE_DECAY_SEC))); - int64_t curTime = spiceData->sourceClock.time; - int64_t curPosition = spiceData->outputPosition; + int64_t curTime = sourceData->sourceClock.time; + int64_t curPosition = sourceData->outputPosition; const double sourceReserveFrames = - max(spiceData->sourcePacketDurationSec * 0.5, - spiceData->sourcePhaseReserveSec) * + max(sourceData->sourcePacketDurationSec * 0.5, + sourceData->sourcePhaseReserveSec) * audio.playback.sampleRate; // Receive the newest timing information from the audio device thread. PlaybackDeviceTick deviceTick; unsigned int deviceSequence; bool deviceClockBecameStable = false; - if (playbackReadDeviceTiming(spiceData->deviceTimingSequence, + if (playbackReadDeviceTiming(sourceData->deviceTimingSequence, &deviceTick, &deviceSequence)) { - spiceData->deviceTimingSequence = deviceSequence; - spiceData->devPeriodFrames = deviceTick.periodFrames; - spiceData->devReadPosition = + sourceData->deviceTimingSequence = deviceSequence; + sourceData->devPeriodFrames = deviceTick.periodFrames; + sourceData->devReadPosition = deviceTick.nextPosition + deviceTick.periodFrames; const bool deviceClockUpdated = - playbackClockUpdate(&spiceData->deviceClock, + playbackClockUpdate(&sourceData->deviceClock, deviceTick.nextTime, deviceTick.nextPosition, nominalFrameSec); const bool outputClockUpdated = !audio.playback.backendResampler || - playbackClockUpdate(&spiceData->outputClock, + playbackClockUpdate(&sourceData->outputClock, deviceTick.nextTime, deviceTick.outputPosition, nominalFrameSec); if (!deviceClockUpdated || !outputClockUpdated) { - playbackDeviceClockAcquireReset(spiceData); + playbackDeviceClockAcquireReset(sourceData); discontinuity = true; } else deviceClockBecameStable = - playbackDeviceClockAcquire(spiceData, deviceTick.nextTime); + playbackDeviceClockAcquire(sourceData, deviceTick.nextTime); } if (deviceClockBecameStable) @@ -1376,16 +1559,16 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) * acquisition model. Their position origins are otherwise unrelated, so * switching models would create a false phase step and drive the resampler * despite an already-correct ring level. Keep the source clock untouched: - * changing it would also disturb SPICE phase and jitter tracking. */ + * changing it would also disturb source phase and jitter tracking. */ const double rawDevicePosition = - playbackClockPosition(&spiceData->deviceClock, curTime); - spiceData->devicePositionOffsetFrames = - spiceData->devReadPosition - sourceReserveFrames - + playbackClockPosition(&sourceData->deviceClock, curTime); + sourceData->devicePositionOffsetFrames = + sourceData->devReadPosition - sourceReserveFrames - rawDevicePosition; } const int maxPeriodFrames = - max(audio.playback.deviceMaxPeriodFrames, spiceData->devPeriodFrames); + max(audio.playback.deviceMaxPeriodFrames, sourceData->devPeriodFrames); /* The device period, delivery jitter, packet phase, and resampler delay * define the minimum viable latency. latencyOffset is strictly an additive * user offset over that same minimum for both startup and steady state. */ @@ -1393,7 +1576,7 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) max(g_params.audioLatencyOffset, 0) * audio.playback.sampleRate / 1000.0; const double arrivalReserveFrames = - (spiceData->arrivalJitterSec + 0.001) * + (sourceData->arrivalJitterSec + 0.001) * audio.playback.sampleRate; const double minimumLowWaterReserveFrames = maxPeriodFrames * 0.1 + arrivalReserveFrames; @@ -1418,28 +1601,28 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) if ((discontinuity || state == STREAM_STATE_KEEP_ALIVE || state == STREAM_STATE_RESUMING) && - spiceData->deviceClock.valid && - spiceData->deviceClockStable) + sourceData->deviceClock.valid && + sourceData->deviceClockStable) { devPosition = computeDevicePosition(curTime); const double slew = devPosition + targetBufferFrames - curPosition; const int slewFrames = clamp(llrint(slew), (int64_t)INT_MIN, (int64_t)INT_MAX); - const int actualSlew = playbackSlewBuffer(spiceData, slewFrames); - spiceData->outputPosition += actualSlew; + const int actualSlew = playbackSlewBuffer(sourceData, slewFrames); + sourceData->outputPosition += actualSlew; curPosition += actualSlew; - spiceData->offsetError = 0.0; - spiceData->offsetErrorIntegral = 0.0; - spiceData->ratioIntegral = 0.0; + sourceData->offsetError = 0.0; + sourceData->offsetErrorIntegral = 0.0; + sourceData->ratioIntegral = 0.0; playbackSetState(STREAM_STATE_RUN); } double actualLatencyFrames = 0.0; double actualOffsetError = 0.0; - if (spiceData->deviceClock.valid) + if (sourceData->deviceClock.valid) { - if (spiceData->deviceClockStable) + if (sourceData->deviceClockStable) { if (devPosition == DBL_MIN) devPosition = computeDevicePosition(curTime); @@ -1452,23 +1635,23 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) else { actualLatencyFrames = - curPosition - spiceData->devReadPosition + + curPosition - sourceData->devReadPosition + sourceReserveFrames + resamplerDelayFrames; actualOffsetError = targetLatencyFrames - actualLatencyFrames; } const double error = - actualOffsetError - spiceData->offsetError; + actualOffsetError - sourceData->offsetError; const double periodSec = frames * nominalFrameSec; const double omega = 2.0 * M_PI * PLAYBACK_OFFSET_FILTER_BANDWIDTH_HZ * periodSec; const double b = M_SQRT2 * omega; const double c = omega * omega; - spiceData->offsetError += b * error + - spiceData->offsetErrorIntegral; - spiceData->offsetErrorIntegral += c * error; + sourceData->offsetError += b * error + + sourceData->offsetErrorIntegral; + sourceData->offsetErrorIntegral += c * error; } /* Feed forward the measured source/device rate ratio, then use a slow, @@ -1492,27 +1675,27 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) const double ki = naturalFrequency * naturalFrequency / audio.playback.sampleRate; if (sourceRateBecameValid) - spiceData->ratioIntegral = 0.0; + sourceData->ratioIntegral = 0.0; - if (spiceData->deviceClockStable && - spiceData->sourceRateValid && + if (sourceData->deviceClockStable && + sourceData->sourceRateValid && rateClock->updates >= 2) { const double clockRatio = clamp( - spiceData->sourceRateFrameSec / + sourceData->sourceRateFrameSec / rateClock->frameSec, 1.0 - PLAYBACK_MAX_RATE_CORRECTION, 1.0 + PLAYBACK_MAX_RATE_CORRECTION); - spiceData->lastClockRatio = clockRatio; + sourceData->lastClockRatio = clockRatio; } const double periodSec = frames * nominalFrameSec; - /* SPICE timestamps have millisecond resolution. Do not resample in + /* source timestamps have millisecond resolution. Do not resample in * response to phase error that cannot be distinguished from quantization; * subtracting the deadband outside it keeps the response continuous. */ const double phaseDeadbandFrames = PLAYBACK_PHASE_DEADBAND_SEC * audio.playback.sampleRate; - const double rawPhaseError = spiceData->offsetError; + const double rawPhaseError = sourceData->offsetError; double phaseError = rawPhaseError; if (fabs(phaseError) <= phaseDeadbandFrames) phaseError = 0.0; @@ -1520,11 +1703,11 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) phaseError -= copysign(phaseDeadbandFrames, phaseError); const bool acquiringDeviceClock = - spiceData->deviceClock.valid && !spiceData->deviceClockStable; + sourceData->deviceClock.valid && !sourceData->deviceClockStable; double controllerKp = kp; double controllerKi = ki; double controllerError = phaseError; - double controllerBase = spiceData->lastClockRatio; + double controllerBase = sourceData->lastClockRatio; if (acquiringDeviceClock) { @@ -1533,7 +1716,7 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) controllerKp = 2.0 * acquireFrequency / audio.playback.sampleRate; controllerBase = 1.0; - spiceData->ratioIntegral = 0.0; + sourceData->ratioIntegral = 0.0; if (actualOffsetError <= 0.0) controllerError = 0.0; @@ -1545,16 +1728,16 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) /* Acquisition correction is transient, not a clock-rate estimate. Start * the stable integral clean; the output-rate slew keeps the applied ratio * continuous across this transition. */ - spiceData->ratioIntegral = 0.0; + sourceData->ratioIntegral = 0.0; } /* Use the unfiltered latency error here so filter lag cannot retain a phase * correction after the target has already been crossed. */ - else if (spiceData->ratioIntegral * actualOffsetError <= 0.0) - spiceData->ratioIntegral = 0.0; + else if (sourceData->ratioIntegral * actualOffsetError <= 0.0) + sourceData->ratioIntegral = 0.0; const double candidateIntegral = acquiringDeviceClock ? 0.0 : - spiceData->ratioIntegral + + sourceData->ratioIntegral + (deviceClockBecameStable ? 0.0 : controllerError * periodSec); const double phaseCorrection = controllerKp * controllerError + @@ -1567,18 +1750,18 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) 1.0 - PLAYBACK_MAX_RATE_CORRECTION, 1.0 + PLAYBACK_MAX_RATE_CORRECTION); - if (!acquiringDeviceClock && spiceData->deviceClockStable && + if (!acquiringDeviceClock && sourceData->deviceClockStable && (desiredRatio == boundedRatio || (desiredRatio > boundedRatio && controllerError < 0.0) || (desiredRatio < boundedRatio && controllerError > 0.0))) - spiceData->ratioIntegral = candidateIntegral; + sourceData->ratioIntegral = candidateIntegral; const double maxRatioStep = PLAYBACK_MAX_RATE_SLEW_PER_SEC * periodSec; const double ratio = clamp(boundedRatio, - spiceData->lastRatio - maxRatioStep, - spiceData->lastRatio + maxRatioStep); - spiceData->lastRatio = ratio; + sourceData->lastRatio - maxRatioStep, + sourceData->lastRatio + maxRatioStep); + sourceData->lastRatio = ratio; if (audio.playback.backendResampler) { @@ -1586,8 +1769,8 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) &audio.playback.backendResampleRatio, ratio, memory_order_release); const int outputFrames = - playbackAppendFrames(spiceData, spiceData->framesIn, frames); - spiceData->outputPosition += outputFrames; + playbackAppendFrames(sourceData, sourceData->framesIn, frames); + sourceData->outputPosition += outputFrames; } else { @@ -1596,18 +1779,18 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) { SRC_DATA srcData = { - .data_in = spiceData->framesIn + + .data_in = sourceData->framesIn + consumed * audio.playback.channels, - .data_out = spiceData->framesOut, + .data_out = sourceData->framesOut, .input_frames = frames - consumed, - .output_frames = spiceData->framesOutSize, + .output_frames = sourceData->framesOutSize, .input_frames_used = 0, .output_frames_gen = 0, .end_of_input = 0, .src_ratio = ratio }; - int error = src_process(spiceData->src, &srcData); + int error = src_process(sourceData->src, &srcData); if (error) { DEBUG_ERROR("Resampling failed: %s", src_strerror(error)); @@ -1623,15 +1806,15 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) } const int outputFrames = playbackAppendFrames( - spiceData, spiceData->framesOut, srcData.output_frames_gen); + sourceData, sourceData->framesOut, srcData.output_frames_gen); consumed += srcData.input_frames_used; - spiceData->outputPosition += outputFrames; + sourceData->outputPosition += outputFrames; } } - spiceData->inputPosition += frames; + sourceData->inputPosition += frames; - if (playbackGetState() == STREAM_STATE_SETUP_SPICE) + if (playbackGetState() == STREAM_STATE_SETUP_SOURCE) { /* At a packet boundary, targetLowWaterFrames is the physical ring target; * sourceReserveFrames accounts for the packet's average delivery phase @@ -1696,10 +1879,10 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) app_invalidateGraph(audio.playback.graph); } - if (now >= spiceData->nextLogTime) + if (now >= sourceData->nextLogTime) { - const double sourcePpm = spiceData->sourceRateValid ? - (spiceData->sourceRateFrameSec / nominalFrameSec - 1.0) * 1.0e6 : + const double sourcePpm = sourceData->sourceRateValid ? + (sourceData->sourceRateFrameSec / nominalFrameSec - 1.0) * 1.0e6 : 0.0; const double devicePpm = rateClock->valid ? (rateClock->frameSec / nominalFrameSec - 1.0) * 1.0e6 : @@ -1713,24 +1896,57 @@ void audio_playbackData(uint8_t * data, size_t size, uint32_t time) softwareLatencyMs, targetLatencyFrames * 1000.0 / audio.playback.sampleRate, (ratio - 1.0) * 1.0e6, sourcePpm, devicePpm, - spiceData->arrivalJitterSec * 1000.0, - underruns, spiceData->bufferOverruns); + sourceData->arrivalJitterSec * 1000.0, + underruns, sourceData->bufferOverruns); - spiceData->bufferOverruns = 0; - spiceData->nextLogTime = now + INT64_C(5000000000); + sourceData->bufferOverruns = 0; + sourceData->nextLogTime = now + INT64_C(5000000000); } } -bool audio_supportsRecord(void) +static bool playbackGetFeedback(LG_AudioClock * clock) +{ + const PlaybackSourceData * sourceData = &audio.playback.sourceData; + if (!clock || !sourceData->deviceClock.valid || + sourceData->deviceClock.position < 0.0 || + sourceData->deviceClock.frameSec <= 0.0) + return false; + + const uint64_t latency = audio.audioDev->playback.latency ? + audio.audioDev->playback.latency() : 0; + + *clock = (LG_AudioClock) + { + .position = llrint(sourceData->deviceClock.position), + .time = sourceData->deviceClock.time + latency * 1000, + .rate = 1.0 / sourceData->deviceClock.frameSec, + .stable = sourceData->deviceClockStable, + }; + return true; +} + +bool lgAudio_supportsRecord(void) { return audio.audioDev && audio.audioDev->record.start; } static void recordPushFrames(uint8_t * data, int frames) { - const int stride = atomic_load_explicit( - &audio.record.stride, memory_order_acquire); - purespice_writeAudio(data, frames * stride, 0); + if (frames <= 0) + return; + + const uint32_t generation = atomic_load_explicit( + &audio.record.streamGeneration, memory_order_acquire); + if (!generation) + return; + + LG_LOCK_SHARED(audio.activeLock); + if (generation == atomic_load_explicit( + &audio.record.streamGeneration, memory_order_acquire) && + audio.active.ops && audio.active.ops->recordData) + audio.active.ops->recordData(audio.active.opaque, + generation, data, frames, NULL); + LG_UNLOCK_SHARED(audio.activeLock); } static MsgBoxHandle recordCancelConfirmLocked(void) @@ -1742,14 +1958,12 @@ static MsgBoxHandle recordCancelConfirmLocked(void) return handle; } -static void realRecordStartLocked( - int channels, int sampleRate, PSAudioFormat format) +static void realRecordStartLocked(const LG_AudioFormat * format) { audio.record.started = true; - atomic_store_explicit(&audio.record.stride, - channels * sizeof(uint16_t), memory_order_release); + audio.record.format = *format; - audio.audioDev->record.start(channels, sampleRate, recordPushFrames); + audio.audioDev->record.start(format, recordPushFrames); // if a volume level was stored, set it before we return if (audio.record.volumeChannels) @@ -1765,13 +1979,6 @@ static void realRecordStartLocked( app_showRecord(true); } -struct AudioFormat -{ - int channels; - int sampleRate; - PSAudioFormat format; -}; - static void recordConfirm(bool yes, void * opaque) { const uint64_t generation = (uint64_t)(uintptr_t)opaque; @@ -1791,10 +1998,7 @@ static void recordConfirm(bool yes, void * opaque) !audio.record.shuttingDown && audio.audioDev) { DEBUG_INFO("Microphone access granted"); - realRecordStartLocked( - audio.record.confirmChannels, - audio.record.confirmSampleRate, - audio.record.confirmFormat); + realRecordStartLocked(&audio.record.confirmFormat); } else if (yes) DEBUG_INFO("Ignoring stale microphone access confirmation"); @@ -1804,11 +2008,14 @@ static void recordConfirm(bool yes, void * opaque) LG_UNLOCK(audio.record.lock); } -void audio_recordStart(int channels, int sampleRate, PSAudioFormat format) +static void recordStart(const LG_AudioFormat * format) { LG_LOCK(audio.record.lock); - if (!audio.audioDev || audio.record.shuttingDown) + if (!audio.audioDev || audio.record.shuttingDown || + !audioFormatValid(format)) { + if (format && !audioFormatValid(format)) + DEBUG_ERROR("Invalid recording format"); LG_UNLOCK(audio.record.lock); return; } @@ -1816,9 +2023,7 @@ void audio_recordStart(int channels, int sampleRate, PSAudioFormat format) const bool restart = audio.record.started; if (audio.record.started) { - if (channels == audio.record.lastChannels && - sampleRate == audio.record.lastSampleRate && - format == audio.record.lastFormat) + if (audioFormatEqual(format, &audio.record.lastFormat)) { LG_UNLOCK(audio.record.lock); return; @@ -1828,25 +2033,21 @@ void audio_recordStart(int channels, int sampleRate, PSAudioFormat format) } MsgBoxHandle oldConfirm = recordCancelConfirmLocked(); - audio.record.requested = true; - audio.record.lastChannels = channels; - audio.record.lastSampleRate = sampleRate; - audio.record.lastFormat = format; + audio.record.requested = true; + audio.record.lastFormat = *format; if (restart) - realRecordStartLocked(channels, sampleRate, format); + realRecordStartLocked(format); else if (g_state.micDefaultState == MIC_DEFAULT_DENY) DEBUG_INFO("Microphone access denied by default"); else if (g_state.micDefaultState == MIC_DEFAULT_ALLOW) { DEBUG_INFO("Microphone access granted by default"); - realRecordStartLocked(channels, sampleRate, format); + realRecordStartLocked(format); } else { - audio.record.confirmChannels = channels; - audio.record.confirmSampleRate = sampleRate; - audio.record.confirmFormat = format; + audio.record.confirmFormat = *format; audio.record.confirmPending = true; const uint64_t generation = ++audio.record.confirmGeneration; LG_UNLOCK(audio.record.lock); @@ -1889,7 +2090,7 @@ static void realRecordStopLocked(void) app_showRecord(false); } -void audio_recordStop(void) +static void recordStop(void) { LG_LOCK(audio.record.lock); audio.record.requested = false; @@ -1905,7 +2106,7 @@ void audio_recordStop(void) app_msgBoxClose(confirm); } -void audio_recordToggleKeybind(int sc, void * opaque) +void lgAudio_recordToggleKeybind(int sc, void * opaque) { LG_LOCK(audio.record.lock); if (!audio.audioDev || audio.record.shuttingDown) @@ -1933,10 +2134,7 @@ void audio_recordToggleKeybind(int sc, void * opaque) else { DEBUG_INFO("Microphone recording started by user"); - realRecordStartLocked( - audio.record.lastChannels, - audio.record.lastSampleRate, - audio.record.lastFormat); + realRecordStartLocked(&audio.record.lastFormat); started = true; } LG_UNLOCK(audio.record.lock); @@ -1946,7 +2144,7 @@ void audio_recordToggleKeybind(int sc, void * opaque) started ? "Microphone enabled" : "Microphone disabled"); } -void audio_recordVolume(int channels, const uint16_t volume[]) +static void recordVolume(int channels, const uint16_t volume[]) { LG_LOCK(audio.record.lock); if (!audio.audioDev || !audio.audioDev->record.volume || @@ -1971,7 +2169,7 @@ void audio_recordVolume(int channels, const uint16_t volume[]) LG_UNLOCK(audio.record.lock); } -void audio_recordMute(bool mute) +static void recordMute(bool mute) { LG_LOCK(audio.record.lock); if (!audio.audioDev || !audio.audioDev->record.mute || @@ -1993,4 +2191,511 @@ void audio_recordMute(bool mute) LG_UNLOCK(audio.record.lock); } +static bool bindingActiveNL(const AudioBinding * binding) +{ + return binding->ops && + audio.active.ops == binding->ops && + audio.active.opaque == binding->opaque && + audio.active.generation == binding->generation; +} + +static void queueFeedback(const LG_AudioOps * ops, void * opaque, + uint32_t bindingGeneration, uint32_t generation, + const LG_AudioClock * clock) +{ + if (!audio.feedback.event || !audio.feedback.thread || !clock) + return; + + LG_LOCK(audio.feedback.lock); + audio.feedback.ops = ops; + audio.feedback.opaque = opaque; + audio.feedback.bindingGeneration = bindingGeneration; + audio.feedback.generation = generation; + audio.feedback.clock = *clock; + audio.feedback.pending = true; + LG_UNLOCK(audio.feedback.lock); + lgSignalEvent(audio.feedback.event); +} + +static void eventPlaybackStart(void * opaque, uint32_t generation, + const LG_AudioFormat * format, const LG_AudioClock * sourceClock) +{ + AudioBinding * binding = opaque; + + LG_LOCK_SHARED(audio.activeLock); + if (bindingActiveNL(binding)) + { + LG_LOCK(audio.playback.sourceLock); + atomic_store_explicit(&audio.playback.streamGeneration, + generation, memory_order_release); + playbackStart(format, sourceClock); + LG_UNLOCK(audio.playback.sourceLock); + } + LG_UNLOCK_SHARED(audio.activeLock); +} + +static void eventPlaybackStop(void * opaque, uint32_t generation) +{ + AudioBinding * binding = opaque; + + LG_LOCK_SHARED(audio.activeLock); + if (bindingActiveNL(binding)) + { + LG_LOCK(audio.playback.sourceLock); + if (atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + { + playbackSourceStop(); + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + } + LG_UNLOCK(audio.playback.sourceLock); + } + LG_UNLOCK_SHARED(audio.activeLock); +} + +static void eventPlaybackVolume(void * opaque, uint32_t generation, + uint8_t channels, const uint16_t volume[]) +{ + AudioBinding * binding = opaque; + + LG_LOCK_SHARED(audio.activeLock); + if (bindingActiveNL(binding)) + { + LG_LOCK(audio.playback.sourceLock); + if (volume && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + playbackVolume(channels, volume); + LG_UNLOCK(audio.playback.sourceLock); + } + LG_UNLOCK_SHARED(audio.activeLock); +} + +static void eventPlaybackMute(void * opaque, uint32_t generation, bool mute) +{ + AudioBinding * binding = opaque; + + LG_LOCK_SHARED(audio.activeLock); + if (bindingActiveNL(binding)) + { + LG_LOCK(audio.playback.sourceLock); + if (atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + playbackMute(mute); + LG_UNLOCK(audio.playback.sourceLock); + } + LG_UNLOCK_SHARED(audio.activeLock); +} + +static void eventPlaybackData(void * opaque, uint32_t generation, + const void * data, size_t frames, const LG_AudioClock * sourceClock) +{ + AudioBinding * binding = opaque; + const LG_AudioOps * ops; + void * providerOpaque; + + LG_LOCK_SHARED(audio.activeLock); + const bool active = bindingActiveNL(binding); + ops = active ? binding->ops : NULL; + providerOpaque = active ? binding->opaque : NULL; + if (active) + { + LG_LOCK(audio.playback.sourceLock); + if (atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + { + playbackData(data, frames, sourceClock); + + LG_AudioClock feedback; + if (ops->clockFeedback && playbackGetFeedback(&feedback)) + queueFeedback(ops, providerOpaque, binding->generation, + generation, &feedback); + } + LG_UNLOCK(audio.playback.sourceLock); + } + LG_UNLOCK_SHARED(audio.activeLock); +} + +static void eventRecordStart(void * opaque, uint32_t generation, + const LG_AudioFormat * format) +{ + AudioBinding * binding = opaque; + + LG_LOCK_SHARED(audio.activeLock); + const bool active = bindingActiveNL(binding) && + binding->ops->recordData; + if (active) + { + atomic_store_explicit(&audio.record.streamGeneration, + generation, memory_order_release); + recordStart(format); + } + LG_UNLOCK_SHARED(audio.activeLock); +} + +static void eventRecordStop(void * opaque, uint32_t generation) +{ + AudioBinding * binding = opaque; + + LG_LOCK_SHARED(audio.activeLock); + const bool active = bindingActiveNL(binding); + if (active && + atomic_load_explicit(&audio.record.streamGeneration, + memory_order_acquire) == generation) + { + atomic_store_explicit( + &audio.record.streamGeneration, 0, memory_order_release); + recordStop(); + } + LG_UNLOCK_SHARED(audio.activeLock); +} + +static void eventRecordVolume(void * opaque, uint32_t generation, + uint8_t channels, const uint16_t volume[]) +{ + AudioBinding * binding = opaque; + + LG_LOCK_SHARED(audio.activeLock); + const bool active = bindingActiveNL(binding); + if (active && volume && + atomic_load_explicit(&audio.record.streamGeneration, + memory_order_acquire) == generation) + recordVolume(channels, volume); + LG_UNLOCK_SHARED(audio.activeLock); +} + +static void eventRecordMute(void * opaque, uint32_t generation, bool mute) +{ + AudioBinding * binding = opaque; + + LG_LOCK_SHARED(audio.activeLock); + const bool active = bindingActiveNL(binding); + if (active && + atomic_load_explicit(&audio.record.streamGeneration, + memory_order_acquire) == generation) + recordMute(mute); + LG_UNLOCK_SHARED(audio.activeLock); +} + +static const LG_AudioEventOps eventOps = +{ + .playbackStart = eventPlaybackStart, + .playbackStop = eventPlaybackStop, + .playbackVolume = eventPlaybackVolume, + .playbackMute = eventPlaybackMute, + .playbackData = eventPlaybackData, + .recordStart = eventRecordStart, + .recordStop = eventRecordStop, + .recordVolume = eventRecordVolume, + .recordMute = eventRecordMute, +}; + +static bool validOps(const LG_AudioOps * ops) +{ + return ops && ops->name && ops->attach && ops->detach; +} + +static AudioBinding makeBinding(const LG_AudioOps * ops, void * opaque) +{ + return (AudioBinding) + { + .ops = ops, + .opaque = opaque, + .available = ops && !ops->setStatusListener, + .generation = 0, + }; +} + +static AudioBinding * nextBindingSlotNL(void) +{ + if (audio.transport.available) + return &audio.transport; + if (audio.fallback.available) + return &audio.fallback; + return NULL; +} + +static void stopStreams(void) +{ + LG_LOCK(audio.playback.sourceLock); + if (audio.audioDev) + playbackStop(); + atomic_store_explicit( + &audio.playback.streamGeneration, 0, memory_order_release); + LG_UNLOCK(audio.playback.sourceLock); + + atomic_store_explicit( + &audio.record.streamGeneration, 0, memory_order_release); + recordStop(); +} + +/* providerLock must be held. dropActive suppresses calls into an endpoint + * which has already disappeared. */ +static void updateActive(bool dropActive) +{ + for (;;) + { + LG_LOCK_EXCLUSIVE(audio.activeLock); + AudioBinding * slot = nextBindingSlotNL(); + AudioBinding next = slot ? *slot : (AudioBinding) { 0 }; + const AudioBinding old = audio.active; + if (old.ops == next.ops && old.opaque == next.opaque && + old.generation == next.generation) + { + audio.active = next; + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + return; + } + audio.active = (AudioBinding) { 0 }; + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + + if (old.ops && !dropActive) + old.ops->detach(old.opaque); + dropActive = false; + stopStreams(); + + LG_LOCK_EXCLUSIVE(audio.activeLock); + slot = nextBindingSlotNL(); + if (!slot) + { + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + DEBUG_INFO("Audio is unavailable"); + return; + } + next = *slot; + audio.active = next; + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + + if (next.ops->attach(next.opaque, &eventOps, slot)) + { + DEBUG_INFO("Using Audio: %s", next.ops->name); + return; + } + + next.ops->detach(next.opaque); + stopStreams(); + + LG_LOCK_EXCLUSIVE(audio.activeLock); + if (audio.active.ops == next.ops && + audio.active.opaque == next.opaque) + audio.active = (AudioBinding) { 0 }; + if (slot->ops == next.ops && slot->opaque == next.opaque) + slot->available = false; + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + + DEBUG_WARN("Failed to attach Audio provider: %s", next.ops->name); + } +} + +static void fallbackStatusChanged(void * opaque, + const LG_AudioStatus * status) +{ + if (!status) + return; + + LG_LOCK(audio.providerLock); + LG_LOCK_EXCLUSIVE(audio.activeLock); + const bool current = audio.fallback.ops && + audio.fallback.opaque == opaque; + if (current) + { + audio.fallback.available = status->available; + audio.fallback.generation = status->generation; + } + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + if (current) + updateActive(false); + LG_UNLOCK(audio.providerLock); +} + +static void transportStatusChanged(void * opaque, + const LG_AudioStatus * status) +{ + if (!status) + return; + + LG_LOCK(audio.providerLock); + LG_LOCK_EXCLUSIVE(audio.activeLock); + const bool current = audio.transport.ops && + audio.transport.opaque == opaque; + if (current) + { + audio.transport.available = status->available; + audio.transport.generation = status->generation; + } + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + if (current) + updateActive(false); + LG_UNLOCK(audio.providerLock); +} + +static void setBinding(AudioBinding * target, const LG_AudioOps * ops, + void * opaque, LG_AudioStatusFn statusFn) +{ + if (ops && !validOps(ops)) + { + DEBUG_ERROR("Invalid audio operations"); + ops = NULL; + opaque = NULL; + } + + LG_LOCK(audio.providerLock); + const AudioBinding old = *target; + LG_UNLOCK(audio.providerLock); + + if (old.ops && old.ops->setStatusListener) + old.ops->setStatusListener(old.opaque, NULL, NULL); + + const AudioBinding next = makeBinding(ops, opaque); + LG_LOCK(audio.providerLock); + LG_LOCK_EXCLUSIVE(audio.activeLock); + *target = next; + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + updateActive(false); + LG_UNLOCK(audio.providerLock); + + if (next.ops && next.ops->setStatusListener) + next.ops->setStatusListener(next.opaque, statusFn, next.opaque); +} + +static int feedbackThread(void * opaque) +{ + while (lgWaitEvent(audio.feedback.event, TIMEOUT_INFINITE)) + { + if (atomic_load_explicit( + &audio.feedback.stop, memory_order_acquire)) + break; + + const LG_AudioOps * ops; + void * providerOpaque; + uint32_t bindingGeneration; + uint32_t generation; + LG_AudioClock clock; + + LG_LOCK(audio.feedback.lock); + const bool pending = audio.feedback.pending; + ops = audio.feedback.ops; + providerOpaque = audio.feedback.opaque; + bindingGeneration = audio.feedback.bindingGeneration; + generation = audio.feedback.generation; + clock = audio.feedback.clock; + audio.feedback.pending = false; + LG_UNLOCK(audio.feedback.lock); + + if (!pending || !ops || !ops->clockFeedback) + continue; + + LG_LOCK_SHARED(audio.activeLock); + if (audio.active.ops == ops && + audio.active.opaque == providerOpaque && + audio.active.generation == bindingGeneration && + atomic_load_explicit(&audio.playback.streamGeneration, + memory_order_acquire) == generation) + ops->clockFeedback(providerOpaque, generation, &clock); + LG_UNLOCK_SHARED(audio.activeLock); + } + + return 0; +} + +void lgAudio_init(void) +{ + LG_LOCK_INIT(audio.providerLock); + LG_RWLOCK_INIT(audio.activeLock); + LG_LOCK_INIT(audio.playback.sourceLock); + LG_LOCK_INIT(audio.record.lock); + LG_LOCK_INIT(audio.feedback.lock); + audio.record.shuttingDown = false; + atomic_init(&audio.playback.streamGeneration, 0); + atomic_init(&audio.record.streamGeneration, 0); + atomic_init(&audio.feedback.stop, false); + atomic_store_explicit( + &audio.playback.callbackState, PLAYBACK_CALLBACK_DISABLED, + memory_order_release); + + audio.feedback.event = lgCreateEvent(true, 0); + if (audio.feedback.event && + !lgCreateThread("audioFeedback", feedbackThread, + NULL, &audio.feedback.thread)) + { + lgFreeEvent(audio.feedback.event); + audio.feedback.event = NULL; + } + + for (int i = 0; i < LG_AUDIODEV_COUNT; ++i) + if (LG_AudioDevs[i]->init()) + { + audio.audioDev = LG_AudioDevs[i]; + DEBUG_INFO("Using AudioDev: %s", audio.audioDev->name); + return; + } + + DEBUG_WARN("Failed to initialize an audio backend"); +} + +void lgAudio_free(void) +{ + lgAudio_setTransport(NULL, NULL); + lgAudio_setFallback(NULL, NULL); + stopStreams(); + + if (audio.feedback.thread) + { + atomic_store_explicit( + &audio.feedback.stop, true, memory_order_release); + lgSignalEvent(audio.feedback.event); + lgJoinThread(audio.feedback.thread, NULL); + audio.feedback.thread = NULL; + } + if (audio.feedback.event) + { + lgFreeEvent(audio.feedback.event); + audio.feedback.event = NULL; + } + + LG_LOCK(audio.record.lock); + audio.record.shuttingDown = true; + audio.record.requested = false; + MsgBoxHandle confirm = recordCancelConfirmLocked(); + struct LG_AudioDevOps * audioDev = audio.audioDev; + audio.audioDev = NULL; + LG_UNLOCK(audio.record.lock); + + app_msgBoxClose(confirm); + if (audioDev) + audioDev->free(); + + LG_RWLOCK_FREE(audio.activeLock); + LG_LOCK_FREE(audio.playback.sourceLock); + LG_LOCK_FREE(audio.providerLock); + LG_LOCK_FREE(audio.record.lock); + LG_LOCK_FREE(audio.feedback.lock); +} + +void lgAudio_setFallback(const LG_AudioOps * ops, void * opaque) +{ + setBinding(&audio.fallback, ops, opaque, fallbackStatusChanged); +} + +void lgAudio_setTransport(const LG_AudioOps * ops, void * opaque) +{ + setBinding(&audio.transport, ops, opaque, transportStatusChanged); +} + +void lgAudio_dropTransport(void) +{ + LG_LOCK(audio.providerLock); + LG_LOCK_EXCLUSIVE(audio.activeLock); + const AudioBinding old = audio.transport; + const bool wasActive = bindingActiveNL(&audio.transport); + audio.transport = (AudioBinding) { 0 }; + LG_UNLOCK_EXCLUSIVE(audio.activeLock); + updateActive(wasActive); + LG_UNLOCK(audio.providerLock); + + if (old.ops && old.ops->setStatusListener) + old.ops->setStatusListener(old.opaque, NULL, NULL); +} + #endif diff --git a/client/src/audio.h b/client/src/audio.h index 9a1d2552..f79f759c 100644 --- a/client/src/audio.h +++ b/client/src/audio.h @@ -18,32 +18,37 @@ * Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#ifndef _H_LG_CLIENT_AUDIO_ +#define _H_LG_CLIENT_AUDIO_ + +#include "interface/audio.h" + #if ENABLE_AUDIO -#include -#include +void lgAudio_init(void); +void lgAudio_free(void); -void audio_init(void); -void audio_free(void); +void lgAudio_setFallback(const LG_AudioOps * ops, void * opaque); +void lgAudio_setTransport(const LG_AudioOps * ops, void * opaque); +void lgAudio_dropTransport(void); -bool audio_supportsPlayback(void); -void audio_playbackStart(int channels, int sampleRate, PSAudioFormat format, - uint32_t time); -void audio_playbackStop(void); -void audio_playbackVolume(int channels, const uint16_t volume[]); -void audio_playbackMute(bool mute); -void audio_playbackData(uint8_t * data, size_t size, uint32_t time); - -bool audio_supportsRecord(void); -void audio_recordStart(int channels, int sampleRate, PSAudioFormat format); -void audio_recordToggleKeybind(int sc, void * opaque); -void audio_recordStop(void); -void audio_recordVolume(int channels, const uint16_t volume[]); -void audio_recordMute(bool mute); +bool lgAudio_supportsPlayback(void); +bool lgAudio_supportsRecord(void); +void lgAudio_recordToggleKeybind(int sc, void * opaque); #else -static inline void audio_init(void) {} -static inline void audio_free(void) {} +static inline void lgAudio_init(void) {} +static inline void lgAudio_free(void) {} +static inline void lgAudio_setFallback( + const LG_AudioOps * ops, void * opaque) {} +static inline void lgAudio_setTransport( + const LG_AudioOps * ops, void * opaque) {} +static inline void lgAudio_dropTransport(void) {} +static inline bool lgAudio_supportsPlayback(void) { return false; } +static inline bool lgAudio_supportsRecord(void) { return false; } +static inline void lgAudio_recordToggleKeybind(int sc, void * opaque) {} + +#endif #endif diff --git a/client/src/audio_spice.c b/client/src/audio_spice.c new file mode 100644 index 00000000..e7a9f719 --- /dev/null +++ b/client/src/audio_spice.c @@ -0,0 +1,842 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "audio_spice.h" + +#include "common/debug.h" +#include "common/locking.h" + +#include +#include + +#define SPICE_AUDIO_TIMESTAMP_DISCONTINUITY_MS 2000 + +typedef struct SpiceAudioStream +{ + bool active; + uint32_t generation; + LG_AudioFormat format; + + bool volumeValid; + uint8_t volumeChannels; + uint16_t volume[LG_AUDIO_MAX_CHANNELS]; + bool muteValid; + bool mute; +} +SpiceAudioStream; + +typedef struct SpiceAudioEventTarget +{ + const LG_AudioEventOps * events; + void * opaque; + uint32_t generation; +} +SpiceAudioEventTarget; + +static struct +{ + LG_RWLock lock; + + bool available; + uint32_t statusGeneration; + LG_AudioStatusFn statusCallback; + void * statusOpaque; + atomic_uint statusInFlight; + + const LG_AudioEventOps * events; + void * eventOpaque; + uint32_t eventGeneration; + atomic_uint inFlight; + + SpiceAudioStream playback; + SpiceAudioStream record; + + bool playbackClockValid; + uint32_t playbackMediaTime; + int64_t playbackTime; + uint64_t playbackPosition; + LG_AudioClock playbackClock; +} +l_spice = +{ + .lock = + { + .readers = ATOMIC_VAR_INIT(0), + .writers = ATOMIC_VAR_INIT(0), + .writer = ATOMIC_FLAG_INIT, + }, + .statusInFlight = ATOMIC_VAR_INIT(0), + .inFlight = ATOMIC_VAR_INIT(0), +}; + +static _Thread_local unsigned int l_eventDepth; +static _Thread_local unsigned int l_statusDepth; + +static uint32_t nextGeneration(uint32_t generation) +{ + if (++generation == 0) + ++generation; + return generation; +} + +/* l_spice.lock must be held exclusively while admitting a callback so detach + * cannot invalidate the target between the snapshot and in-flight increment. */ +static bool beginEventNL(SpiceAudioEventTarget * target) +{ + if (!l_spice.events) + return false; + + target->events = l_spice.events; + target->opaque = l_spice.eventOpaque; + target->generation = l_spice.eventGeneration; + atomic_fetch_add_explicit(&l_spice.inFlight, 1, memory_order_relaxed); + ++l_eventDepth; + return true; +} + +static bool playbackEventCurrent(const SpiceAudioEventTarget * target, + uint32_t generation, bool active) +{ + LG_LOCK_SHARED(l_spice.lock); + const bool result = + target->events == l_spice.events && + target->opaque == l_spice.eventOpaque && + target->generation == l_spice.eventGeneration && + l_spice.playback.generation == generation && + l_spice.playback.active == active; + LG_UNLOCK_SHARED(l_spice.lock); + return result; +} + +static bool recordEventCurrent(const SpiceAudioEventTarget * target, + uint32_t generation, bool active) +{ + LG_LOCK_SHARED(l_spice.lock); + const bool result = + target->events == l_spice.events && + target->opaque == l_spice.eventOpaque && + target->generation == l_spice.eventGeneration && + l_spice.record.generation == generation && + l_spice.record.active == active; + LG_UNLOCK_SHARED(l_spice.lock); + return result; +} + +static void endEvent(void) +{ + --l_eventDepth; + atomic_fetch_sub_explicit(&l_spice.inFlight, 1, memory_order_release); +} + +static bool sampleFormat(PSAudioFormat source, + LG_AudioSampleFormat * format) +{ + switch (source) + { + case PS_AUDIO_FMT_S16: + *format = LG_AUDIO_FMT_S16_LE; + return true; + + default: + return false; + } +} + +static void channelLayout(LG_AudioFormat * format) +{ + memset(format->channels, 0, sizeof(format->channels)); + + switch (format->channelCount) + { + case 1: + format->channels[0] = LG_AUDIO_CH_MONO; + break; + + case 2: + format->channels[0] = LG_AUDIO_CH_FRONT_LEFT; + format->channels[1] = LG_AUDIO_CH_FRONT_RIGHT; + break; + + case 3: + format->channels[0] = LG_AUDIO_CH_FRONT_LEFT; + format->channels[1] = LG_AUDIO_CH_FRONT_RIGHT; + format->channels[2] = LG_AUDIO_CH_FRONT_CENTER; + break; + + case 4: + format->channels[0] = LG_AUDIO_CH_FRONT_LEFT; + format->channels[1] = LG_AUDIO_CH_FRONT_RIGHT; + format->channels[2] = LG_AUDIO_CH_REAR_LEFT; + format->channels[3] = LG_AUDIO_CH_REAR_RIGHT; + break; + + case 5: + format->channels[0] = LG_AUDIO_CH_FRONT_LEFT; + format->channels[1] = LG_AUDIO_CH_FRONT_RIGHT; + format->channels[2] = LG_AUDIO_CH_FRONT_CENTER; + format->channels[3] = LG_AUDIO_CH_REAR_LEFT; + format->channels[4] = LG_AUDIO_CH_REAR_RIGHT; + break; + + case 6: + format->channels[0] = LG_AUDIO_CH_FRONT_LEFT; + format->channels[1] = LG_AUDIO_CH_FRONT_RIGHT; + format->channels[2] = LG_AUDIO_CH_FRONT_CENTER; + format->channels[3] = LG_AUDIO_CH_LFE; + format->channels[4] = LG_AUDIO_CH_REAR_LEFT; + format->channels[5] = LG_AUDIO_CH_REAR_RIGHT; + break; + + case 7: + format->channels[0] = LG_AUDIO_CH_FRONT_LEFT; + format->channels[1] = LG_AUDIO_CH_FRONT_RIGHT; + format->channels[2] = LG_AUDIO_CH_FRONT_CENTER; + format->channels[3] = LG_AUDIO_CH_LFE; + format->channels[4] = LG_AUDIO_CH_REAR_LEFT; + format->channels[5] = LG_AUDIO_CH_REAR_RIGHT; + format->channels[6] = LG_AUDIO_CH_REAR_CENTER; + break; + + case 8: + format->channels[0] = LG_AUDIO_CH_FRONT_LEFT; + format->channels[1] = LG_AUDIO_CH_FRONT_RIGHT; + format->channels[2] = LG_AUDIO_CH_FRONT_CENTER; + format->channels[3] = LG_AUDIO_CH_LFE; + format->channels[4] = LG_AUDIO_CH_REAR_LEFT; + format->channels[5] = LG_AUDIO_CH_REAR_RIGHT; + format->channels[6] = LG_AUDIO_CH_SIDE_LEFT; + format->channels[7] = LG_AUDIO_CH_SIDE_RIGHT; + break; + + default: + break; + } +} + +static bool makeFormat(int channels, int sampleRate, PSAudioFormat source, + LG_AudioFormat * format) +{ + if (channels < 1 || channels > LG_AUDIO_MAX_CHANNELS || sampleRate < 1 || + !sampleFormat(source, &format->sampleFormat)) + return false; + + format->sampleRate = sampleRate; + format->channelCount = channels; + channelLayout(format); + return true; +} + +static size_t sampleSize(LG_AudioSampleFormat format) +{ + switch (format) + { + case LG_AUDIO_FMT_U8: return 1; + case LG_AUDIO_FMT_S16_LE: return 2; + case LG_AUDIO_FMT_S24_LE: return 3; + case LG_AUDIO_FMT_S32_LE: return 4; + case LG_AUDIO_FMT_F32_LE: return 4; + case LG_AUDIO_FMT_F64_LE: return 8; + } + + return 0; +} + +static LG_AudioClock playbackClockNL(uint32_t time) +{ + bool discontinuity = false; + + if (!l_spice.playbackClockValid) + { + l_spice.playbackClockValid = true; + l_spice.playbackMediaTime = time; + l_spice.playbackTime = 0; + } + else + { + const int32_t delta = (int32_t)(time - l_spice.playbackMediaTime); + l_spice.playbackMediaTime = time; + if (delta < 0 || delta > SPICE_AUDIO_TIMESTAMP_DISCONTINUITY_MS) + { + l_spice.playbackTime = 0; + discontinuity = true; + } + else + l_spice.playbackTime += (int64_t)delta * 1000000; + } + + l_spice.playbackClock = (LG_AudioClock) + { + .position = l_spice.playbackPosition, + .time = l_spice.playbackTime, + .rate = 0.0, + .stable = !discontinuity, + .discontinuity = discontinuity, + }; + return l_spice.playbackClock; +} + +static void spiceSetStatusListener(void * opaque, + LG_AudioStatusFn callback, void * callbackOpaque) +{ + LG_AudioStatus status; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + l_spice.statusCallback = callback; + l_spice.statusOpaque = callbackOpaque; + status = (LG_AudioStatus) + { + .available = l_spice.available, + .generation = l_spice.statusGeneration, + }; + if (callback) + { + atomic_fetch_add_explicit( + &l_spice.statusInFlight, 1, memory_order_relaxed); + ++l_statusDepth; + } + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (callback) + { + callback(callbackOpaque, &status); + --l_statusDepth; + atomic_fetch_sub_explicit( + &l_spice.statusInFlight, 1, memory_order_release); + } + else + while (atomic_load_explicit( + &l_spice.statusInFlight, memory_order_acquire) > l_statusDepth) + ; +} + +static bool spiceAttach(void * opaque, const LG_AudioEventOps * events, + void * eventOpaque) +{ + if (!events) + return false; + + SpiceAudioEventTarget target; + SpiceAudioStream playback; + SpiceAudioStream record; + LG_AudioClock playbackClock; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.available) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return false; + } + + l_spice.events = events; + l_spice.eventOpaque = eventOpaque; + l_spice.eventGeneration = + nextGeneration(l_spice.eventGeneration); + playback = l_spice.playback; + record = l_spice.record; + playbackClock = l_spice.playbackClock; + dispatch = beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return true; + + if (playback.active) + { + if (playbackEventCurrent(&target, playback.generation, true) && + target.events->playbackStart) + { + target.events->playbackStart(target.opaque, + playback.generation, &playback.format, &playbackClock); + if (playback.volumeValid && + playbackEventCurrent(&target, playback.generation, true) && + target.events->playbackVolume) + target.events->playbackVolume(target.opaque, + playback.generation, + playback.volumeChannels, playback.volume); + if (playback.muteValid && + playbackEventCurrent(&target, playback.generation, true) && + target.events->playbackMute) + target.events->playbackMute(target.opaque, + playback.generation, playback.mute); + } + } + + if (record.active) + { + if (recordEventCurrent(&target, record.generation, true) && + target.events->recordStart) + { + target.events->recordStart(target.opaque, + record.generation, &record.format); + if (record.volumeValid && + recordEventCurrent(&target, record.generation, true) && + target.events->recordVolume) + target.events->recordVolume(target.opaque, record.generation, + record.volumeChannels, record.volume); + if (record.muteValid && + recordEventCurrent(&target, record.generation, true) && + target.events->recordMute) + target.events->recordMute(target.opaque, + record.generation, record.mute); + } + } + + endEvent(); + return true; +} + +static void spiceDetach(void * opaque) +{ + LG_LOCK_EXCLUSIVE(l_spice.lock); + l_spice.events = NULL; + l_spice.eventOpaque = NULL; + l_spice.eventGeneration = + nextGeneration(l_spice.eventGeneration); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + while (atomic_load_explicit( + &l_spice.inFlight, memory_order_acquire) > l_eventDepth) + ; +} + +static bool spiceRecordData(void * opaque, uint32_t generation, + const void * data, size_t frames, const LG_AudioClock * sourceClock) +{ + size_t size = 0; + bool valid = false; + + LG_LOCK_SHARED(l_spice.lock); + if (l_spice.available && l_spice.events && l_spice.record.active && + generation == l_spice.record.generation) + { + const size_t bytesPerSample = + sampleSize(l_spice.record.format.sampleFormat); + const size_t channels = l_spice.record.format.channelCount; + if (bytesPerSample && frames <= SIZE_MAX / bytesPerSample / channels && + (frames == 0 || data)) + { + size = frames * bytesPerSample * channels; + valid = true; + } + } + LG_UNLOCK_SHARED(l_spice.lock); + + return valid && purespice_writeAudio((void *)data, size, 0); +} + +const LG_AudioOps LGA_Spice = +{ + .name = "SPICE", + .setStatusListener = spiceSetStatusListener, + .attach = spiceAttach, + .detach = spiceDetach, + .recordData = spiceRecordData, + .clockFeedback = NULL, +}; + +void lgaSpice_setAvailable(bool available) +{ + LG_AudioStatusFn callback; + void * callbackOpaque; + LG_AudioStatus status; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (l_spice.available == available) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + l_spice.available = available; + l_spice.statusGeneration = + nextGeneration(l_spice.statusGeneration); + if (!available) + { + l_spice.playback.active = false; + l_spice.record.active = false; + l_spice.playbackClockValid = false; + l_spice.playbackPosition = 0; + l_spice.playbackClock = (LG_AudioClock) { 0 }; + l_spice.playback.volumeValid = false; + l_spice.playback.muteValid = false; + l_spice.record.volumeValid = false; + l_spice.record.muteValid = false; + } + + callback = l_spice.statusCallback; + callbackOpaque = l_spice.statusOpaque; + status = (LG_AudioStatus) + { + .available = available, + .generation = l_spice.statusGeneration, + }; + if (callback) + { + atomic_fetch_add_explicit( + &l_spice.statusInFlight, 1, memory_order_relaxed); + ++l_statusDepth; + } + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (callback) + { + callback(callbackOpaque, &status); + --l_statusDepth; + atomic_fetch_sub_explicit( + &l_spice.statusInFlight, 1, memory_order_release); + } +} + +void lgaSpice_playbackStart(int channels, int sampleRate, + PSAudioFormat sourceFormat, uint32_t time) +{ + LG_AudioFormat format; + if (!makeFormat(channels, sampleRate, sourceFormat, &format)) + { + DEBUG_ERROR("Invalid SPICE playback format: %d channels, %d Hz, %d", + channels, sampleRate, sourceFormat); + return; + } + + SpiceAudioEventTarget target; + SpiceAudioStream playback; + LG_AudioClock clock; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.available) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + l_spice.playback.active = true; + l_spice.playback.generation = + nextGeneration(l_spice.playback.generation); + l_spice.playback.format = format; + l_spice.playbackClockValid = false; + l_spice.playbackPosition = 0; + clock = playbackClockNL(time); + clock.stable = false; + clock.discontinuity = true; + l_spice.playbackClock = clock; + playback = l_spice.playback; + dispatch = beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (playbackEventCurrent(&target, playback.generation, true) && + target.events->playbackStart) + { + target.events->playbackStart(target.opaque, + playback.generation, &playback.format, &clock); + if (playback.volumeValid && + playbackEventCurrent(&target, playback.generation, true) && + target.events->playbackVolume) + target.events->playbackVolume(target.opaque, + playback.generation, playback.volumeChannels, playback.volume); + if (playback.muteValid && + playbackEventCurrent(&target, playback.generation, true) && + target.events->playbackMute) + target.events->playbackMute(target.opaque, + playback.generation, playback.mute); + } + endEvent(); +} + +void lgaSpice_playbackStop(void) +{ + SpiceAudioEventTarget target; + uint32_t generation; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.playback.active) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + generation = l_spice.playback.generation; + l_spice.playback.active = false; + l_spice.playbackClockValid = false; + dispatch = beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (target.events->playbackStop && + playbackEventCurrent(&target, generation, false)) + target.events->playbackStop(target.opaque, generation); + endEvent(); +} + +void lgaSpice_playbackVolume(int channels, const uint16_t volume[]) +{ + if (channels < 1 || channels > LG_AUDIO_MAX_CHANNELS || !volume) + return; + + SpiceAudioEventTarget target; + uint32_t generation; + uint16_t snapshot[LG_AUDIO_MAX_CHANNELS]; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.available) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + memcpy(l_spice.playback.volume, volume, + (size_t)channels * sizeof(*volume)); + l_spice.playback.volumeChannels = channels; + l_spice.playback.volumeValid = true; + generation = l_spice.playback.generation; + memcpy(snapshot, volume, (size_t)channels * sizeof(*volume)); + dispatch = l_spice.playback.active && beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (target.events->playbackVolume && + playbackEventCurrent(&target, generation, true)) + target.events->playbackVolume( + target.opaque, generation, channels, snapshot); + endEvent(); +} + +void lgaSpice_playbackMute(bool mute) +{ + SpiceAudioEventTarget target; + uint32_t generation; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.available) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + l_spice.playback.mute = mute; + l_spice.playback.muteValid = true; + generation = l_spice.playback.generation; + dispatch = l_spice.playback.active && beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (target.events->playbackMute && + playbackEventCurrent(&target, generation, true)) + target.events->playbackMute(target.opaque, generation, mute); + endEvent(); +} + +void lgaSpice_playbackData(uint8_t * data, size_t size, uint32_t time) +{ + SpiceAudioEventTarget target; + uint32_t generation; + size_t frames; + LG_AudioClock clock; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.available || !l_spice.playback.active) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + const size_t bytesPerSample = + sampleSize(l_spice.playback.format.sampleFormat); + const size_t stride = + bytesPerSample * l_spice.playback.format.channelCount; + if (!stride || !size || !data || size % stride) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + DEBUG_ERROR("Invalid SPICE playback packet size: %zu", size); + return; + } + + frames = size / stride; + generation = l_spice.playback.generation; + clock = playbackClockNL(time); + l_spice.playbackPosition += frames; + dispatch = beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (target.events->playbackData && + playbackEventCurrent(&target, generation, true)) + target.events->playbackData(target.opaque, generation, + data, frames, &clock); + endEvent(); +} + +void lgaSpice_recordStart(int channels, int sampleRate, + PSAudioFormat sourceFormat) +{ + LG_AudioFormat format; + if (!makeFormat(channels, sampleRate, sourceFormat, &format)) + { + DEBUG_ERROR("Invalid SPICE record format: %d channels, %d Hz, %d", + channels, sampleRate, sourceFormat); + return; + } + + SpiceAudioEventTarget target; + SpiceAudioStream record; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.available) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + l_spice.record.active = true; + l_spice.record.generation = nextGeneration(l_spice.record.generation); + l_spice.record.format = format; + record = l_spice.record; + dispatch = beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (recordEventCurrent(&target, record.generation, true) && + target.events->recordStart) + { + target.events->recordStart( + target.opaque, record.generation, &record.format); + if (record.volumeValid && + recordEventCurrent(&target, record.generation, true) && + target.events->recordVolume) + target.events->recordVolume(target.opaque, record.generation, + record.volumeChannels, record.volume); + if (record.muteValid && + recordEventCurrent(&target, record.generation, true) && + target.events->recordMute) + target.events->recordMute( + target.opaque, record.generation, record.mute); + } + endEvent(); +} + +void lgaSpice_recordStop(void) +{ + SpiceAudioEventTarget target; + uint32_t generation; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.record.active) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + generation = l_spice.record.generation; + l_spice.record.active = false; + dispatch = beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (target.events->recordStop && + recordEventCurrent(&target, generation, false)) + target.events->recordStop(target.opaque, generation); + endEvent(); +} + +void lgaSpice_recordVolume(int channels, const uint16_t volume[]) +{ + if (channels < 1 || channels > LG_AUDIO_MAX_CHANNELS || !volume) + return; + + SpiceAudioEventTarget target; + uint32_t generation; + uint16_t snapshot[LG_AUDIO_MAX_CHANNELS]; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.available) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + memcpy(l_spice.record.volume, volume, + (size_t)channels * sizeof(*volume)); + l_spice.record.volumeChannels = channels; + l_spice.record.volumeValid = true; + generation = l_spice.record.generation; + memcpy(snapshot, volume, (size_t)channels * sizeof(*volume)); + dispatch = l_spice.record.active && beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (target.events->recordVolume && + recordEventCurrent(&target, generation, true)) + target.events->recordVolume( + target.opaque, generation, channels, snapshot); + endEvent(); +} + +void lgaSpice_recordMute(bool mute) +{ + SpiceAudioEventTarget target; + uint32_t generation; + bool dispatch; + + LG_LOCK_EXCLUSIVE(l_spice.lock); + if (!l_spice.available) + { + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + return; + } + + l_spice.record.mute = mute; + l_spice.record.muteValid = true; + generation = l_spice.record.generation; + dispatch = l_spice.record.active && beginEventNL(&target); + LG_UNLOCK_EXCLUSIVE(l_spice.lock); + + if (!dispatch) + return; + + if (target.events->recordMute && + recordEventCurrent(&target, generation, true)) + target.events->recordMute(target.opaque, generation, mute); + endEvent(); +} diff --git a/client/src/audio_spice.h b/client/src/audio_spice.h new file mode 100644 index 00000000..55dfbe41 --- /dev/null +++ b/client/src/audio_spice.h @@ -0,0 +1,45 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef _H_LG_CLIENT_AUDIO_SPICE_ +#define _H_LG_CLIENT_AUDIO_SPICE_ + +#include "interface/audio.h" + +#include + +extern const LG_AudioOps LGA_Spice; + +void lgaSpice_setAvailable(bool available); + +void lgaSpice_playbackStart(int channels, int sampleRate, + PSAudioFormat format, uint32_t time); +void lgaSpice_playbackStop(void); +void lgaSpice_playbackVolume(int channels, const uint16_t volume[]); +void lgaSpice_playbackMute(bool mute); +void lgaSpice_playbackData(uint8_t * data, size_t size, uint32_t time); + +void lgaSpice_recordStart(int channels, int sampleRate, + PSAudioFormat format); +void lgaSpice_recordStop(void); +void lgaSpice_recordVolume(int channels, const uint16_t volume[]); +void lgaSpice_recordMute(bool mute); + +#endif diff --git a/client/src/keybind.c b/client/src/keybind.c index e28bb30e..cbf37e90 100644 --- a/client/src/keybind.c +++ b/client/src/keybind.c @@ -207,9 +207,9 @@ void keybind_inputRegister(void) "Send RWin to the guest"); #if ENABLE_AUDIO - if (audio_supportsRecord()) + if (lgAudio_supportsRecord()) { - app_registerKeybind(KEY_E, audio_recordToggleKeybind, NULL, + app_registerKeybind(KEY_E, lgAudio_recordToggleKeybind, NULL, "Toggle audio recording"); app_registerKeybind(KEY_C, bind_toggleMicDefault, NULL, "Cycle audio recording default"); diff --git a/client/src/main.c b/client/src/main.c index b7e7519f..cc549bb5 100644 --- a/client/src/main.c +++ b/client/src/main.c @@ -58,6 +58,9 @@ #include "core.h" #include "app.h" #include "audio.h" +#if ENABLE_AUDIO +#include "audio_spice.h" +#endif #include "keybind.h" #include "clipboard.h" #include "kb.h" @@ -1025,9 +1028,15 @@ static int renderThread(void * unused) if (g_state.transport && g_state.transportOps->sessionValid(g_state.transport)) + { lgInput_setTransport(NULL, NULL); + lgAudio_setTransport(NULL, NULL); + } else + { lgInput_dropTransport(); + lgAudio_dropTransport(); + } core_stopCursorThread(); core_stopFrameThread(); @@ -1075,7 +1084,10 @@ int main_cursorThread(void * unused) } if (status == LG_TRANSPORT_DISCONNECTED) + { lgInput_dropTransport(); + lgAudio_dropTransport(); + } app_setState(status == LG_TRANSPORT_DISCONNECTED ? APP_STATE_RESTART : APP_STATE_SHUTDOWN); if (status != LG_TRANSPORT_DISCONNECTED) @@ -1216,6 +1228,7 @@ int main_frameThread(void * unused) if (status == LG_TRANSPORT_DISCONNECTED) { lgInput_dropTransport(); + lgAudio_dropTransport(); app_setState(APP_STATE_RESTART); } else if (status == LG_TRANSPORT_END) @@ -1503,6 +1516,13 @@ void spiceReady(void) atomic_store_explicit(&g_state.spiceReady, true, memory_order_release); if (g_params.useSpiceInput) lgInput_setFallback(&LGI_Spice, NULL); +#if ENABLE_AUDIO + if (g_params.useSpiceAudio) + { + lgaSpice_setAvailable(true); + lgAudio_setFallback(&LGA_Spice, NULL); + } +#endif if (atomic_load_explicit(&g_state.spiceDisplayRequested, memory_order_acquire)) @@ -1687,9 +1707,6 @@ static void spice_setCursorState(bool visible, int x, int y) int spiceThread(void * arg) { - if (g_params.useSpiceAudio) - audio_init(); - const struct PSConfig config = { .host = g_params.spiceHost, @@ -1730,22 +1747,22 @@ int spiceThread(void * arg) #if ENABLE_AUDIO .playback = { - .enable = audio_supportsPlayback(), + .enable = g_params.useSpiceAudio && lgAudio_supportsPlayback(), .autoConnect = true, - .start = audio_playbackStart, - .volume = audio_playbackVolume, - .mute = audio_playbackMute, - .stop = audio_playbackStop, - .data = audio_playbackData + .start = lgaSpice_playbackStart, + .volume = lgaSpice_playbackVolume, + .mute = lgaSpice_playbackMute, + .stop = lgaSpice_playbackStop, + .data = lgaSpice_playbackData }, .record = { - .enable = audio_supportsRecord(), + .enable = g_params.useSpiceAudio && lgAudio_supportsRecord(), .autoConnect = true, - .start = audio_recordStart, - .volume = audio_recordVolume, - .mute = audio_recordMute, - .stop = audio_recordStop + .start = lgaSpice_recordStart, + .volume = lgaSpice_recordVolume, + .mute = lgaSpice_recordMute, + .stop = lgaSpice_recordStop } #endif }; @@ -1770,12 +1787,19 @@ int spiceThread(void * arg) } lgInput_setFallback(NULL, NULL); + lgAudio_setFallback(NULL, NULL); +#if ENABLE_AUDIO + lgaSpice_setAvailable(false); +#endif purespice_disconnect(); end: lgInput_setFallback(NULL, NULL); - audio_free(); + lgAudio_setFallback(NULL, NULL); +#if ENABLE_AUDIO + lgaSpice_setAvailable(false); +#endif // if the connection was disconnected intentionally we don't want to shutdown // so that the user can see the message box and take action @@ -1886,6 +1910,7 @@ static int lg_run(void) LG_LOCK_INIT(l_cursorRepaint.lock); frameTimingInit(); lgInput_init(); + lgAudio_init(); #ifdef ENABLE_TESTS memset(&l_testCapture, 0, sizeof(l_testCapture)); @@ -2356,6 +2381,11 @@ restart: g_state.transportOps->getInputOps(g_state.transport, &inputOpaque) : NULL; lgInput_setTransport(inputOps, inputOpaque); + void * audioOpaque = NULL; + const LG_AudioOps * audioOps = g_state.transportOps->getAudioOps ? + g_state.transportOps->getAudioOps(g_state.transport, &audioOpaque) : NULL; + lgAudio_setTransport(audioOps, audioOpaque); + if (inputOps || lgInput_available()) keybind_inputRegister(); checkUUID(); @@ -2373,6 +2403,7 @@ restart: if (unlikely(!g_state.transportOps->sessionValid(g_state.transport))) { lgInput_dropTransport(); + lgAudio_dropTransport(); atomic_store_explicit( &g_state.lgHostConnected, false, memory_order_release); DEBUG_INFO("Waiting for the host to restart..."); @@ -2394,6 +2425,7 @@ restart: core_stopFrameThread(); core_stopCursorThread(); lgInput_dropTransport(); + lgAudio_dropTransport(); g_state.transportOps->disconnect(g_state.transport); app_setState(APP_STATE_RUNNING); @@ -2440,10 +2472,16 @@ static void lg_shutdown(void) if (g_state.transportOps) { lgInput_dropTransport(); + if (g_state.transport && + g_state.transportOps->sessionValid(g_state.transport)) + lgAudio_setTransport(NULL, NULL); + else + lgAudio_dropTransport(); g_state.transportOps->destroy(&g_state.transport); } lgInput_free(); + lgAudio_free(); if (g_state.frameEvent) {