[client] audio: add optional LGMP transport

client/tests/CMakeLists.txt

@@ -41,6 +41,11 @@ target_link_libraries(lgmp-transport-tests
   lg_common
   lgmp
 )
+if(ENABLE_AUDIO)
+  target_compile_definitions(lgmp-transport-tests PRIVATE
+    ENABLE_AUDIO
+  )
+endif()
 add_test(NAME lgmp-transport-tests
   COMMAND lgmp-transport-tests
 )
@@ -141,6 +146,31 @@ foreach(name IN LISTS LGMP_CLIPBOARD_CASES)
   )
 endforeach()
 
+if(ENABLE_AUDIO)
+  add_executable(lgmp-audio-tests
+    lgmp_audio_test.c
+    ../src/kb_hid.c
+  )
+  target_compile_definitions(lgmp-audio-tests PRIVATE
+    ENABLE_AUDIO
+  )
+  target_include_directories(lgmp-audio-tests PRIVATE
+    "${CMAKE_CURRENT_SOURCE_DIR}/../transports/LGMP"
+  )
+  target_link_libraries(lgmp-audio-tests
+    ${EXE_FLAGS}
+    transport_LGMP
+    lg_common
+    lgmp
+  )
+  add_test(NAME lgmp-audio
+    COMMAND lgmp-audio-tests
+  )
+  set_tests_properties(lgmp-audio PROPERTIES
+    TIMEOUT 20
+  )
+endif()
+
 add_executable(wayland-motion-tests
   motion_test.c
 )

client/tests/lgmp_transport_test.c

@@ -21,6 +21,9 @@
 #include "interface/transport.h"
 
 #include <LGProtocol/KVMFR.h>
+#ifdef ENABLE_AUDIO
+#include "common/KVMFRAudio.h"
+#endif
 #include <LGProtocol/KVMFRRecovery.h>
 #include <LGProtocol/LGMPConfig.h>
 #include "common/debug.h"
@@ -96,6 +99,42 @@ static bool waitForQueuesEmpty(PLGMPHost host, PLGMPHostQueue frameQueue,
 
 struct VideoStatusTrace;
 
+#ifdef ENABLE_AUDIO
+struct AudioStatusTrace
+{
+  atomic_uint count;
+  atomic_bool available;
+};
+
+static void audioStatusChanged(void * opaque,
+    const LG_AudioStatus * status)
+{
+  struct AudioStatusTrace * trace = opaque;
+  atomic_store_explicit(
+      &trace->available, status->available, memory_order_relaxed);
+  atomic_fetch_add_explicit(&trace->count, 1, memory_order_release);
+}
+
+static bool readAudioClaim(PLGMPHost host, PLGMPHostQueue queue,
+    KVMFRAudioClaim * claim, uint32_t * clientID)
+{
+  for (unsigned i = 0; i < WAIT_TIMEOUT; ++i)
+  {
+    size_t size = sizeof(*claim);
+    const LGMP_STATUS status = lgmpHostReadDataWithSource(
+        queue, claim, &size, clientID);
+    if (status == LGMP_OK)
+      return size == sizeof(*claim) && *clientID != 0 &&
+        lgmpHostAckData(queue) == LGMP_OK;
+    if (status != LGMP_ERR_QUEUE_EMPTY ||
+        lgmpHostProcess(host) != LGMP_OK)
+      return false;
+    usleep(1000);
+  }
+  return false;
+}
+#endif
+
 struct WaitState
 {
   LG_Transport      * transport;
@@ -330,12 +369,20 @@ int main(void)
   PLGMPMemory malformedFrameMemory = NULL;
   PLGMPMemory pointerMemory = NULL;
   PLGMPMemory malformedPointerMemory = NULL;
+#ifdef ENABLE_AUDIO
+  PLGMPMemory audioResponseMemory = NULL;
+#endif
   LG_Transport * transport = NULL;
   LG_Transport * secondTransport = NULL;
   LG_Transport * thirdTransport = NULL;
   LG_Transport * nestedTransport = NULL;
   LG_Transport * admittedTransport = NULL;
   const LG_FrameOps * frameOps = NULL;
+#ifdef ENABLE_AUDIO
+  const LG_AudioOps * audioOps = NULL;
+  void * audioOpaque = NULL;
+  struct AudioStatusTrace audioTrace = { 0 };
+#endif
   struct VideoStatusTrace videoTrace;
   struct VideoStatusTrace secondVideoTrace;
   memset(&videoTrace, 0, sizeof(videoTrace));
@@ -364,6 +411,7 @@ int main(void)
   KVMFR session = { 0 };
   memcpy(session.magic, KVMFR_MAGIC, sizeof(session.magic));
   session.version = KVMFR_VERSION;
+  session.features = KVMFR_FEATURE_AUDIO;
   memcpy(session.hostver, "transport-test", sizeof("transport-test"));
 
   CHECK(lgmpHostInit(hostMemory, TEST_SHM_SIZE, &host, sizeof(session),
@@ -415,6 +463,10 @@ int main(void)
       LGMP_OK);
   CHECK(lgmpHostMemAlloc(host, sizeof(KVMFRCursor),
         &malformedPointerMemory) == LGMP_OK);
+#ifdef ENABLE_AUDIO
+  CHECK(lgmpHostMemAlloc(host, sizeof(KVMFRAudioStreamResponse),
+        &audioResponseMemory) == LGMP_OK);
+#endif
   KVMFRCursor * malformedPointer = lgmpHostMemPtr(malformedPointerMemory);
   malformedPointer->height = 2;
   malformedPointer->pitch  = UINT32_MAX;
@@ -484,6 +536,14 @@ int main(void)
   LG_TransportSession admittedSession;
   CHECK(LGT_LGMP.connectCancellable(
         admittedTransport, &admittedSession, NULL, NULL) == LG_TRANSPORT_OK);
+  CHECK(transportSession.features & LG_TRANSPORT_FEATURE_AUDIO);
+#ifdef ENABLE_AUDIO
+  CHECK(LGT_LGMP.getAudioOps);
+  audioOps = LGT_LGMP.getAudioOps(transport, &audioOpaque);
+  CHECK(audioOps && audioOpaque);
+#else
+  CHECK(!LGT_LGMP.getAudioOps);
+#endif
   videoTrace.transport = transport;
   videoTrace.ops       = frameOps;
   frameOps->setStatusListener(
@@ -560,6 +620,69 @@ int main(void)
         &videoTrace.pointerEpoch, memory_order_acquire));
   frameOps->releasePointer(transport, &pointer);
 
+#ifdef ENABLE_AUDIO
+  audioOps->setStatusListener(audioOpaque,
+      audioStatusChanged, &audioTrace);
+  CHECK(atomic_load_explicit(&audioTrace.count, memory_order_acquire) == 1);
+  CHECK(!atomic_load_explicit(
+        &audioTrace.available, memory_order_acquire));
+
+  struct WaitState audioPointerWait =
+  {
+    .transport = transport,
+    .ops       = frameOps,
+    .status    = LG_TRANSPORT_ERROR,
+  };
+  pthread_t audioPointerThread;
+  CHECK(pthread_create(&audioPointerThread, NULL,
+        waitForVideo, &audioPointerWait) == 0);
+
+  KVMFRAudioClaim audioClaim;
+  uint32_t audioClientID = 0;
+  CHECK(readAudioClaim(host, pointerQueue,
+        &audioClaim, &audioClientID));
+  CHECK(audioClaim.messageType == KVMFR_MESSAGE_AUDIO_CLAIM);
+  CHECK(audioClaim.version == KVMFR_AUDIO_CONTROL_VERSION);
+  CHECK(audioClaim.size == sizeof(audioClaim));
+  frameOps->cancelPointerWait(transport);
+  CHECK(pthread_join(audioPointerThread, NULL) == 0);
+  CHECK(audioPointerWait.status == LG_TRANSPORT_TIMEOUT);
+
+  KVMFRAudioStreamResponse * audioResponse =
+    lgmpHostMemPtr(audioResponseMemory);
+  *audioResponse = (KVMFRAudioStreamResponse)
+  {
+    .magic             = KVMFR_AUDIO_RESPONSE_MAGIC,
+    .version           = KVMFR_AUDIO_CONTROL_VERSION,
+    .size              = sizeof(*audioResponse),
+    .requestGeneration = audioClaim.requestGeneration,
+    .boundClientID     = audioClientID,
+    .result            = KVMFR_AUDIO_RESPONSE_BUSY,
+    .claimToken        = audioClaim.claimToken,
+  };
+
+  unsigned audioRecipients = 0;
+  CHECK(lgmpHostQueuePostForClients(pointerQueue,
+        UINT64_C(0x7f7f7f7f) << 32, pointerMemory,
+        &audioClientID, 1, &audioRecipients) == LGMP_OK);
+  CHECK(audioRecipients == 1);
+  CHECK(lgmpHostQueuePostForClients(pointerQueue,
+        KVMFR_AUDIO_CONTROL_UDATA, pointerMemory,
+        &audioClientID, 1, &audioRecipients) == LGMP_OK);
+  CHECK(audioRecipients == 1);
+  CHECK(lgmpHostQueuePostForClients(pointerQueue,
+        KVMFR_AUDIO_CONTROL_UDATA, audioResponseMemory,
+        &audioClientID, 1, &audioRecipients) == LGMP_OK);
+  CHECK(audioRecipients == 1);
+  CHECK(lgmpHostQueuePost(pointerQueue, CURSOR_FLAG_POSITION,
+        pointerMemory) == LGMP_OK);
+  CHECK(frameOps->nextPointer(transport, &pointer) == LG_TRANSPORT_OK);
+  CHECK(pointer.x == wirePointer->x && pointer.y == wirePointer->y);
+  frameOps->releasePointer(transport, &pointer);
+  CHECK(waitForQueuesEmpty(host, frameQueue, pointerQueue));
+  audioOps->setStatusListener(audioOpaque, NULL, NULL);
+#endif
+
   const uint64_t frameEpoch = atomic_load_explicit(
       &videoTrace.frameEpoch, memory_order_acquire);
   const uint64_t pointerEpoch = atomic_load_explicit(
@@ -1024,6 +1147,9 @@ cleanup:
   if (transport)
     LGT_LGMP.destroy(&transport);
   option_free();
+#ifdef ENABLE_AUDIO
+  lgmpHostMemFree(&audioResponseMemory);
+#endif
   lgmpHostMemFree(&pointerMemory);
   lgmpHostMemFree(&malformedPointerMemory);
   lgmpHostMemFree(&malformedFrameMemory);

client/transports/LGMP/CMakeLists.txt

@@ -1,12 +1,22 @@
 cmake_minimum_required(VERSION 3.10)
 project(transport_LGMP LANGUAGES C)
 
-add_library(transport_LGMP STATIC
+set(TRANSPORT_LGMP_SOURCES
   clipboard.c
   input.c
   lgmp.c
 )
 
+if(ENABLE_AUDIO)
+  list(APPEND TRANSPORT_LGMP_SOURCES audio.c)
+endif()
+
+add_library(transport_LGMP STATIC ${TRANSPORT_LGMP_SOURCES})
+
+if(ENABLE_AUDIO)
+  target_compile_definitions(transport_LGMP PRIVATE ENABLE_AUDIO)
+endif()
+
 target_include_directories(transport_LGMP PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/../../src
 )

client/transports/LGMP/audio.h

@@ -0,0 +1,53 @@
+/**
+ * Looking Glass
+ * Copyright © 2017-2026 The Looking Glass Authors
+ * https://looking-glass.io
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#ifndef _H_LG_CLIENT_TRANSPORT_LGMP_AUDIO_
+#define _H_LG_CLIENT_TRANSPORT_LGMP_AUDIO_
+
+#include "interface/audio.h"
+
+#include "common/KVMFRAudio.h"
+#include "common/event.h"
+
+#include <lgmp/client.h>
+
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef struct LGMPAudio LGMPAudio;
+
+bool lgmpAudio_create(PLGMPClient client, LGEvent * pointerWake,
+    LGMPAudio ** result);
+void lgmpAudio_destroy(LGMPAudio ** audio);
+
+bool lgmpAudio_connect(LGMPAudio * audio, uint32_t clientID);
+void lgmpAudio_disconnect(LGMPAudio * audio);
+
+/* Called only while the transport owns the existing pointer subscription. */
+void lgmpAudio_pumpClaim(LGMPAudio * audio,
+    PLGMPClientQueue pointerQueue);
+
+/* The caller must copy the response and release its pointer message first. */
+void lgmpAudio_handleResponse(LGMPAudio * audio,
+    const KVMFRAudioStreamResponse * response);
+
+const LG_AudioOps * lgmpAudio_getOps(void);
+
+#endif

client/transports/LGMP/lgmp.c

@@ -22,6 +22,9 @@
 
 #include "clipboard.h"
 #include "input.h"
+#ifdef ENABLE_AUDIO
+#include "audio.h"
+#endif
 
 #include <LGProtocol/KVMFR.h>
 #include <LGProtocol/KVMFRRecovery.h>
@@ -120,6 +123,9 @@ struct LG_Transport
   PLGMPClientQueue  pointerQueue;
   LGMPInput        * input;
   LGMPClipboard    * clipboard;
+#ifdef ENABLE_AUDIO
+  LGMPAudio        * audio;
+#endif
   LG_Lock           frameLock;
   LG_Lock           pointerLock;
   LG_RWLock         videoStatusLock;
@@ -145,6 +151,9 @@ struct LG_Transport
   bool     frameScheduleSupported;
   atomic_bool inputSupported;
   atomic_bool clipboardSupported;
+#ifdef ENABLE_AUDIO
+  atomic_bool audioSupported;
+#endif
   uint64_t frameLeaseHandle;
   uint32_t frameGeneration;
   uint32_t clientID;
@@ -671,6 +680,9 @@ static bool lgmp_create(LG_Transport ** result)
   atomic_init(&this->connected, false);
   atomic_init(&this->inputSupported, false);
   atomic_init(&this->clipboardSupported, false);
+#ifdef ENABLE_AUDIO
+  atomic_init(&this->audioSupported, false);
+#endif
   atomic_init(&this->destroyPending, false);
 
   this->frameWake   = lgCreateEvent(true, 0);
@@ -884,6 +896,10 @@ static void lgmp_destroyNow(LG_Transport * this)
 {
   if (this->client)
   {
+    lgmp_stopPointer(this);
+#ifdef ENABLE_AUDIO
+    lgmpAudio_destroy(&this->audio);
+#endif
     lgmpClipboard_destroy(&this->clipboard);
     lgmpInput_destroy(&this->input);
     lgmp_closeQueues(this);
@@ -986,6 +1002,9 @@ static bool lgmp_parseSession(struct LG_Transport * this,
     session->features |= LG_TRANSPORT_FEATURE_INPUT;
   if (header.features & KVMFR_FEATURE_CLIPBOARD)
     session->features |= LG_TRANSPORT_FEATURE_CLIPBOARD;
+  if (header.features & KVMFR_FEATURE_AUDIO)
+    session->features |= LG_TRANSPORT_FEATURE_AUDIO;
+
   data += sizeof(header);
   size -= sizeof(header);
   while (size >= sizeof(KVMFRRecord))
@@ -1095,6 +1114,18 @@ static LG_TransportStatus lgmp_connectInternal(LG_Transport * this,
       if (!lgmp_parseSession(this, data, size, session))
         return LG_TRANSPORT_INVALID_VERSION;
 
+#ifdef ENABLE_AUDIO
+      const bool audioSupported =
+        (session->features & LG_TRANSPORT_FEATURE_AUDIO) != 0;
+      if (audioSupported && !this->audio &&
+          !lgmpAudio_create(this->client, this->pointerWake,
+            &this->audio))
+        DEBUG_WARN("Failed to create the optional LGMP audio transport");
+      if (audioSupported && this->audio &&
+          !lgmpAudio_connect(this->audio, this->clientID))
+        DEBUG_WARN("Failed to start the optional LGMP audio transport");
+#endif
+
       LGMPVideoStatusDispatch dispatch;
       LG_LOCK(this->frameLock);
       if (++this->frameGeneration == 0)
@@ -1108,6 +1139,10 @@ static LG_TransportStatus lgmp_connectInternal(LG_Transport * this,
       atomic_store_explicit(&this->clipboardSupported,
           session->features & LG_TRANSPORT_FEATURE_CLIPBOARD,
           memory_order_release);
+#ifdef ENABLE_AUDIO
+      atomic_store_explicit(&this->audioSupported, audioSupported,
+          memory_order_release);
+#endif
       this->frameSerial            = 0;
       this->frameSerialValid       = false;
       this->formatValid            = false;
@@ -1169,6 +1204,10 @@ static void lgmp_disconnect(LG_Transport * this)
 
   lgmpInput_disconnect(this->input);
   lgmpClipboard_disconnect(this->clipboard);
+  lgmp_stopPointer(this);
+#ifdef ENABLE_AUDIO
+  lgmpAudio_disconnect(this->audio);
+#endif
 
   LG_LOCK(this->frameLock);
   if (++this->frameGeneration == 0)
@@ -1180,6 +1219,10 @@ static void lgmp_disconnect(LG_Transport * this)
       &this->inputSupported, false, memory_order_release);
   atomic_store_explicit(
       &this->clipboardSupported, false, memory_order_release);
+#ifdef ENABLE_AUDIO
+  atomic_store_explicit(
+      &this->audioSupported, false, memory_order_release);
+#endif
   this->clientID               = 0;
   LG_UNLOCK(this->frameLock);
 
@@ -2346,6 +2389,10 @@ static LG_TransportStatus lgmp_nextPointer(LG_Transport * this,
         &this->pointerQueue);
   }
   pointerQueue = this->pointerQueue;
+#ifdef ENABLE_AUDIO
+  if (status == LG_TRANSPORT_OK && pointerQueue && this->audio)
+    lgmpAudio_pumpClaim(this->audio, pointerQueue);
+#endif
   LG_UNLOCK(this->pointerLock);
   if (status == LG_TRANSPORT_TIMEOUT)
     lgmp_waitPoll(this->pointerWake, 1000);
@@ -2353,10 +2400,56 @@ static LG_TransportStatus lgmp_nextPointer(LG_Transport * this,
   {
     LGMPMessage message;
     PLGMPClientQueue processedQueue = pointerQueue;
-    status = lgmp_process(&processedQueue, this->cursorPollInterval,
-        this->pointerWake, &message);
-    if (!processedQueue)
-      lgmp_clearPointerQueue(this, pointerQueue);
+    const uint64_t deadline = this->cursorPollInterval ?
+      microtime() + this->cursorPollInterval : 0;
+    unsigned interval = this->cursorPollInterval;
+    for (unsigned skipped = 0;; ++skipped)
+    {
+      status = lgmp_process(&processedQueue, interval,
+          this->pointerWake, &message);
+      if (!processedQueue)
+        lgmp_clearPointerQueue(this, pointerQueue);
+      if (status != LG_TRANSPORT_OK)
+        break;
+
+      const uint32_t tag = (uint32_t)(message.udata >> 32);
+      if (!tag)
+        break;
+
+#ifdef ENABLE_AUDIO
+      KVMFRAudioStreamResponse audioResponse;
+      const bool audioResponseValid =
+        tag == KVMFR_AUDIO_CONTROL_TAG && this->audio &&
+        message.size == sizeof(audioResponse);
+      if (audioResponseValid)
+        memcpy(&audioResponse, message.mem, sizeof(audioResponse));
+#endif
+      status = lgmp_donePointerMessage(this, pointerQueue);
+      if (status != LG_TRANSPORT_OK)
+        break;
+#ifdef ENABLE_AUDIO
+      if (audioResponseValid)
+        lgmpAudio_handleResponse(this->audio, &audioResponse);
+#endif
+
+      if (skipped + 1 >= LGMP_Q_POINTER_LEN)
+      {
+        status = LG_TRANSPORT_TIMEOUT;
+        break;
+      }
+      if (!deadline)
+      {
+        interval = 0;
+        continue;
+      }
+      const uint64_t now = microtime();
+      if (now >= deadline)
+      {
+        status = LG_TRANSPORT_TIMEOUT;
+        break;
+      }
+      interval = (unsigned)(deadline - now);
+    }
     if (status == LG_TRANSPORT_OK)
     {
       if (!message.mem || message.size < sizeof(KVMFRCursor) ||
@@ -2623,6 +2716,26 @@ static const LG_ClipboardOps * lgmp_getClipboardOps(
   return lgmpClipboard_getOps();
 }
 
+#ifdef ENABLE_AUDIO
+static const LG_AudioOps * lgmp_getAudioOps(
+    LG_Transport * this, void ** opaque)
+{
+  *opaque = NULL;
+  if (!atomic_load_explicit(&this->connected, memory_order_acquire) ||
+      !atomic_load_explicit(
+          &this->audioSupported, memory_order_acquire))
+    return NULL;
+
+  if (!this->audio &&
+      !lgmpAudio_create(this->client, this->pointerWake, &this->audio))
+    return NULL;
+  if (!lgmpAudio_connect(this->audio, this->clientID))
+    return NULL;
+  *opaque = this->audio;
+  return lgmpAudio_getOps();
+}
+#endif
+
 static void lgmp_probeVideoStatus(LG_Transport * this)
 {
   bool frameAvailable   = false;
@@ -2766,6 +2879,9 @@ const LG_TransportOps LGT_LGMP =
   .sessionValid     = lgmp_sessionValid,
   .getVideoOps      = lgmp_getVideoOps,
   .getInputOps      = lgmp_getInputOps,
+#ifdef ENABLE_AUDIO
+  .getAudioOps      = lgmp_getAudioOps,
+#endif
   .getClipboardOps  = lgmp_getClipboardOps,
   .getRecoveryInfo  = lgmp_getRecoveryInfo,
   .requestRecovery  = lgmp_requestRecovery,