[client/idd] network: add GPU video transport

This commit is contained in:
Geoffrey McRae
2026-10-01 03:51:33 +10:00
parent d24d11c3b7
commit 53c25dd5a9
19 changed files with 6528 additions and 73 deletions

View File

@@ -93,6 +93,7 @@
<ClCompile Include="transport\network\CNetworkServer.cpp" />
<ClCompile Include="transport\network\CNetworkSession.cpp" />
<ClCompile Include="transport\network\CNetworkTransport.cpp" />
<ClCompile Include="transport\network\CNetworkVideo.cpp" />
<ClCompile Include="transport\network\NetworkAuthCrypto.cpp" />
<ClCompile Include="transport\network\NetworkSettings.cpp" />
<ClCompile Include="config\CSettings.cpp" />
@@ -180,6 +181,7 @@
<ClInclude Include="transport\network\CNetworkServer.h" />
<ClInclude Include="transport\network\CNetworkSession.h" />
<ClInclude Include="transport\network\CNetworkTransport.h" />
<ClInclude Include="transport\network\CNetworkVideo.h" />
<ClInclude Include="transport\network\NetworkAuthCrypto.h" />
<ClInclude Include="transport\network\NetworkSettings.h" />
<ClInclude Include="config\CSettings.h" />

View File

@@ -301,6 +301,9 @@
<ClInclude Include="transport\network\CNetworkTransport.h">
<Filter>Transport\Network</Filter>
</ClInclude>
<ClInclude Include="transport\network\CNetworkVideo.h">
<Filter>Transport\Network</Filter>
</ClInclude>
<ClInclude Include="transport\network\NetworkAuthCrypto.h">
<Filter>Transport\Network</Filter>
</ClInclude>
@@ -498,6 +501,9 @@
<ClCompile Include="transport\network\CNetworkTransport.cpp">
<Filter>Transport\Network</Filter>
</ClCompile>
<ClCompile Include="transport\network\CNetworkVideo.cpp">
<Filter>Transport\Network</Filter>
</ClCompile>
<ClCompile Include="transport\network\NetworkAuthCrypto.cpp">
<Filter>Transport\Network</Filter>
</ClCompile>

View File

@@ -439,10 +439,12 @@ struct CNetworkServer::Impl
};
Impl(CNetworkTransport& transport, CNetworkAudio& audio,
CNetworkClipboard& clipboard, const NetworkSettings& settings) :
CNetworkClipboard& clipboard, CNetworkVideo& video,
const NetworkSettings& settings) :
transport(transport),
audio(audio),
clipboard(clipboard),
video(video),
settings(settings)
{
}
@@ -463,6 +465,7 @@ struct CNetworkServer::Impl
CNetworkTransport& transport;
CNetworkAudio& audio;
CNetworkClipboard& clipboard;
CNetworkVideo& video;
NetworkSettings settings;
const QUIC_API_TABLE * api = nullptr;
HQUIC registration = nullptr;
@@ -536,6 +539,15 @@ struct CNetworkServer::Impl
};
}
static NetworkVideoPeer VideoPeer(const Connection& connection)
{
return
{
connection.client,
connection.generation,
};
}
bool AudioPeerInfo(const NetworkAudioPeer& peer,
NetworkAudioPeerInfo& info)
{
@@ -679,6 +691,87 @@ struct CNetworkServer::Impl
return NetworkClipboardSendResult::FAILED;
}
bool VideoPeerInfo(const NetworkVideoPeer& peer,
NetworkVideoPeerInfo& info)
{
CSRWSharedLock guard(lock);
for (Connection * connection : connections)
{
if (connection->client != peer.client ||
connection->generation != peer.generation)
continue;
CSRWSharedLock connectionGuard(connection->lock);
const uint16_t serviceMask = UINT16_C(1) << LG_NET_SERVICE_VIDEO;
if (connection->closing ||
connection->handshake != HandshakeState::READY ||
!(connection->selectedServices & serviceMask))
return false;
const LGNetCapability& capability =
connection->selectedCapabilities[LG_NET_SERVICE_VIDEO];
info.serviceVersion = capability.versionMin;
info.capabilityFlags = capability.flags;
info.maxReliablePayload = capability.maxPayloadLength;
info.maxDatagramPayload = capability.maxDatagramPayload;
info.maxBulkPayload =
capability.flags & LG_NET_CAPABILITY_BULK ?
capability.maxPayloadLength : 0;
return true;
}
return false;
}
NetworkVideoSendResult SendVideo(
const NetworkVideoPeer& peer, NetworkVideoDelivery delivery,
uint16_t messageType, LGNetEnvelopeFlags flags,
uint64_t componentEpoch, uint64_t requestID,
const void * payload, uint32_t payloadLength)
{
CSRWSharedLock guard(lock);
for (Connection * connection : connections)
{
if (connection->client != peer.client ||
connection->generation != peer.generation)
continue;
SendResult result = SendResult::FAILED;
switch (delivery)
{
case NetworkVideoDelivery::RELIABLE:
result = TrySendPacket(*connection, LG_NET_SERVICE_VIDEO,
messageType, flags, componentEpoch, requestID,
payload, payloadLength);
break;
case NetworkVideoDelivery::DATAGRAM:
result = TrySendDatagram(*connection, LG_NET_SERVICE_VIDEO,
messageType, flags, componentEpoch, requestID,
payload, payloadLength, true);
break;
case NetworkVideoDelivery::BULK:
result = TrySendBulk(*connection, LG_NET_SERVICE_VIDEO,
messageType, flags, componentEpoch, requestID,
payload, payloadLength);
break;
}
switch (result)
{
case SendResult::SENT:
return NetworkVideoSendResult::ACCEPTED;
case SendResult::BLOCKED:
return NetworkVideoSendResult::BLOCKED;
case SendResult::FAILED:
return NetworkVideoSendResult::FAILED;
}
}
return NetworkVideoSendResult::FAILED;
}
void Shutdown(Connection& connection, QUIC_UINT62 error)
{
HQUIC handle = nullptr;
@@ -1048,6 +1141,31 @@ struct CNetworkServer::Impl
ClipboardPeer(connection), LG_NET_SERVICE_FILE, bulk);
}
void NotifyVideoSendReady(Connection& connection, Delivery delivery)
{
NetworkVideoDelivery videoDelivery = NetworkVideoDelivery::RELIABLE;
switch (delivery)
{
case Delivery::RELIABLE:
videoDelivery = NetworkVideoDelivery::RELIABLE;
break;
case Delivery::DATAGRAM:
videoDelivery = NetworkVideoDelivery::DATAGRAM;
break;
case Delivery::BULK:
videoDelivery = NetworkVideoDelivery::BULK;
break;
default:
return;
}
if (!ConsumeSendReady(connection, LG_NET_SERVICE_VIDEO, delivery))
return;
video.SendReady(VideoPeer(connection), videoDelivery);
}
SendResult TrySendPacket(Connection& connection, LGNetService service,
uint16_t messageType, LGNetEnvelopeFlags flags,
uint64_t componentEpoch, uint64_t requestID, const void * payload,
@@ -1894,6 +2012,7 @@ retry:
bool activated = false;
bool audioSelected = false;
bool clipboardSelected = false;
bool videoSelected = false;
{
CSRWExclusiveLock guard(connection.lock);
activated = !connection.closing &&
@@ -1909,15 +2028,17 @@ retry:
connection.clientNonce, sizeof(connection.clientNonce));
NetworkAuthCrypto::SecureZero(
connection.serverNonce, sizeof(connection.serverNonce));
connection.handshake = HandshakeState::READY;
audioSelected = (connection.selectedServices &
connection.handshake = HandshakeState::READY;
audioSelected = (connection.selectedServices &
(UINT16_C(1) << LG_NET_SERVICE_AUDIO)) != 0;
const uint16_t clipboardServices =
(UINT16_C(1) << LG_NET_SERVICE_CLIPBOARD) |
(UINT16_C(1) << LG_NET_SERVICE_FILE);
clipboardSelected =
clipboardSelected =
(connection.selectedServices & clipboardServices) ==
clipboardServices;
videoSelected = (connection.selectedServices &
(UINT16_C(1) << LG_NET_SERVICE_VIDEO)) != 0;
}
}
if (!activated)
@@ -1930,6 +2051,8 @@ retry:
if (clipboardSelected &&
!clipboard.PeerConnected(ClipboardPeer(connection)))
return false;
if (videoSelected && !video.PeerConnected(VideoPeer(connection)))
return false;
DEBUG_INFO("Network client %u completed the control handshake",
connection.client);
return true;
@@ -3421,6 +3544,9 @@ retry:
const bool audioSubscribe =
packet.envelope.service == LG_NET_SERVICE_AUDIO &&
packet.envelope.messageType == LG_NET_AUDIO_MESSAGE_SUBSCRIBE;
const bool videoSubscribe =
packet.envelope.service == LG_NET_SERVICE_VIDEO &&
packet.envelope.messageType == LG_NET_VIDEO_MESSAGE_SUBSCRIBE;
const bool audioResponse =
packet.envelope.service == LG_NET_SERVICE_AUDIO &&
packet.envelope.messageType == LG_NET_AUDIO_MESSAGE_STATE_ACK;
@@ -3429,7 +3555,7 @@ retry:
packet.envelope.service == LG_NET_SERVICE_FILE;
LGNetEnvelopeFlags expectedFlags =
LG_NET_ENVELOPE_RELIABLE | LG_NET_ENVELOPE_FINAL;
if (audioSubscribe)
if (audioSubscribe || videoSubscribe)
expectedFlags |= LG_NET_ENVELOPE_ACK_REQUIRED;
if (cursorStatus || audioResponse)
expectedFlags |= LG_NET_ENVELOPE_RESPONSE;
@@ -3544,6 +3670,11 @@ retry:
return audio.HandlePacket(AudioPeer(connection), packet, false) !=
NetworkAudioPacketResult::INVALID;
case LG_NET_SERVICE_VIDEO:
return video.HandlePacket(VideoPeer(connection), packet,
NetworkVideoDelivery::RELIABLE) !=
NetworkVideoPacketResult::INVALID;
case LG_NET_SERVICE_CLIPBOARD:
case LG_NET_SERVICE_FILE:
return clipboard.HandlePacket(
@@ -3621,6 +3752,10 @@ retry:
if (packet.envelope.service == LG_NET_SERVICE_AUDIO)
return audio.HandlePacket(AudioPeer(connection), packet, true) !=
NetworkAudioPacketResult::INVALID;
if (packet.envelope.service == LG_NET_SERVICE_VIDEO)
return video.HandlePacket(VideoPeer(connection), packet,
NetworkVideoDelivery::DATAGRAM) !=
NetworkVideoPacketResult::INVALID;
return false;
}
@@ -3994,6 +4129,7 @@ retry:
static_cast<unsigned>(service));
session = std::move(connection.session);
}
video.PeerDisconnected(VideoPeer(connection));
clipboard.PeerDisconnected(ClipboardPeer(connection));
audio.PeerDisconnected(AudioPeer(connection));
if (session)
@@ -4156,6 +4292,7 @@ retry:
ReadyServicesLocked(connection, Delivery::DATAGRAM);
}
owner.NotifyAudioSendReady(connection, true);
owner.NotifyVideoSendReady(connection, Delivery::DATAGRAM);
return QUIC_STATUS_SUCCESS;
}
@@ -4170,6 +4307,7 @@ retry:
ReleaseSendLocked(connection, *send);
}
owner.NotifyAudioSendReady(connection, true);
owner.NotifyVideoSendReady(connection, Delivery::DATAGRAM);
}
return QUIC_STATUS_SUCCESS;
@@ -4215,6 +4353,7 @@ retry:
ReleaseSendLocked(connection, *send);
}
owner.NotifyAudioSendReady(connection, false);
owner.NotifyVideoSendReady(connection, Delivery::RELIABLE);
owner.NotifyClipboardSendReady(connection, Delivery::RELIABLE);
return QUIC_STATUS_SUCCESS;
}
@@ -4352,7 +4491,10 @@ retry:
stream.streamID = event->START_COMPLETE.ID;
stream.bound = true;
if (stream.openingReturned)
{
ready = ReadyServicesLocked(connection, Delivery::BULK);
connection.bulkSend.writableServices |= ready;
}
}
}
if (QUIC_FAILED(status))
@@ -4367,7 +4509,10 @@ retry:
send.release();
if (ready)
{
owner.NotifyVideoSendReady(connection, Delivery::BULK);
owner.NotifyClipboardSendReady(connection, Delivery::BULK);
}
return QUIC_STATUS_SUCCESS;
}
@@ -4385,6 +4530,7 @@ retry:
CSRWExclusiveLock guard(connection.lock);
ReleaseSendLocked(connection, *send);
}
owner.NotifyVideoSendReady(connection, Delivery::BULK);
owner.NotifyClipboardSendReady(connection, Delivery::BULK);
return QUIC_STATUS_SUCCESS;
}
@@ -4438,6 +4584,7 @@ retry:
{
const uint64_t now = GetTickCount64();
audio.Process(now);
video.Process(now);
clipboard.Process(now);
CSRWSharedLock guard(lock);
if (stopping || !api)
@@ -4666,8 +4813,8 @@ retry:
CNetworkServer::CNetworkServer(CNetworkTransport& transport,
CNetworkAudio& audio, CNetworkClipboard& clipboard,
const NetworkSettings& settings) :
m_impl(new Impl(transport, audio, clipboard, settings))
CNetworkVideo& video, const NetworkSettings& settings) :
m_impl(new Impl(transport, audio, clipboard, video, settings))
{
}
@@ -4725,3 +4872,21 @@ NetworkClipboardSendResult CNetworkServer::SendClipboard(
componentEpoch, requestID, payload, payloadLength) :
NetworkClipboardSendResult::FAILED;
}
bool CNetworkServer::VideoPeerInfo(
const NetworkVideoPeer& peer, NetworkVideoPeerInfo& info)
{
return m_impl && m_impl->VideoPeerInfo(peer, info);
}
NetworkVideoSendResult CNetworkServer::SendVideo(
const NetworkVideoPeer& peer, NetworkVideoDelivery delivery,
uint16_t messageType, LGNetEnvelopeFlags flags,
uint64_t componentEpoch, uint64_t requestID,
const void * payload, uint32_t payloadLength)
{
return m_impl ?
m_impl->SendVideo(peer, delivery, messageType, flags,
componentEpoch, requestID, payload, payloadLength) :
NetworkVideoSendResult::FAILED;
}

View File

@@ -22,6 +22,7 @@
#include "transport/network/CNetworkAudio.h"
#include "transport/network/CNetworkClipboard.h"
#include "transport/network/CNetworkVideo.h"
#include <memory>
@@ -30,7 +31,8 @@ struct NetworkSettings;
class CNetworkServer final :
public INetworkAudioSender,
public INetworkClipboardSender
public INetworkClipboardSender,
public INetworkVideoSender
{
private:
struct Impl;
@@ -38,7 +40,8 @@ private:
public:
CNetworkServer(CNetworkTransport& transport, CNetworkAudio& audio,
CNetworkClipboard& clipboard, const NetworkSettings& settings);
CNetworkClipboard& clipboard, CNetworkVideo& video,
const NetworkSettings& settings);
~CNetworkServer();
CNetworkServer(const CNetworkServer&) = delete;
@@ -63,4 +66,12 @@ public:
uint16_t messageType, LGNetEnvelopeFlags flags,
uint64_t componentEpoch, uint64_t requestID,
const void * payload, uint32_t payloadLength) override;
bool VideoPeerInfo(const NetworkVideoPeer& peer,
NetworkVideoPeerInfo& info) override;
NetworkVideoSendResult SendVideo(
const NetworkVideoPeer& peer, NetworkVideoDelivery delivery,
uint16_t messageType, LGNetEnvelopeFlags flags,
uint64_t componentEpoch, uint64_t requestID,
const void * payload, uint32_t payloadLength) override;
};

View File

@@ -77,7 +77,8 @@ bool CNetworkTransport::Initialize()
try
{
m_server.reset(
new CNetworkServer(*this, m_audio, m_clipboard, m_settings));
new CNetworkServer(
*this, m_audio, m_clipboard, m_video, m_settings));
}
catch (...)
{
@@ -97,11 +98,25 @@ bool CNetworkTransport::Initialize()
m_server.reset();
return false;
}
if (!m_video.AttachSender(*m_server))
{
DEBUG_ERROR("Failed to attach the network video sender");
m_clipboard.DetachSender();
m_audio.DetachSender();
m_server.reset();
return false;
}
if ((m_config.services & TRANSPORT_SERVICE_FRAME) && !m_video.Start())
{
DEBUG_ERROR("Failed to start the network video endpoint");
m_video.DetachSender();
m_clipboard.DetachSender();
m_audio.DetachSender();
m_server.reset();
return false;
}
m_state = State::INITIALIZED;
if (m_config.services & TRANSPORT_SERVICE_FRAME)
DEBUG_WARN("Network video is unavailable until the GPU codec and "
"texture sink are attached");
return true;
}
@@ -171,6 +186,7 @@ void CNetworkTransport::Stop()
if (server)
server->Stop();
m_video.DetachSender();
m_clipboard.DetachSender();
m_audio.DetachSender();
@@ -183,6 +199,7 @@ void CNetworkTransport::Stop()
session->Close();
m_input.Stop();
m_video.Stop();
m_audio.Stop();
m_clipboard.Stop();
m_control.SetControlEvents(nullptr, {});
@@ -208,33 +225,42 @@ void CNetworkTransport::RecoveryStatus(const SourceKey& source,
const FrameProfile * CNetworkTransport::Profiles(unsigned& count) const
{
count = 0;
return nullptr;
if (!(m_config.services & TRANSPORT_SERVICE_FRAME))
{
count = 0;
return nullptr;
}
return m_video.Profiles(count);
}
CfgResult CNetworkTransport::Probe(const FrameCfg& cfg) const
{
(void)cfg;
return CfgResult::REJECTED;
return (m_config.services & TRANSPORT_SERVICE_FRAME) ?
m_video.Probe(cfg) : CfgResult::REJECTED;
}
CfgResult CNetworkTransport::Prepare(const FrameCfg& cfg)
{
(void)cfg;
return CfgResult::REJECTED;
return (m_config.services & TRANSPORT_SERVICE_FRAME) ?
m_video.Prepare(cfg) : CfgResult::REJECTED;
}
void CNetworkTransport::Commit()
{
if (m_config.services & TRANSPORT_SERVICE_FRAME)
m_video.Commit();
}
void CNetworkTransport::Abort()
{
if (m_config.services & TRANSPORT_SERVICE_FRAME)
m_video.Abort();
}
std::shared_ptr<const FrameCaps> CNetworkTransport::GetFrameCaps() const
{
return std::shared_ptr<const FrameCaps>();
return (m_config.services & TRANSPORT_SERVICE_FRAME) ?
m_video.GetFrameCaps() : std::shared_ptr<const FrameCaps>();
}
DirectFrameBufferMemory CNetworkTransport::GetDirectMemory() const
@@ -264,6 +290,11 @@ IInputSource * CNetworkTransport::Input()
return (m_config.services & TRANSPORT_SERVICE_INPUT) ? &m_input : nullptr;
}
ITexSink * CNetworkTransport::TexSink()
{
return (m_config.services & TRANSPORT_SERVICE_FRAME) ? &m_video : nullptr;
}
bool CNetworkTransport::ServiceEnabled(LGNetService service) const
{
switch (service)

View File

@@ -25,6 +25,7 @@
#include "transport/network/CNetworkAudio.h"
#include "transport/network/CNetworkClipboard.h"
#include "transport/network/CNetworkSession.h"
#include "transport/network/CNetworkVideo.h"
#include "transport/network/NetworkSettings.h"
#include <memory>
@@ -32,9 +33,8 @@
class CNetworkServer;
// This provider owns transport-independent lifecycle and session state. It
// deliberately advertises no frame capabilities and no texture sink until a
// GPU codec backend is attached; it must not be mistaken for working media.
// This provider owns the transport-independent lifecycle, media endpoints,
// and authenticated session state used by the QUIC server.
class CNetworkTransport final : public ITransport
{
private:
@@ -53,6 +53,7 @@ private:
NetworkSettings m_settings;
CNetworkAudio m_audio;
CNetworkClipboard m_clipboard;
CNetworkVideo m_video;
std::unique_ptr<CNetworkServer> m_server;
mutable CSRWLock m_lifecycleLock;
@@ -97,6 +98,7 @@ public:
IClipboardSource * Clipboard() override;
IControlSink * Control() override;
IInputSource * Input() override;
ITexSink * TexSink() override;
const NetworkSettings& Settings() const { return m_settings; }
bool ServiceEnabled(LGNetService service) const;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,118 @@
/**
* Looking Glass
* Copyright © 2017-2026 The Looking Glass Authors
* https://looking-glass.io
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
* Software Foundation; either version 2 of the License, or (at your option)
* any later version.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc., 59
* Temple Place, Suite 330, Boston, MA 02111-1307 USA
*/
#pragma once
#include "transport/FrameCaps.h"
#include "transport/ITexSink.h"
#include <LGProtocol/NetworkProtocol.h>
#include <LGProtocol/NetworkServices.h>
#include <memory>
#include <stdint.h>
struct NetworkVideoPeer
{
uint32_t client = 0;
uint32_t generation = 0;
};
struct NetworkVideoPeerInfo
{
uint16_t serviceVersion = 0;
LGNetCapabilityFlags capabilityFlags = 0;
uint32_t maxReliablePayload = 0;
uint32_t maxDatagramPayload = 0;
uint32_t maxBulkPayload = 0;
};
enum class NetworkVideoDelivery
{
RELIABLE,
DATAGRAM,
BULK,
};
enum class NetworkVideoSendResult
{
ACCEPTED,
BLOCKED,
FAILED,
};
enum class NetworkVideoPacketResult
{
HANDLED,
IGNORED,
INVALID,
};
class INetworkVideoSender
{
public:
virtual ~INetworkVideoSender() = default;
virtual bool VideoPeerInfo(
const NetworkVideoPeer& peer, NetworkVideoPeerInfo& info) = 0;
virtual NetworkVideoSendResult SendVideo(
const NetworkVideoPeer& peer, NetworkVideoDelivery delivery,
uint16_t messageType, LGNetEnvelopeFlags flags,
uint64_t componentEpoch, uint64_t requestID,
const void * payload, uint32_t payloadLength) = 0;
};
class CNetworkVideo final : public ITexSink
{
private:
struct Impl;
std::unique_ptr<Impl> m_impl;
public:
CNetworkVideo();
~CNetworkVideo() override;
CNetworkVideo(const CNetworkVideo&) = delete;
CNetworkVideo& operator=(const CNetworkVideo&) = delete;
bool AttachSender(INetworkVideoSender& sender);
void DetachSender();
bool Start();
void Stop();
void Process(uint64_t nowMs);
bool PeerConnected(const NetworkVideoPeer& peer);
void PeerDisconnected(const NetworkVideoPeer& peer);
NetworkVideoPacketResult HandlePacket(const NetworkVideoPeer& peer,
const LGNetPacketView& packet, NetworkVideoDelivery delivery);
void SendReady(
const NetworkVideoPeer& peer, NetworkVideoDelivery delivery);
const FrameProfile * Profiles(unsigned& count) const;
std::shared_ptr<const FrameCaps> GetFrameCaps() const;
CfgResult Probe(const FrameCfg& cfg) const;
CfgResult Prepare(const FrameCfg& cfg);
void Commit();
void Abort();
PushResult Push(FrameIn frame, TexLease lease) noexcept override;
PushResult Push(FrameIn frame, D11Lease lease) noexcept override;
};