From 53c25dd5a999ba9a4dd18a28ed673999677ead57 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Thu, 1 Oct 2026 03:51:33 +1000 Subject: [PATCH] [client/idd] network: add GPU video transport --- client/transports/Network/CMakeLists.txt | 4 + client/transports/Network/connection.c | 3 +- client/transports/Network/network.c | 98 +- client/transports/Network/video.c | 1671 +++++++++++ client/transports/Network/video.h | 65 + client/transports/Network/video/decoder.h | 67 + client/transports/Network/video/decoder_egl.c | 1081 +++++++ client/transports/Network/video/pyrowave.c | 324 +++ client/transports/Network/video/pyrowave.h | 42 + client/transports/Network/video/shaders.h | 332 +++ idd/LGIdd/LGIdd.vcxproj | 2 + idd/LGIdd/LGIdd.vcxproj.filters | 6 + .../transport/network/CNetworkServer.cpp | 179 +- idd/LGIdd/transport/network/CNetworkServer.h | 15 +- .../transport/network/CNetworkTransport.cpp | 53 +- .../transport/network/CNetworkTransport.h | 8 +- idd/LGIdd/transport/network/CNetworkVideo.cpp | 2531 +++++++++++++++++ idd/LGIdd/transport/network/CNetworkVideo.h | 118 + repos/LGProtocol | 2 +- 19 files changed, 6528 insertions(+), 73 deletions(-) create mode 100644 client/transports/Network/video.c create mode 100644 client/transports/Network/video.h create mode 100644 client/transports/Network/video/decoder.h create mode 100644 client/transports/Network/video/decoder_egl.c create mode 100644 client/transports/Network/video/pyrowave.c create mode 100644 client/transports/Network/video/pyrowave.h create mode 100644 client/transports/Network/video/shaders.h create mode 100644 idd/LGIdd/transport/network/CNetworkVideo.cpp create mode 100644 idd/LGIdd/transport/network/CNetworkVideo.h diff --git a/client/transports/Network/CMakeLists.txt b/client/transports/Network/CMakeLists.txt index 055e27e5..86b51c35 100644 --- a/client/transports/Network/CMakeLists.txt +++ b/client/transports/Network/CMakeLists.txt @@ -15,6 +15,9 @@ set(TRANSPORT_NETWORK_SOURCES known_hosts.c network.c recovery.c + video.c + video/decoder_egl.c + video/pyrowave.c ) if(WIN32) @@ -40,6 +43,7 @@ target_include_directories(transport_Network PRIVATE target_link_libraries(transport_Network PRIVATE lg_common + lg_egl_provider msquic::msquic ) diff --git a/client/transports/Network/connection.c b/client/transports/Network/connection.c index 57726445..f32391cf 100644 --- a/client/transports/Network/connection.c +++ b/client/transports/Network/connection.c @@ -1071,8 +1071,7 @@ static void networkConnectionSetSessionFeatures(NetworkConnection * this) if (services & (UINT16_C(1) << LG_NET_SERVICE_CONTROL)) this->session.features |= LG_TRANSPORT_FEATURE_SET_CURSOR_POS | - LG_TRANSPORT_FEATURE_WINDOW_SIZE | - LG_TRANSPORT_FEATURE_FRAME_SCHEDULE; + LG_TRANSPORT_FEATURE_WINDOW_SIZE; } static bool networkConnectionHandleAuthChallenge(NetworkConnection * this, diff --git a/client/transports/Network/network.c b/client/transports/Network/network.c index bbfa5d01..74f69bee 100644 --- a/client/transports/Network/network.c +++ b/client/transports/Network/network.c @@ -27,6 +27,7 @@ #include "input.h" #include "known_hosts.h" #include "recovery.h" +#include "video.h" #include "app.h" #include "interface/renderer.h" @@ -56,6 +57,9 @@ (LG_NET_CLIPBOARD_CHUNK_HEADER_WIRE_SIZE + NETWORK_TRANSFER_CHUNK) #define NETWORK_FILE_MAX_PAYLOAD \ (LG_NET_FILE_CHUNK_HEADER_WIRE_SIZE + NETWORK_TRANSFER_CHUNK) +#define NETWORK_VIDEO_MAX_PAYLOAD \ + (LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE + \ + LG_NET_VIDEO_MAX_FRAGMENT_LENGTH) static const LGNetCapability NETWORK_CAPABILITIES[] = { @@ -94,6 +98,19 @@ static const LGNetCapability NETWORK_CAPABILITIES[] = .maxBidirectionalStreams = 0, .maxUnidirectionalStreams = 0, }, + { + .service = LG_NET_SERVICE_VIDEO, + .versionMin = + LG_NET_VIDEO_BLOCK_FRAGMENTATION_INTRODUCED_SERVICE_VERSION, + .versionMax = LG_NET_VIDEO_VERSION_CURRENT, + .flags = LG_NET_CAPABILITY_RELIABLE | + LG_NET_CAPABILITY_DATAGRAM | LG_NET_CAPABILITY_BULK, + .features = 0, + .maxPayloadLength = NETWORK_VIDEO_MAX_PAYLOAD, + .maxDatagramPayload = NETWORK_MAX_DATAGRAM, + .maxBidirectionalStreams = 0, + .maxUnidirectionalStreams = 1, + }, { .service = LG_NET_SERVICE_AUDIO, .versionMin = @@ -160,8 +177,8 @@ struct LG_Transport NetworkCursor * cursor; NetworkInput * input; NetworkRecovery * recovery; + NetworkVideo * video; NetworkKnownHosts knownHosts; - LGEvent * frameEvent; LGEvent * promptIdle; LG_Lock callbackLock; @@ -172,7 +189,6 @@ struct LG_Transport LG_VideoStatusFn videoStatusFn; void * videoStatusOpaque; - bool rendererAttached; bool promptCallbackActive; struct NetworkTrustPrompt prompt; }; @@ -340,6 +356,17 @@ static void networkNotifyVideoStatus(LG_Transport * this) LG_UNLOCK(this->callbackLock); } +static void networkVideoStatus(void * opaque, + const LG_VideoComponentStatus * status) +{ + LG_Transport * this = opaque; + + LG_LOCK(this->stateLock); + this->videoStatus.frame = *status; + LG_UNLOCK(this->stateLock); + networkNotifyVideoStatus(this); +} + static uint64_t networkNextEpoch(uint64_t epoch) { ++epoch; @@ -375,18 +402,17 @@ static void networkConnectionState(void * opaque, bool connected, networkClipboardConnectionState(this->clipboard, connected); networkControlConnectionState(this->control, connected); networkRecoveryConnectionState(this->recovery, connected); + networkVideoConnectionState(this->video, connected); const bool cursorAvailable = networkCursorConnectionState(this->cursor, connected); LG_LOCK(this->stateLock); - networkSetUnavailable(&this->videoStatus.frame, status); if (cursorAvailable) networkSetAvailable(&this->videoStatus.pointer); else networkSetUnavailable(&this->videoStatus.pointer, status); LG_UNLOCK(this->stateLock); - lgSignalEvent(this->frameEvent); networkNotifyVideoStatus(this); } @@ -416,6 +442,10 @@ static void networkPacket(void * opaque, const LGNetPacketView * packet, (datagram ? networkAudioDatagram(this->audio, packet) : networkAudioPacket(this->audio, packet)); break; + case LG_NET_SERVICE_VIDEO: + valid = datagram ? networkVideoDatagram(this->video, packet) : + networkVideoPacket(this->video, packet); + break; case LG_NET_SERVICE_CLIPBOARD: case LG_NET_SERVICE_FILE: valid = delivery != NETWORK_PACKET_DELIVERY_DATAGRAM && @@ -451,7 +481,9 @@ static void networkSendReady(void * opaque, LGNetService service, if ((service == LG_NET_SERVICE_CLIPBOARD || service == LG_NET_SERVICE_FILE) && !datagram) networkClipboardSendReady(this->clipboard, service, delivery); - lgSignalEvent(this->frameEvent); + if (service == LG_NET_SERVICE_VIDEO && + delivery != NETWORK_PACKET_DELIVERY_BULK) + networkVideoSendReady(this->video, datagram); } static bool networkCreate(LG_Transport ** result) @@ -474,10 +506,8 @@ static bool networkCreate(LG_Transport ** result) if (!networkKnownHostsInit(&this->knownHosts)) goto fail; - this->frameEvent = lgCreateEvent(true, 0); this->promptIdle = lgCreateEvent(false, 0); - if (!this->frameEvent || !this->promptIdle || - !lgSignalEvent(this->promptIdle)) + if (!this->promptIdle || !lgSignalEvent(this->promptIdle)) goto fail; const char * address = option_get_string("network", "address"); @@ -516,13 +546,16 @@ static bool networkCreate(LG_Transport ** result) !networkControlCreate(&this->control, this->connection) || !networkCursorCreate(&this->cursor, this->connection) || !networkInputCreate(&this->input, this->connection) || - !networkRecoveryCreate(&this->recovery, this->connection)) + !networkRecoveryCreate(&this->recovery, this->connection) || + !networkVideoCreate(&this->video, this->connection)) goto fail; + networkVideoSetStatusListener(this->video, networkVideoStatus, this); *result = this; return true; fail: + networkVideoDestroy(&this->video); networkRecoveryDestroy(&this->recovery); networkInputDestroy(&this->input); networkCursorDestroy(&this->cursor); @@ -530,8 +563,6 @@ fail: networkClipboardDestroy(&this->clipboard); networkAudioDestroy(&this->audio); networkConnectionDestroy(&this->connection); - if (this->frameEvent) - lgFreeEvent(this->frameEvent); if (this->promptIdle) lgFreeEvent(this->promptIdle); networkKnownHostsFree(&this->knownHosts); @@ -560,13 +591,13 @@ static void networkDestroy(LG_Transport ** transport) LG_UNLOCK(this->callbackLock); networkRecoveryDestroy(&this->recovery); + networkVideoDestroy(&this->video); networkInputDestroy(&this->input); networkCursorDestroy(&this->cursor); networkControlDestroy(&this->control); networkClipboardDestroy(&this->clipboard); networkAudioDestroy(&this->audio); networkConnectionDestroy(&this->connection); - lgFreeEvent(this->frameEvent); lgFreeEvent(this->promptIdle); networkKnownHostsFree(&this->knownHosts); LG_LOCK_FREE(this->promptLock); @@ -597,9 +628,9 @@ static void networkDisconnect(LG_Transport * this) networkClipboardConnectionState(this->clipboard, false); networkControlConnectionState(this->control, false); networkRecoveryConnectionState(this->recovery, false); + networkVideoConnectionState(this->video, false); networkCursorConnectionState(this->cursor, false); networkConnectionDisconnect(this->connection); - lgSignalEvent(this->frameEvent); } static bool networkSessionValid(LG_Transport * this) @@ -633,67 +664,40 @@ static bool networkSupportsDMA(LG_Transport * this) static bool networkAttachRenderer(LG_Transport * this, const LG_RendererInterop * interop) { - const size_t requiredSize = offsetof(LG_RendererInterop, - egl.sharedTextureImport) + sizeof(interop->egl.sharedTextureImport); - if (!interop || - interop->version < LG_RENDERER_INTEROP_EGL_TEXTURE_VERSION || - interop->size < requiredSize || - interop->type != LG_RENDERER_INTEROP_EGL || - !interop->egl.display || !interop->egl.config || - !interop->egl.shareContext || !interop->egl.sharedTextureImport) - { - DEBUG_ERROR("The network transport requires EGL shared textures"); - return false; - } - - this->rendererAttached = true; - return true; + return networkVideoAttachRenderer(this->video, interop); } static void networkDetachRenderer(LG_Transport * this) { - this->rendererAttached = false; + networkVideoDetachRenderer(this->video); } static LG_TransportStatus networkNextFrame(LG_Transport * this, bool useDMA, LG_TransportFrame * frame) { - (void)useDMA; - (void)frame; - - if (!this->rendererAttached) - return LG_TRANSPORT_ERROR; - if (!networkSessionValid(this)) - return LG_TRANSPORT_DISCONNECTED; - - lgWaitEvent(this->frameEvent, 100); - return networkSessionValid(this) ? - LG_TRANSPORT_UNAVAILABLE : LG_TRANSPORT_DISCONNECTED; + return networkVideoNextFrame(this->video, useDMA, frame); } static void networkGetFrameTiming(LG_Transport * this, const LG_TransportFrame * frame, LG_TransportFrameTiming * timing) { - (void)this; - (void)frame; - memset(timing, 0, sizeof(*timing)); + networkVideoGetFrameTiming(this->video, frame, timing); } static void networkReleaseFrame(LG_Transport * this, LG_TransportFrame * frame) { - (void)this; - (void)frame; + networkVideoReleaseFrame(this->video, frame); } static void networkCancelFrameWait(LG_Transport * this) { - lgSignalEvent(this->frameEvent); + networkVideoCancelFrameWait(this->video); } static void networkStopFrame(LG_Transport * this) { - lgSignalEvent(this->frameEvent); + networkVideoStopFrame(this->video); } static LG_TransportStatus networkNextPointer(LG_Transport * this, diff --git a/client/transports/Network/video.c b/client/transports/Network/video.c new file mode 100644 index 00000000..c6ebcf09 --- /dev/null +++ b/client/transports/Network/video.c @@ -0,0 +1,1671 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "video.h" + +#include "video/decoder.h" +#include "video/pyrowave.h" + +#include "common/debug.h" +#include "common/event.h" +#include "common/locking.h" +#include "common/time.h" + +#include +#include + +#include +#include +#include +#include + +#define NETWORK_VIDEO_TIMER_MS 10U +#define NETWORK_VIDEO_SUBSCRIBE_RETRY_MS 500U +#define NETWORK_VIDEO_FEEDBACK_INTERVAL_MS 16U +#define NETWORK_VIDEO_KEYFRAME_RETRY_MS 100U +#define NETWORK_VIDEO_MAX_WIDTH 8192U +#define NETWORK_VIDEO_MAX_HEIGHT 8192U +#define NETWORK_VIDEO_MAX_FRAME_LENGTH \ + LG_NET_VIDEO_MAX_FRAME_LENGTH +#define NETWORK_VIDEO_MAX_FRAGMENTS LG_NET_VIDEO_MAX_FRAGMENTS +#define NETWORK_VIDEO_MAX_BLOCKS LG_NET_VIDEO_MAX_BLOCKS +#define NETWORK_VIDEO_MAX_FRAME_LATENCY_MS 50U +#define NETWORK_VIDEO_MAX_FRAME_RATE 240U +#define NETWORK_VIDEO_SCHEDULE_CACHE 8U +#define NETWORK_VIDEO_MAX_LEASES 8U + +#define NETWORK_VIDEO_FORBIDDEN_FLAGS \ + (LG_NET_ENVELOPE_ACK_REQUIRED | LG_NET_ENVELOPE_ERROR | \ + LG_NET_ENVELOPE_EARLY_DATA) + +struct NetworkVideoBlockCoverage +{ + uint32_t base; + uint32_t length; + uint32_t received; + bool known; + bool start; + bool end; +}; + +struct NetworkVideoAssembly +{ + bool active; + uint64_t configEpoch; + uint64_t frameID; + uint64_t firstReceiveNs; + uint64_t deadlineNs; + uint32_t frameLength; + uint32_t fragmentCount; + uint32_t blockCount; + uint32_t receivedBytes; + + uint8_t * data; + uint64_t * coverage; + size_t coverageWords; + uint64_t * fragmentSeen; + uint32_t * fragmentOffsets; + uint32_t * fragmentLengths; + uint32_t * fragmentBlocks; + + struct NetworkVideoBlockCoverage * blocks; + + bool scheduleValid; + LGNetVideoSchedule schedule; +}; + +struct NetworkVideoScheduleEntry +{ + bool valid; + LGNetVideoSchedule schedule; +}; + +struct NetworkVideoOutput +{ + struct NetworkVideoOutput * next; + + NetworkVideoDecodedFrame decoded; + uint64_t frameID; + uint64_t handle; + uint64_t epoch; + uint64_t timestamp; + uint64_t captureTimestampNs; + uint64_t presentationTimestampNs; + uint64_t receiveTimeNs; + bool releaseRequested; +}; + +struct NetworkVideoEncoded +{ + LGNetVideoStreamConfig config; + const uint8_t * data; + uint8_t * ownedData; + uint32_t length; + uint32_t blockCount; + uint64_t frameID; + uint64_t captureTimestampNs; + uint64_t presentationTimestampNs; + uint64_t receiveTimeNs; + LGNetVideoFrameFlags flags; + bool scheduleValid; + LGNetVideoSchedule schedule; +}; + +struct NetworkVideo +{ + NetworkConnection * connection; + NetworkVideoDecoder * decoder; + LGTimer * timer; + LGEvent * frameEvent; + LG_Lock stateLock; + LG_Lock callbackLock; + + bool destroying; + bool connected; + bool serviceAvailable; + bool datagramsAvailable; + bool rendererAttached; + bool wireConfigured; + bool decoderReady; + bool subscribePending; + bool waitCancelled; + uint16_t serviceVersion; + + uint64_t subscriberID; + uint64_t requestSerial; + uint64_t subscribeRequestID; + uint64_t nextSubscribeNs; + uint64_t nextFeedbackNs; + uint64_t nextKeyframeNs; + + LGNetVideoStreamConfig config; + uint32_t localConfigGeneration; + uint64_t highestSeenFrameID; + uint64_t lastCompleteFrameID; + uint64_t lastDecodedFrameID; + uint64_t statusSequence; + uint32_t lostFrames; + uint32_t lostFragments; + + LGNetVideoFeedbackFlags feedbackFlags; + bool feedbackPending; + bool keyframePending; + LGNetVideoControlReason keyframeReason; + uint64_t keyframeFrameID; + + struct NetworkVideoAssembly assembly; + struct NetworkVideoScheduleEntry schedules[NETWORK_VIDEO_SCHEDULE_CACHE]; + struct NetworkVideoOutput * output; + struct NetworkVideoOutput * leases; + uint64_t leaseSerial; + uint32_t leaseCount; + + LG_VideoComponentStatus status; + NetworkVideoStatusFn statusCallback; + void * statusOpaque; +}; + +enum NetworkVideoCoverageState +{ + NETWORK_VIDEO_COVERAGE_EMPTY, + NETWORK_VIDEO_COVERAGE_FULL, + NETWORK_VIDEO_COVERAGE_PARTIAL, +}; + +static uint64_t networkVideoNext64(uint64_t value) +{ + ++value; + return value ? value : 1U; +} + +static uint32_t networkVideoSaturatingIncrement(uint32_t value) +{ + return value == UINT32_MAX ? value : value + 1U; +} + +static bool networkVideoVersionSupported(uint16_t version) +{ + return version >= LG_NET_VIDEO_BLOCK_FRAGMENTATION_INTRODUCED_SERVICE_VERSION && + version <= LG_NET_VIDEO_VERSION_CURRENT; +} + +static bool networkVideoConfigEqual( + const LGNetVideoStreamConfig * left, + const LGNetVideoStreamConfig * right) +{ + return left->streamID == right->streamID && + left->codec == right->codec && + left->codecVersion == right->codecVersion && + left->configEpoch == right->configEpoch && + left->width == right->width && + left->height == right->height && + left->refreshNumerator == right->refreshNumerator && + left->refreshDenominator == right->refreshDenominator && + left->pixelFormat == right->pixelFormat && + left->chromaSubsampling == right->chromaSubsampling && + left->colorPrimaries == right->colorPrimaries && + left->colorTransfer == right->colorTransfer && + left->colorMatrix == right->colorMatrix && + left->colorRange == right->colorRange && + left->planeCount == right->planeCount && + left->bitDepth == right->bitDepth && + left->flags == right->flags && + left->maxFrameLength == right->maxFrameLength && + left->maxFragmentLength == right->maxFragmentLength && + left->maxFrameLatencyMs == right->maxFrameLatencyMs; +} + +static void networkVideoFreeOutput(struct NetworkVideoOutput ** output) +{ + if (!output || !*output) + return; + networkVideoDecoderRelease(&(*output)->decoded); + free(*output); + *output = NULL; +} + +static struct NetworkVideoOutput * networkVideoFindLeaseNL( + NetworkVideo * video, uint64_t handle) +{ + for (struct NetworkVideoOutput * output = video->leases; + output; output = output->next) + if (output->handle == handle) + return output; + return NULL; +} + +static struct NetworkVideoOutput * networkVideoTakeLeaseNL( + NetworkVideo * video, uint64_t handle) +{ + struct NetworkVideoOutput ** link = &video->leases; + while (*link && (*link)->handle != handle) + link = &(*link)->next; + if (!*link) + return NULL; + + struct NetworkVideoOutput * output = *link; + *link = output->next; + output->next = NULL; + if (video->leaseCount) + --video->leaseCount; + return output; +} + +static void networkVideoDrainLeasesNL(NetworkVideo * video) +{ + struct NetworkVideoOutput ** link = &video->leases; + while (*link) + { + struct NetworkVideoOutput * output = *link; + if (!output->releaseRequested) + { + link = &output->next; + continue; + } + + *link = output->next; + output->next = NULL; + if (video->leaseCount) + --video->leaseCount; + networkVideoFreeOutput(&output); + } +} + +static void networkVideoFreeLeasesNL(NetworkVideo * video) +{ + while (video->leases) + { + struct NetworkVideoOutput * output = video->leases; + video->leases = output->next; + output->next = NULL; + networkVideoFreeOutput(&output); + } + video->leaseCount = 0; +} + +static void networkVideoReleaseLease(void * opaque, uint64_t handle) +{ + NetworkVideo * video = opaque; + if (!video || !handle) + return; + + LG_LOCK(video->stateLock); + struct NetworkVideoOutput * output = + networkVideoFindLeaseNL(video, handle); + if (output) + output->releaseRequested = true; + LG_UNLOCK(video->stateLock); + lgSignalEvent(video->frameEvent); +} + +static void networkVideoClearAssembly(struct NetworkVideoAssembly * assembly) +{ + if (!assembly) + return; + free(assembly->blocks); + free(assembly->fragmentLengths); + free(assembly->fragmentOffsets); + free(assembly->fragmentBlocks); + free(assembly->fragmentSeen); + free(assembly->coverage); + free(assembly->data); + memset(assembly, 0, sizeof(*assembly)); +} + +static bool networkVideoSetStatusNL(NetworkVideo * video, bool available, + LG_TransportStatus reason) +{ + if (video->status.available == available && video->status.reason == reason) + return false; + if (video->status.available != available) + { + video->status.epoch = networkVideoNext64(video->status.epoch); + if (video->status.epoch == 0) + video->status.epoch = 1; + } + video->status.available = available; + video->status.reason = available ? LG_TRANSPORT_OK : reason; + return true; +} + +static bool networkVideoUpdateStatusNL(NetworkVideo * video) +{ + const bool available = video->connected && video->serviceAvailable && + video->rendererAttached && video->wireConfigured && video->decoderReady; + LG_TransportStatus reason = LG_TRANSPORT_UNAVAILABLE; + if (!video->connected) + reason = LG_TRANSPORT_DISCONNECTED; + else if (!video->serviceAvailable) + reason = LG_TRANSPORT_INVALID_VERSION; + return networkVideoSetStatusNL(video, available, reason); +} + +static void networkVideoNotifyStatus(NetworkVideo * video) +{ + LG_VideoComponentStatus status; + NetworkVideoStatusFn callback; + void * opaque; + + LG_LOCK(video->callbackLock); + LG_LOCK(video->stateLock); + callback = video->statusCallback; + opaque = video->statusOpaque; + status = video->status; + LG_UNLOCK(video->stateLock); + if (callback) + callback(opaque, &status); + LG_UNLOCK(video->callbackLock); +} + +static void networkVideoQueueFeedbackNL(NetworkVideo * video, + LGNetVideoFeedbackFlags flags) +{ + video->feedbackFlags |= flags; + video->feedbackPending = true; +} + +static void networkVideoRequestKeyframeNL(NetworkVideo * video, + LGNetVideoControlReason reason, uint64_t frameID) +{ + video->keyframePending = true; + video->keyframeReason = reason; + video->keyframeFrameID = frameID; + networkVideoQueueFeedbackNL(video, + LG_NET_VIDEO_FEEDBACK_REQUEST_KEYFRAME); +} + +static bool networkVideoPacketFlags( + const LGNetPacketView * packet, bool datagram) +{ + if (!packet || !(packet->envelope.flags & LG_NET_ENVELOPE_FINAL) || + (packet->envelope.flags & NETWORK_VIDEO_FORBIDDEN_FLAGS)) + return false; + const LGNetEnvelopeFlags transport = packet->envelope.flags & + (LG_NET_ENVELOPE_RELIABLE | LG_NET_ENVELOPE_DATAGRAM); + if (datagram) + return transport == LG_NET_ENVELOPE_DATAGRAM; + + /* Bulk stream envelopes deliberately carry neither delivery flag. The + * connection has already authenticated and route-validated the bound + * stream before invoking this endpoint. */ + return transport == LG_NET_ENVELOPE_RELIABLE || !transport; +} + +static bool networkVideoResponseNL(NetworkVideo * video, + const LGNetPacketView * packet) +{ + const bool response = + (packet->envelope.flags & LG_NET_ENVELOPE_RESPONSE) != 0; + if (!response) + return packet->envelope.requestID == 0; + if (!packet->envelope.requestID || !video->requestSerial || + packet->envelope.requestID > video->requestSerial) + return false; + if (packet->envelope.requestID == video->subscribeRequestID) + video->subscribeRequestID = 0; + return true; +} + +static bool networkVideoUnsolicited(const LGNetPacketView * packet) +{ + return !(packet->envelope.flags & LG_NET_ENVELOPE_RESPONSE) && + !packet->envelope.requestID; +} + +static LG_TransportStatus networkVideoSendReliableNL(NetworkVideo * video, + uint16_t messageType, LGNetEnvelopeFlags flags, uint64_t componentEpoch, + uint64_t requestID, const void * payload, uint32_t payloadLength) +{ + return networkConnectionSend(video->connection, LG_NET_SERVICE_VIDEO, + messageType, video->serviceVersion, flags | LG_NET_ENVELOPE_FINAL, + componentEpoch, requestID, payload, payloadLength); +} + +static LG_TransportStatus networkVideoSendDatagramNL(NetworkVideo * video, + uint16_t messageType, uint64_t componentEpoch, + const void * payload, uint32_t payloadLength) +{ + return networkConnectionSendDatagram(video->connection, + LG_NET_SERVICE_VIDEO, messageType, video->serviceVersion, 0, + componentEpoch, 0, payload, payloadLength, true); +} + +static void networkVideoSendSubscribeNL(NetworkVideo * video, uint64_t now) +{ + if (!video->connected || !video->serviceAvailable || + video->wireConfigured || video->destroying || + (!video->subscribePending && now < video->nextSubscribeNs)) + return; + + const LGNetVideoSubscribe subscribe = + { + .subscriptionID = video->subscriberID, + .preferredCodec = LG_NET_VIDEO_CODEC_PYROWAVE, + .flags = LG_NET_VIDEO_SUBSCRIBE_LOW_LATENCY | + (video->datagramsAvailable ? + LG_NET_VIDEO_SUBSCRIBE_ALLOW_DATAGRAMS : 0U), + .maxWidth = NETWORK_VIDEO_MAX_WIDTH, + .maxHeight = NETWORK_VIDEO_MAX_HEIGHT, + .maxFrameLength = NETWORK_VIDEO_MAX_FRAME_LENGTH, + .maxFrameLatencyMs = NETWORK_VIDEO_MAX_FRAME_LATENCY_MS, + .maxFrameRate = NETWORK_VIDEO_MAX_FRAME_RATE, + }; + uint8_t payload[LG_NET_VIDEO_SUBSCRIBE_WIRE_SIZE]; + if (!lgNetVideoSubscribeEncode(payload, sizeof(payload), &subscribe)) + return; + + if (!video->subscribeRequestID) + { + video->requestSerial = networkVideoNext64(video->requestSerial); + video->subscribeRequestID = video->requestSerial; + } + (void)networkVideoSendReliableNL(video, + LG_NET_VIDEO_MESSAGE_SUBSCRIBE, LG_NET_ENVELOPE_ACK_REQUIRED, + 0, video->subscribeRequestID, payload, sizeof(payload)); + video->subscribePending = false; + video->nextSubscribeNs = now + + (uint64_t)NETWORK_VIDEO_SUBSCRIBE_RETRY_MS * UINT64_C(1000000); +} + +static void networkVideoSendFeedbackNL(NetworkVideo * video, uint64_t now) +{ + if (!video->feedbackPending || !video->wireConfigured || + now < video->nextFeedbackNs) + return; + + const LGNetVideoFeedback feedback = + { + .streamID = video->config.streamID, + .flags = video->feedbackFlags, + .lastCompleteFrameID = video->lastCompleteFrameID, + .highestSeenFrameID = video->highestSeenFrameID, + .lastDecodedFrameID = video->lastDecodedFrameID, + .lostFrames = video->lostFrames, + .lostFragments = video->lostFragments, + .receiveQueueUs = 0, + .decodeQueueUs = 0, + .roundTripUs = 0, + }; + uint8_t payload[LG_NET_VIDEO_FEEDBACK_WIRE_SIZE]; + if (!lgNetVideoFeedbackEncode(payload, sizeof(payload), &feedback)) + return; + + const LG_TransportStatus status = video->datagramsAvailable ? + networkVideoSendDatagramNL(video, LG_NET_VIDEO_MESSAGE_FEEDBACK, + video->config.configEpoch, payload, sizeof(payload)) : + networkVideoSendReliableNL(video, LG_NET_VIDEO_MESSAGE_FEEDBACK, 0, + video->config.configEpoch, 0, payload, sizeof(payload)); + if (status == LG_TRANSPORT_OK) + { + video->feedbackFlags = 0; + video->feedbackPending = false; + } + video->nextFeedbackNs = now + + (uint64_t)NETWORK_VIDEO_FEEDBACK_INTERVAL_MS * UINT64_C(1000000); +} + +static void networkVideoSendKeyframeNL(NetworkVideo * video, uint64_t now) +{ + if (!video->keyframePending || !video->wireConfigured || + now < video->nextKeyframeNs) + return; + + const LGNetVideoControl control = + { + .streamID = video->config.streamID, + .reason = video->keyframeReason, + .configEpoch = video->config.configEpoch, + .frameID = video->keyframeFrameID, + }; + uint8_t payload[LG_NET_VIDEO_CONTROL_WIRE_SIZE]; + if (!lgNetVideoControlEncode(payload, sizeof(payload), &control)) + return; + + const LG_TransportStatus status = video->datagramsAvailable ? + networkVideoSendDatagramNL(video, + LG_NET_VIDEO_MESSAGE_KEYFRAME_REQUEST, + video->config.configEpoch, payload, sizeof(payload)) : + networkVideoSendReliableNL(video, + LG_NET_VIDEO_MESSAGE_KEYFRAME_REQUEST, 0, + video->config.configEpoch, 0, payload, sizeof(payload)); + if (status == LG_TRANSPORT_OK) + video->keyframePending = false; + + video->nextKeyframeNs = now + + (uint64_t)NETWORK_VIDEO_KEYFRAME_RETRY_MS * UINT64_C(1000000); +} + +static uint64_t networkVideoCoverageMask( + uint64_t word, uint64_t begin, uint64_t end) +{ + const unsigned low = begin > word * 64U ? + (unsigned)(begin - word * 64U) : 0U; + const unsigned high = end < (word + 1U) * 64U ? + (unsigned)(end - word * 64U) : 64U; + const uint64_t lower = low ? (UINT64_C(1) << low) - 1U : 0U; + const uint64_t upper = high == 64U ? UINT64_MAX : + (UINT64_C(1) << high) - 1U; + return upper & ~lower; +} + +static enum NetworkVideoCoverageState networkVideoCoverage( + const struct NetworkVideoAssembly * assembly, + uint32_t offset, uint32_t length) +{ + const uint64_t begin = offset; + const uint64_t end = begin + length; + const uint64_t first = begin / 64U; + const uint64_t last = (end - 1U) / 64U; + + bool any = false; + bool all = true; + for (uint64_t word = first; word <= last; ++word) + { + const uint64_t mask = networkVideoCoverageMask(word, begin, end); + const uint64_t set = assembly->coverage[word] & mask; + + any |= set != 0; + all &= set == mask; + } + if (all) + return NETWORK_VIDEO_COVERAGE_FULL; + return any ? NETWORK_VIDEO_COVERAGE_PARTIAL : NETWORK_VIDEO_COVERAGE_EMPTY; +} + +static void networkVideoSetCoverage(struct NetworkVideoAssembly * assembly, + uint32_t offset, uint32_t length) +{ + const uint64_t begin = offset; + const uint64_t end = begin + length; + const uint64_t first = begin / 64U; + const uint64_t last = (end - 1U) / 64U; + for (uint64_t word = first; word <= last; ++word) + assembly->coverage[word] |= networkVideoCoverageMask(word, begin, end); +} + +static bool networkVideoFragmentSeen( + const struct NetworkVideoAssembly * assembly, uint32_t index) +{ + return (assembly->fragmentSeen[index >> 6] >> (index & 63U)) & 1U; +} + +static void networkVideoSetFragmentSeen( + struct NetworkVideoAssembly * assembly, uint32_t index) +{ + assembly->fragmentSeen[index >> 6] |= UINT64_C(1) << (index & 63U); +} + +static bool networkVideoFindScheduleNL(NetworkVideo * video, + uint64_t configEpoch, uint64_t frameID, LGNetVideoSchedule * schedule) +{ + struct NetworkVideoScheduleEntry * entry = + &video->schedules[frameID % NETWORK_VIDEO_SCHEDULE_CACHE]; + if (!entry->valid || entry->schedule.configEpoch != configEpoch || + entry->schedule.frameID != frameID) + return false; + *schedule = entry->schedule; + entry->valid = false; + return true; +} + +static bool networkVideoAllocateAssemblyNL(NetworkVideo * video, + const LGNetVideoFragment * fragment, uint64_t now) +{ + if (fragment->frameLength > NETWORK_VIDEO_MAX_FRAME_LENGTH || + fragment->frameLength > video->config.maxFrameLength || + fragment->fragmentCount > NETWORK_VIDEO_MAX_FRAGMENTS || + fragment->blockCount > NETWORK_VIDEO_MAX_BLOCKS) + return false; + + struct NetworkVideoAssembly assembly = + { + .active = true, + .configEpoch = fragment->configEpoch, + .frameID = fragment->frameID, + .firstReceiveNs = now, + .deadlineNs = now + + (uint64_t)video->config.maxFrameLatencyMs * UINT64_C(1000000), + .frameLength = fragment->frameLength, + .fragmentCount = fragment->fragmentCount, + .blockCount = fragment->blockCount, + .coverageWords = ((size_t)fragment->frameLength + 63U) / 64U, + }; + const size_t fragmentWords = + ((size_t)fragment->fragmentCount + 63U) / 64U; + assembly.data = malloc(fragment->frameLength); + assembly.coverage = calloc(assembly.coverageWords, + sizeof(*assembly.coverage)); + assembly.fragmentSeen = calloc(fragmentWords, + sizeof(*assembly.fragmentSeen)); + assembly.fragmentOffsets = malloc( + (size_t)fragment->fragmentCount * sizeof(*assembly.fragmentOffsets)); + assembly.fragmentLengths = malloc( + (size_t)fragment->fragmentCount * sizeof(*assembly.fragmentLengths)); + assembly.fragmentBlocks = malloc( + (size_t)fragment->fragmentCount * sizeof(*assembly.fragmentBlocks)); + assembly.blocks = calloc(fragment->blockCount, + sizeof(*assembly.blocks)); + if (!assembly.data || !assembly.coverage || !assembly.fragmentSeen || + !assembly.fragmentOffsets || !assembly.fragmentLengths || + !assembly.fragmentBlocks || !assembly.blocks) + { + networkVideoClearAssembly(&assembly); + return false; + } + + assembly.scheduleValid = networkVideoFindScheduleNL(video, + fragment->configEpoch, fragment->frameID, &assembly.schedule); + video->assembly = assembly; + return true; +} + +static bool networkVideoAssemblyComplete( + const struct NetworkVideoAssembly * assembly) +{ + if (assembly->receivedBytes != assembly->frameLength) + return false; + for (uint32_t index = 0; index < assembly->blockCount; ++index) + { + const struct NetworkVideoBlockCoverage * block = &assembly->blocks[index]; + if (!block->known || !block->start || !block->end || + block->received != block->length) + return false; + } + return true; +} + +static bool networkVideoAddFragmentNL(NetworkVideo * video, + const LGNetVideoFragment * fragment, uint64_t now, + struct NetworkVideoEncoded * encoded) +{ + struct NetworkVideoAssembly * assembly = &video->assembly; + if (!assembly->active || fragment->frameID > assembly->frameID) + { + if (assembly->active) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoClearAssembly(assembly); + } + if (!networkVideoAllocateAssemblyNL(video, fragment, now)) + return false; + assembly = &video->assembly; + } + else if (fragment->frameID < assembly->frameID) + return true; + + if (assembly->configEpoch != fragment->configEpoch || + assembly->frameLength != fragment->frameLength || + assembly->fragmentCount != fragment->fragmentCount || + assembly->blockCount != fragment->blockCount) + return false; + + if (networkVideoFragmentSeen(assembly, fragment->fragmentIndex)) + { + const uint32_t offset = + assembly->fragmentOffsets[fragment->fragmentIndex]; + const uint32_t length = + assembly->fragmentLengths[fragment->fragmentIndex]; + return offset == fragment->frameOffset && + length == fragment->payloadLength && + assembly->fragmentBlocks[fragment->fragmentIndex] == + fragment->blockIndex && + !memcmp(assembly->data + offset, + fragment->payload, fragment->payloadLength); + } + + struct NetworkVideoBlockCoverage * block = + &assembly->blocks[fragment->blockIndex]; + const uint32_t blockBase = + fragment->frameOffset - fragment->blockOffset; + if (!block->known) + { + block->base = blockBase; + block->length = fragment->blockLength; + block->known = true; + } + else if (block->base != blockBase || + block->length != fragment->blockLength) + return false; + + const enum NetworkVideoCoverageState coverage = networkVideoCoverage( + assembly, fragment->frameOffset, fragment->payloadLength); + if (coverage == NETWORK_VIDEO_COVERAGE_PARTIAL) + return false; + if (coverage == NETWORK_VIDEO_COVERAGE_FULL) + { + if (memcmp(assembly->data + fragment->frameOffset, + fragment->payload, fragment->payloadLength)) + return false; + networkVideoSetFragmentSeen(assembly, fragment->fragmentIndex); + assembly->fragmentOffsets[fragment->fragmentIndex] = + fragment->frameOffset; + assembly->fragmentLengths[fragment->fragmentIndex] = + fragment->payloadLength; + assembly->fragmentBlocks[fragment->fragmentIndex] = + fragment->blockIndex; + return true; + } + + memcpy(assembly->data + fragment->frameOffset, + fragment->payload, fragment->payloadLength); + networkVideoSetCoverage(assembly, + fragment->frameOffset, fragment->payloadLength); + networkVideoSetFragmentSeen(assembly, fragment->fragmentIndex); + assembly->fragmentOffsets[fragment->fragmentIndex] = fragment->frameOffset; + assembly->fragmentLengths[fragment->fragmentIndex] = fragment->payloadLength; + assembly->fragmentBlocks[fragment->fragmentIndex] = fragment->blockIndex; + + assembly->receivedBytes += fragment->payloadLength; + block->received += fragment->payloadLength; + block->start |= + (fragment->flags & LG_NET_VIDEO_FRAGMENT_BLOCK_START) != 0; + block->end |= + (fragment->flags & LG_NET_VIDEO_FRAGMENT_BLOCK_END) != 0; + + if (!networkVideoAssemblyComplete(assembly)) + return true; + + *encoded = (struct NetworkVideoEncoded) + { + .config = video->config, + .data = assembly->data, + .ownedData = assembly->data, + .length = assembly->frameLength, + .blockCount = assembly->blockCount, + .frameID = assembly->frameID, + .captureTimestampNs = assembly->scheduleValid ? + assembly->schedule.captureTimestampNs : now, + .presentationTimestampNs = assembly->scheduleValid ? + assembly->schedule.presentationTimestampNs : now, + .receiveTimeNs = now - assembly->firstReceiveNs, + .flags = LG_NET_VIDEO_FRAME_KEYFRAME, + .scheduleValid = assembly->scheduleValid, + .schedule = assembly->schedule, + }; + assembly->data = NULL; + networkVideoClearAssembly(assembly); + return true; +} + +static void networkVideoDecodeFailure(NetworkVideo * video, + uint64_t configEpoch, uint64_t frameID, NetworkVideoDecodeResult result) +{ + bool notify = false; + LG_LOCK(video->stateLock); + if (video->wireConfigured && + video->config.configEpoch == configEpoch) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + + const LGNetVideoFeedbackFlags flags = result == + NETWORK_VIDEO_DECODE_ERROR ? + LG_NET_VIDEO_FEEDBACK_DECODER_STALLED : 0; + networkVideoQueueFeedbackNL(video, + flags | LG_NET_VIDEO_FEEDBACK_REQUEST_KEYFRAME); + networkVideoRequestKeyframeNL(video, + result == NETWORK_VIDEO_DECODE_CORRUPT ? + LG_NET_VIDEO_CONTROL_CORRUPT : LG_NET_VIDEO_CONTROL_STALLED, + frameID); + if (result == NETWORK_VIDEO_DECODE_ERROR || + result == NETWORK_VIDEO_DECODE_UNSUPPORTED) + { + video->decoderReady = false; + notify = networkVideoUpdateStatusNL(video); + } + } + LG_UNLOCK(video->stateLock); + if (notify) + networkVideoNotifyStatus(video); +} + +static void networkVideoDecode(NetworkVideo * video, + struct NetworkVideoEncoded * encoded) +{ + uint8_t sequence; + const NetworkPyroWaveResult validation = networkPyroWaveValidate( + &encoded->config, encoded->data, encoded->length, + encoded->blockCount, &sequence); + (void)sequence; + if (validation != NETWORK_PYROWAVE_OK) + { + networkVideoDecodeFailure(video, encoded->config.configEpoch, + encoded->frameID, validation == NETWORK_PYROWAVE_UNSUPPORTED ? + NETWORK_VIDEO_DECODE_UNSUPPORTED : NETWORK_VIDEO_DECODE_CORRUPT); + free(encoded->ownedData); + return; + } + + NetworkVideoDecodedFrame decoded; + const NetworkVideoDecodeResult result = networkVideoDecoderDecode( + video->decoder, &encoded->config, encoded->data, + encoded->length, &decoded); + free(encoded->ownedData); + if (result != NETWORK_VIDEO_DECODE_OK) + { + networkVideoDecodeFailure(video, encoded->config.configEpoch, + encoded->frameID, result); + return; + } + + struct NetworkVideoOutput * output = calloc(1, sizeof(*output)); + if (!output) + { + networkVideoDecoderRelease(&decoded); + networkVideoDecodeFailure(video, encoded->config.configEpoch, + encoded->frameID, NETWORK_VIDEO_DECODE_ERROR); + return; + } + output->decoded = decoded; + output->frameID = encoded->frameID; + output->timestamp = encoded->presentationTimestampNs; + output->captureTimestampNs = encoded->captureTimestampNs; + output->presentationTimestampNs = encoded->presentationTimestampNs; + output->receiveTimeNs = encoded->receiveTimeNs; + LG_LOCK(video->stateLock); + if (!video->wireConfigured || + video->config.configEpoch != encoded->config.configEpoch || + !video->rendererAttached || !video->decoderReady || + output->frameID < video->highestSeenFrameID || + output->frameID <= video->lastDecodedFrameID) + { + LG_UNLOCK(video->stateLock); + networkVideoFreeOutput(&output); + return; + } + + if (video->output) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoFreeOutput(&video->output); + } + video->lastCompleteFrameID = output->frameID; + video->lastDecodedFrameID = output->frameID; + output->decoded.format.version = video->localConfigGeneration; + output->epoch = video->status.epoch; + video->output = output; + networkVideoQueueFeedbackNL(video, 0); + LG_UNLOCK(video->stateLock); + lgSignalEvent(video->frameEvent); +} + +static bool networkVideoHandleConfig(NetworkVideo * video, + const LGNetPacketView * packet) +{ + LGNetVideoStreamConfig config; + if (lgNetVideoStreamConfigDecode(&config, + packet->payload, packet->payloadLength) != LG_NET_PARSE_OK || + (packet->envelope.componentEpoch && + packet->envelope.componentEpoch != config.configEpoch) || + config.width > NETWORK_VIDEO_MAX_WIDTH || + config.height > NETWORK_VIDEO_MAX_HEIGHT || + config.maxFrameLength > NETWORK_VIDEO_MAX_FRAME_LENGTH || + config.maxFragmentLength > LG_NET_VIDEO_MAX_FRAGMENT_LENGTH || + config.maxFrameLatencyMs > NETWORK_VIDEO_MAX_FRAME_LATENCY_MS || + !networkPyroWaveProfileValid(&config)) + return false; + + bool attached; + bool decoderReady; + LG_LOCK(video->stateLock); + if (!video->connected || !video->serviceAvailable) + { + LG_UNLOCK(video->stateLock); + return true; + } + if (!networkVideoResponseNL(video, packet)) + { + LG_UNLOCK(video->stateLock); + return false; + } + if ((config.flags & LG_NET_VIDEO_STREAM_DATAGRAMS) && + !video->datagramsAvailable) + { + LG_UNLOCK(video->stateLock); + return false; + } + if (video->wireConfigured && + video->config.streamID == config.streamID && + video->config.configEpoch == config.configEpoch && + !networkVideoConfigEqual(&video->config, &config)) + { + LG_UNLOCK(video->stateLock); + return false; + } + const bool changed = !video->wireConfigured || + video->config.streamID != config.streamID || + video->config.configEpoch != config.configEpoch; + if (changed) + { + networkVideoClearAssembly(&video->assembly); + networkVideoFreeOutput(&video->output); + memset(video->schedules, 0, sizeof(video->schedules)); + video->highestSeenFrameID = 0; + video->lastCompleteFrameID = 0; + video->lastDecodedFrameID = 0; + video->statusSequence = 0; + video->lostFrames = 0; + video->lostFragments = 0; + video->decoderReady = false; + video->localConfigGeneration++; + if (!video->localConfigGeneration) + video->localConfigGeneration = 1; + } + video->config = config; + video->wireConfigured = true; + video->subscribePending = false; + video->subscribeRequestID = 0; + attached = video->rendererAttached; + decoderReady = video->decoderReady; + LG_UNLOCK(video->stateLock); + + const NetworkVideoDecodeResult result = decoderReady ? + NETWORK_VIDEO_DECODE_OK : attached ? + networkVideoDecoderConfigure(video->decoder, &config) : + NETWORK_VIDEO_DECODE_UNSUPPORTED; + bool notify; + LG_LOCK(video->stateLock); + if (video->wireConfigured && video->rendererAttached && + networkVideoConfigEqual(&video->config, &config)) + { + video->decoderReady = result == NETWORK_VIDEO_DECODE_OK; + notify = false; + if (changed && video->status.available && + result == NETWORK_VIDEO_DECODE_OK) + { + video->status.epoch = networkVideoNext64(video->status.epoch); + notify = true; + } + notify |= networkVideoUpdateStatusNL(video); + if (result != NETWORK_VIDEO_DECODE_OK && attached) + { + networkVideoQueueFeedbackNL(video, + LG_NET_VIDEO_FEEDBACK_DECODER_STALLED | + LG_NET_VIDEO_FEEDBACK_REQUEST_KEYFRAME); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_STALLED, 0); + } + } + else + notify = false; + LG_UNLOCK(video->stateLock); + if (notify) + networkVideoNotifyStatus(video); + return true; +} + +static bool networkVideoHandleFrame(NetworkVideo * video, + const LGNetPacketView * packet) +{ + LGNetVideoFrame frame; + if (lgNetVideoFrameDecode(&frame, + packet->payload, packet->payloadLength) != LG_NET_PARSE_OK) + return false; + + if (frame.flags & LG_NET_VIDEO_FRAME_HAS_CHECKSUM) + { + uint32_t checksum; + if (!lgNetCRC32C(frame.data, frame.encodedLength, &checksum) || + checksum != frame.checksum) + { + LG_LOCK(video->stateLock); + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_CORRUPT, frame.frameID); + LG_UNLOCK(video->stateLock); + return true; + } + } + + const uint64_t receiveStart = nanotime(); + struct NetworkVideoEncoded encoded; + LG_LOCK(video->stateLock); + if (!video->wireConfigured || frame.streamID != video->config.streamID || + frame.configEpoch != video->config.configEpoch || + frame.codec != video->config.codec || + frame.encodedLength > video->config.maxFrameLength || + frame.deadlineMs > video->config.maxFrameLatencyMs || + (packet->envelope.componentEpoch && + packet->envelope.componentEpoch != frame.configEpoch)) + { + LG_UNLOCK(video->stateLock); + return true; + } + if (frame.frameID <= video->lastDecodedFrameID || + frame.frameID < video->highestSeenFrameID) + { + LG_UNLOCK(video->stateLock); + return true; + } + if (video->assembly.active && + video->assembly.frameID <= frame.frameID) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoClearAssembly(&video->assembly); + } + if (frame.frameID > video->highestSeenFrameID) + { + video->highestSeenFrameID = frame.frameID; + if (video->output && video->output->frameID < frame.frameID) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoFreeOutput(&video->output); + } + } + encoded = (struct NetworkVideoEncoded) + { + .config = video->config, + .data = frame.data, + .length = frame.encodedLength, + .blockCount = frame.blockCount, + .frameID = frame.frameID, + .captureTimestampNs = frame.captureTimestampNs, + .presentationTimestampNs = frame.presentationTimestampNs, + .receiveTimeNs = nanotime() - receiveStart, + .flags = frame.flags, + }; + encoded.scheduleValid = networkVideoFindScheduleNL(video, + frame.configEpoch, frame.frameID, &encoded.schedule); + LG_UNLOCK(video->stateLock); + networkVideoDecode(video, &encoded); + return true; +} + +static bool networkVideoHandleFragment(NetworkVideo * video, + const LGNetPacketView * packet) +{ + LGNetVideoFragment fragment; + if (lgNetVideoFragmentDecode(&fragment, + packet->payload, packet->payloadLength) != LG_NET_PARSE_OK) + return false; + + if (fragment.flags & LG_NET_VIDEO_FRAGMENT_CHECKSUM) + { + uint32_t checksum; + if (!lgNetCRC32C(fragment.payload, + fragment.payloadLength, &checksum) || checksum != fragment.checksum) + { + LG_LOCK(video->stateLock); + video->lostFragments = networkVideoSaturatingIncrement( + video->lostFragments); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_CORRUPT, fragment.frameID); + LG_UNLOCK(video->stateLock); + return true; + } + } + + struct NetworkVideoEncoded encoded = {0}; + bool complete = false; + bool valid; + const uint64_t now = nanotime(); + LG_LOCK(video->stateLock); + if (!video->wireConfigured || + fragment.streamID != video->config.streamID || + fragment.configEpoch != video->config.configEpoch || + (packet->envelope.componentEpoch && + packet->envelope.componentEpoch != fragment.configEpoch) || + fragment.frameID <= video->lastDecodedFrameID || + fragment.frameID < video->highestSeenFrameID) + { + LG_UNLOCK(video->stateLock); + return true; + } + if (fragment.frameID > video->highestSeenFrameID) + { + video->highestSeenFrameID = fragment.frameID; + if (video->output && video->output->frameID < fragment.frameID) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoFreeOutput(&video->output); + } + } + valid = networkVideoAddFragmentNL(video, &fragment, now, &encoded); + complete = valid && encoded.data; + if (!valid) + { + video->lostFragments = networkVideoSaturatingIncrement( + video->lostFragments); + networkVideoClearAssembly(&video->assembly); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_CORRUPT, fragment.frameID); + } + LG_UNLOCK(video->stateLock); + if (complete) + networkVideoDecode(video, &encoded); + return true; +} + +static bool networkVideoHandleSchedule(NetworkVideo * video, + const LGNetPacketView * packet) +{ + LGNetVideoSchedule schedule; + if (lgNetVideoScheduleDecode(&schedule, + packet->payload, packet->payloadLength) != LG_NET_PARSE_OK) + return false; + + LG_LOCK(video->stateLock); + if (!video->wireConfigured || + schedule.streamID != video->config.streamID || + schedule.configEpoch != video->config.configEpoch || + schedule.frameID <= video->lastDecodedFrameID || + (packet->envelope.componentEpoch && + packet->envelope.componentEpoch != schedule.configEpoch)) + { + LG_UNLOCK(video->stateLock); + return true; + } + if (video->assembly.active && + video->assembly.frameID == schedule.frameID) + { + video->assembly.schedule = schedule; + video->assembly.scheduleValid = true; + } + else + { + struct NetworkVideoScheduleEntry * entry = + &video->schedules[schedule.frameID % NETWORK_VIDEO_SCHEDULE_CACHE]; + entry->valid = true; + entry->schedule = schedule; + } + if (schedule.flags & LG_NET_VIDEO_SCHEDULE_REQUEST_KEYFRAME) + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_RECOVERY, schedule.frameID); + LG_UNLOCK(video->stateLock); + return true; +} + +static bool networkVideoHandleStatus(NetworkVideo * video, + const LGNetPacketView * packet) +{ + LGNetVideoStatus status; + if (lgNetVideoStatusDecode(&status, + packet->payload, packet->payloadLength) != LG_NET_PARSE_OK) + return false; + + bool notify = false; + bool reset = false; + const bool response = + (packet->envelope.flags & LG_NET_ENVELOPE_RESPONSE) != 0; + LG_LOCK(video->stateLock); + if (!networkVideoResponseNL(video, packet)) + { + LG_UNLOCK(video->stateLock); + return false; + } + const bool initialError = !video->wireConfigured && response && + status.state == LG_NET_VIDEO_STATE_ERROR; + if ((!video->wireConfigured && !initialError) || + (video->wireConfigured && + (status.streamID != video->config.streamID || + status.configEpoch != video->config.configEpoch)) || + (packet->envelope.componentEpoch && + packet->envelope.componentEpoch != status.configEpoch) || + status.statusSequence <= video->statusSequence) + { + LG_UNLOCK(video->stateLock); + return true; + } + video->statusSequence = status.statusSequence; + if (status.flags & LG_NET_VIDEO_STATUS_KEYFRAME_PENDING) + networkVideoQueueFeedbackNL(video, 0); + if (status.state == LG_NET_VIDEO_STATE_STALLED) + { + networkVideoQueueFeedbackNL(video, + LG_NET_VIDEO_FEEDBACK_DECODER_STALLED | + LG_NET_VIDEO_FEEDBACK_REQUEST_KEYFRAME); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_STALLED, status.lastFrameID); + } + if (status.state == LG_NET_VIDEO_STATE_STOPPED || + status.state == LG_NET_VIDEO_STATE_ERROR) + { + video->wireConfigured = false; + video->decoderReady = false; + video->subscribePending = false; + video->subscribeRequestID = 0; + video->nextSubscribeNs = nanotime() + + (uint64_t)NETWORK_VIDEO_SUBSCRIBE_RETRY_MS * UINT64_C(1000000); + networkVideoClearAssembly(&video->assembly); + networkVideoFreeOutput(&video->output); + memset(video->schedules, 0, sizeof(video->schedules)); + notify = networkVideoSetStatusNL(video, false, + status.state == LG_NET_VIDEO_STATE_ERROR ? + LG_TRANSPORT_ERROR : LG_TRANSPORT_UNAVAILABLE); + reset = true; + lgSignalEvent(video->frameEvent); + } + LG_UNLOCK(video->stateLock); + if (reset) + networkVideoDecoderReset(video->decoder); + if (notify) + networkVideoNotifyStatus(video); + if (reset) + networkVideoProcess(video); + return true; +} + +static bool networkVideoHandlePacket(NetworkVideo * video, + const LGNetPacketView * packet, bool datagram) +{ + if (!video || !packet || + packet->envelope.service != LG_NET_SERVICE_VIDEO || + !networkVideoVersionSupported(packet->envelope.serviceVersion) || + !networkVideoPacketFlags(packet, datagram)) + return false; + + const bool bulk = !datagram && + !(packet->envelope.flags & LG_NET_ENVELOPE_RELIABLE); + + switch (packet->envelope.messageType) + { + case LG_NET_VIDEO_MESSAGE_STREAM_CONFIG: + return !datagram && !bulk && networkVideoHandleConfig(video, packet); + + case LG_NET_VIDEO_MESSAGE_FRAME: + return !datagram && networkVideoUnsolicited(packet) && + networkVideoHandleFrame(video, packet); + + case LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT: + return networkVideoUnsolicited(packet) && + networkVideoHandleFragment(video, packet); + + case LG_NET_VIDEO_MESSAGE_SCHEDULE: + return !datagram && !bulk && networkVideoUnsolicited(packet) && + networkVideoHandleSchedule(video, packet); + + case LG_NET_VIDEO_MESSAGE_STATUS: + return !datagram && !bulk && networkVideoHandleStatus(video, packet); + + default: + return false; + } +} + +bool networkVideoPacket(NetworkVideo * video, + const LGNetPacketView * packet) +{ + return networkVideoHandlePacket(video, packet, false); +} + +bool networkVideoDatagram(NetworkVideo * video, + const LGNetPacketView * packet) +{ + return networkVideoHandlePacket(video, packet, true); +} + +void networkVideoProcess(NetworkVideo * video) +{ + if (!video) + return; + + const uint64_t now = nanotime(); + LG_LOCK(video->stateLock); + if (video->destroying) + { + LG_UNLOCK(video->stateLock); + return; + } + if (video->assembly.active && now >= video->assembly.deadlineNs) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoClearAssembly(&video->assembly); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_STALLED, video->highestSeenFrameID); + } + networkVideoSendSubscribeNL(video, now); + networkVideoSendKeyframeNL(video, now); + networkVideoSendFeedbackNL(video, now); + LG_UNLOCK(video->stateLock); +} + +static bool networkVideoTimer(void * opaque) +{ + networkVideoProcess(opaque); + return true; +} + +void networkVideoSendReady(NetworkVideo * video, bool datagram) +{ + if (!video) + return; + LG_LOCK(video->stateLock); + if (datagram) + { + video->nextFeedbackNs = 0; + video->nextKeyframeNs = 0; + } + else + { + video->subscribePending = true; + video->nextFeedbackNs = 0; + video->nextKeyframeNs = 0; + } + LG_UNLOCK(video->stateLock); + networkVideoProcess(video); +} + +bool networkVideoCreate(NetworkVideo ** result, + NetworkConnection * connection) +{ + if (!result || !connection) + return false; + + NetworkVideo * video = calloc(1, sizeof(*video)); + if (!video) + return false; + LG_LOCK_INIT(video->stateLock); + LG_LOCK_INIT(video->callbackLock); + video->connection = connection; + video->subscriberID = nanotime() ^ ((uint64_t)(uintptr_t)video << 17); + video->status.epoch = 1; + video->status.reason = LG_TRANSPORT_DISCONNECTED; + if (!video->subscriberID) + video->subscriberID = 1; + + video->frameEvent = lgCreateEvent(true, 0); + if (!video->frameEvent || + !networkVideoDecoderCreate(&video->decoder) || + !lgCreateTimer(NETWORK_VIDEO_TIMER_MS, + networkVideoTimer, video, &video->timer)) + { + networkVideoDecoderDestroy(&video->decoder); + if (video->frameEvent) + lgFreeEvent(video->frameEvent); + LG_LOCK_FREE(video->callbackLock); + LG_LOCK_FREE(video->stateLock); + free(video); + return false; + } + + *result = video; + return true; +} + +void networkVideoConnectionState(NetworkVideo * video, bool connected) +{ + if (!video) + return; + + networkVideoDecoderReset(video->decoder); + + uint16_t serviceVersion = 0; + const bool serviceAvailable = connected && + networkConnectionServiceAvailable(video->connection, + LG_NET_SERVICE_VIDEO, LG_NET_CAPABILITY_RELIABLE) && + networkConnectionServiceVersion(video->connection, + LG_NET_SERVICE_VIDEO, &serviceVersion) && + networkVideoVersionSupported(serviceVersion); + const bool datagramsAvailable = serviceAvailable && + networkConnectionServiceAvailable(video->connection, + LG_NET_SERVICE_VIDEO, LG_NET_CAPABILITY_DATAGRAM); + + LG_LOCK(video->stateLock); + video->connected = connected; + video->serviceAvailable = serviceAvailable; + video->datagramsAvailable = datagramsAvailable; + video->serviceVersion = serviceAvailable ? serviceVersion : 0; + video->wireConfigured = false; + video->decoderReady = false; + video->subscribePending = serviceAvailable; + video->subscribeRequestID = 0; + video->nextSubscribeNs = 0; + video->nextFeedbackNs = 0; + video->nextKeyframeNs = 0; + video->feedbackFlags = 0; + video->feedbackPending = false; + video->keyframePending = false; + video->statusSequence = 0; + video->lostFrames = 0; + video->lostFragments = 0; + video->waitCancelled = true; + memset(&video->config, 0, sizeof(video->config)); + memset(video->schedules, 0, sizeof(video->schedules)); + networkVideoClearAssembly(&video->assembly); + networkVideoFreeOutput(&video->output); + const bool notify = networkVideoUpdateStatusNL(video); + LG_UNLOCK(video->stateLock); + lgSignalEvent(video->frameEvent); + if (notify) + networkVideoNotifyStatus(video); + if (serviceAvailable) + networkVideoProcess(video); +} + +void networkVideoDestroy(NetworkVideo ** result) +{ + if (!result || !*result) + return; + NetworkVideo * video = *result; + networkVideoSetStatusListener(video, NULL, NULL); + LG_LOCK(video->stateLock); + video->destroying = true; + LG_UNLOCK(video->stateLock); + lgTimerDestroy(video->timer); + networkVideoConnectionState(video, false); + networkVideoClearAssembly(&video->assembly); + networkVideoFreeOutput(&video->output); + networkVideoFreeLeasesNL(video); + networkVideoDecoderDestroy(&video->decoder); + lgFreeEvent(video->frameEvent); + LG_LOCK_FREE(video->callbackLock); + LG_LOCK_FREE(video->stateLock); + free(video); + *result = NULL; +} + +void networkVideoSetStatusListener(NetworkVideo * video, + NetworkVideoStatusFn callback, void * opaque) +{ + if (!video) + return; + + LG_VideoComponentStatus status; + LG_LOCK(video->callbackLock); + LG_LOCK(video->stateLock); + video->statusCallback = callback; + video->statusOpaque = opaque; + status = video->status; + LG_UNLOCK(video->stateLock); + if (callback) + callback(opaque, &status); + LG_UNLOCK(video->callbackLock); +} + +LG_VideoComponentStatus networkVideoGetStatus(NetworkVideo * video) +{ + LG_VideoComponentStatus status = + { + .available = false, + .epoch = 1, + .reason = LG_TRANSPORT_DISCONNECTED, + }; + if (!video) + return status; + LG_LOCK(video->stateLock); + status = video->status; + LG_UNLOCK(video->stateLock); + return status; +} + +bool networkVideoAttachRenderer(NetworkVideo * video, + const LG_RendererInterop * interop) +{ + if (!video || !networkVideoDecoderAttach(video->decoder, interop)) + return false; + + LGNetVideoStreamConfig config; + bool configured; + LG_LOCK(video->stateLock); + video->rendererAttached = true; + configured = video->wireConfigured; + config = video->config; + LG_UNLOCK(video->stateLock); + const NetworkVideoDecodeResult result = configured ? + networkVideoDecoderConfigure(video->decoder, &config) : + NETWORK_VIDEO_DECODE_UNSUPPORTED; + + LG_LOCK(video->stateLock); + if (video->rendererAttached && video->wireConfigured && configured && + video->config.configEpoch == config.configEpoch) + video->decoderReady = result == NETWORK_VIDEO_DECODE_OK; + + const bool notify = networkVideoUpdateStatusNL(video); + LG_UNLOCK(video->stateLock); + if (notify) + networkVideoNotifyStatus(video); + return true; +} + +void networkVideoDetachRenderer(NetworkVideo * video) +{ + if (!video) + return; + LG_LOCK(video->stateLock); + video->rendererAttached = false; + video->decoderReady = false; + video->waitCancelled = true; + networkVideoFreeOutput(&video->output); + networkVideoFreeLeasesNL(video); + const bool notify = networkVideoUpdateStatusNL(video); + LG_UNLOCK(video->stateLock); + networkVideoDecoderDetach(video->decoder); + lgSignalEvent(video->frameEvent); + if (notify) + networkVideoNotifyStatus(video); +} + +LG_TransportStatus networkVideoNextFrame(NetworkVideo * video, bool useDMA, + LG_TransportFrame * frame) +{ + (void)useDMA; + if (!video || !frame) + return LG_TRANSPORT_ERROR; + memset(frame, 0, sizeof(*frame)); + + for (;;) + { + LG_LOCK(video->stateLock); + networkVideoDrainLeasesNL(video); + if (video->waitCancelled) + { + video->waitCancelled = false; + LG_UNLOCK(video->stateLock); + return LG_TRANSPORT_UNAVAILABLE; + } + if (!video->connected) + { + LG_UNLOCK(video->stateLock); + return LG_TRANSPORT_DISCONNECTED; + } + if (!video->status.available) + { + const LG_TransportStatus status = video->status.reason; + LG_UNLOCK(video->stateLock); + return status; + } + struct NetworkVideoOutput * output = video->output; + if (output && video->leaseCount < NETWORK_VIDEO_MAX_LEASES) + { + video->output = NULL; + video->leaseSerial = networkVideoNext64(video->leaseSerial); + output->handle = video->leaseSerial; + output->next = video->leases; + video->leases = output; + ++video->leaseCount; + frame->serial = output->frameID; + frame->epoch = output->epoch; + frame->timestamp = output->timestamp; + frame->format = &output->decoded.format; + frame->externalFrame = + lgExternalFrameRetain(output->decoded.externalFrame); + frame->releaseFn = networkVideoReleaseLease; + frame->releaseOpaque = video; + frame->releaseHandle = output->handle; + LG_UNLOCK(video->stateLock); + return LG_TRANSPORT_OK; + } + LG_UNLOCK(video->stateLock); + if (!lgWaitEvent(video->frameEvent, 100)) + networkVideoProcess(video); + } +} + +void networkVideoGetFrameTiming(NetworkVideo * video, + const LG_TransportFrame * frame, LG_TransportFrameTiming * timing) +{ + (void)video; + if (!timing) + return; + memset(timing, 0, sizeof(*timing)); + if (!video || !frame || !frame->releaseHandle) + return; + + LG_LOCK(video->stateLock); + const struct NetworkVideoOutput * output = + networkVideoFindLeaseNL(video, frame->releaseHandle); + if (!output) + { + LG_UNLOCK(video->stateLock); + return; + } + *timing = output->decoded.timing; + timing->providerValid = true; + timing->receiveTime = output->receiveTimeNs; + timing->prepareTime = output->decoded.decodeTimeNs; + LG_UNLOCK(video->stateLock); +} + +void networkVideoReleaseFrame(NetworkVideo * video, + LG_TransportFrame * frame) +{ + if (!video || !frame) + return; + + lgExternalFrameRelease(frame->externalFrame); + frame->externalFrame = NULL; + + LG_LOCK(video->stateLock); + struct NetworkVideoOutput * output = + networkVideoTakeLeaseNL(video, frame->releaseHandle); + LG_UNLOCK(video->stateLock); + networkVideoFreeOutput(&output); + memset(frame, 0, sizeof(*frame)); +} + +void networkVideoCancelFrameWait(NetworkVideo * video) +{ + if (!video) + return; + LG_LOCK(video->stateLock); + video->waitCancelled = true; + LG_UNLOCK(video->stateLock); + lgSignalEvent(video->frameEvent); +} + +void networkVideoStopFrame(NetworkVideo * video) +{ + if (!video) + return; + LG_LOCK(video->stateLock); + networkVideoFreeOutput(&video->output); + networkVideoFreeLeasesNL(video); + LG_UNLOCK(video->stateLock); + lgSignalEvent(video->frameEvent); +} diff --git a/client/transports/Network/video.h b/client/transports/Network/video.h new file mode 100644 index 00000000..d0564629 --- /dev/null +++ b/client/transports/Network/video.h @@ -0,0 +1,65 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#pragma once + +#include "connection.h" + +#include "interface/transport.h" + +#include + +#include +#include + +typedef struct NetworkVideo NetworkVideo; + +typedef void (*NetworkVideoStatusFn)(void * opaque, + const LG_VideoComponentStatus * status); + +bool networkVideoCreate(NetworkVideo ** video, + NetworkConnection * connection); +void networkVideoDestroy(NetworkVideo ** video); + +void networkVideoConnectionState(NetworkVideo * video, bool connected); +void networkVideoProcess(NetworkVideo * video); +void networkVideoSendReady(NetworkVideo * video, bool datagram); + +bool networkVideoPacket(NetworkVideo * video, + const LGNetPacketView * packet); +bool networkVideoDatagram(NetworkVideo * video, + const LGNetPacketView * packet); + +void networkVideoSetStatusListener(NetworkVideo * video, + NetworkVideoStatusFn callback, void * opaque); +LG_VideoComponentStatus networkVideoGetStatus(NetworkVideo * video); + +bool networkVideoAttachRenderer(NetworkVideo * video, + const LG_RendererInterop * interop); +void networkVideoDetachRenderer(NetworkVideo * video); + +LG_TransportStatus networkVideoNextFrame(NetworkVideo * video, bool useDMA, + LG_TransportFrame * frame); +void networkVideoGetFrameTiming(NetworkVideo * video, + const LG_TransportFrame * frame, LG_TransportFrameTiming * timing); +void networkVideoReleaseFrame(NetworkVideo * video, + LG_TransportFrame * frame); +void networkVideoCancelFrameWait(NetworkVideo * video); +void networkVideoStopFrame(NetworkVideo * video); diff --git a/client/transports/Network/video/decoder.h b/client/transports/Network/video/decoder.h new file mode 100644 index 00000000..0b796d0c --- /dev/null +++ b/client/transports/Network/video/decoder.h @@ -0,0 +1,67 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#pragma once + +#include "interface/external_frame.h" +#include "interface/renderer.h" +#include "interface/transport.h" + +#include + +#include +#include +#include + +typedef struct NetworkVideoDecoder NetworkVideoDecoder; + +typedef enum NetworkVideoDecodeResult +{ + NETWORK_VIDEO_DECODE_OK, + NETWORK_VIDEO_DECODE_UNSUPPORTED, + NETWORK_VIDEO_DECODE_CORRUPT, + NETWORK_VIDEO_DECODE_ERROR, +} +NetworkVideoDecodeResult; + +typedef struct NetworkVideoDecodedFrame +{ + LG_ExternalFrame * externalFrame; + LG_TransportFrameFormat format; + LG_TransportFrameTiming timing; + uint64_t decodeTimeNs; +} +NetworkVideoDecodedFrame; + +bool networkVideoDecoderCreate(NetworkVideoDecoder ** decoder); +void networkVideoDecoderDestroy(NetworkVideoDecoder ** decoder); + +bool networkVideoDecoderAttach(NetworkVideoDecoder * decoder, + const LG_RendererInterop * interop); +void networkVideoDecoderDetach(NetworkVideoDecoder * decoder); +void networkVideoDecoderReset(NetworkVideoDecoder * decoder); + +NetworkVideoDecodeResult networkVideoDecoderConfigure( + NetworkVideoDecoder * decoder, const LGNetVideoStreamConfig * config); +NetworkVideoDecodeResult networkVideoDecoderDecode( + NetworkVideoDecoder * decoder, const LGNetVideoStreamConfig * config, + const void * data, size_t size, NetworkVideoDecodedFrame * frame); + +void networkVideoDecoderRelease(NetworkVideoDecodedFrame * frame); diff --git a/client/transports/Network/video/decoder_egl.c b/client/transports/Network/video/decoder_egl.c new file mode 100644 index 00000000..592485dd --- /dev/null +++ b/client/transports/Network/video/decoder_egl.c @@ -0,0 +1,1081 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* + * The PyroWave block layout, inverse CDF 9/7 orchestration, and + * dequantization logic in this file derive from PyroWave at commit + * 89f7e47d4abbf650c91fae766728af866c5e32a0. + * + * Copyright (c) 2025 Hans-Kristian Arntzen + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "decoder.h" + +#include "pyrowave.h" +#include "shaders.h" + +#include "common/debug.h" +#include "common/locking.h" +#include "common/time.h" + +#include +#include + +#include +#include +#include +#include + +#define NETWORK_VIDEO_LEVELS 5U +#define NETWORK_VIDEO_COMPONENTS 3U +#define NETWORK_VIDEO_BANDS 4U +#define NETWORK_VIDEO_ALIGNMENT (1U << NETWORK_VIDEO_LEVELS) +#define NETWORK_VIDEO_MIN_SIZE (4U << NETWORK_VIDEO_LEVELS) + +struct NetworkVideoBlockMeta +{ + uint32_t offset; + uint32_t stride; + uint32_t count; +}; + +struct NetworkVideoCurrent +{ + EGLDisplay display; + EGLSurface draw; + EGLSurface read; + EGLContext context; + EGLenum api; +}; + +struct NetworkVideoDecoder +{ + LG_Lock lock; + + EGLDisplay display; + EGLConfig eglConfig; + EGLContext shareContext; + EGLContext context; + + GLuint dequantProgram; + GLuint idwtProgram; + GLuint yuvProgram; + GLuint coefficientTexture; + GLuint planeTextures[NETWORK_VIDEO_COMPONENTS]; + GLuint offsetBuffer; + GLuint payloadBuffer; + + LGNetVideoStreamConfig config; + struct NetworkVideoBlockMeta + blocks[NETWORK_VIDEO_COMPONENTS][NETWORK_VIDEO_LEVELS] + [NETWORK_VIDEO_BANDS]; + + uint32_t alignedWidth; + uint32_t alignedHeight; + uint32_t blockCapacity; + GLint maxTextureSize; + GLint maxArrayLayers; + GLint maxWorkGroupCount[2]; + GLint64 maxStorageBlockSize; + bool attached; + bool configured; +}; + +struct NetworkVideoExternalFrame +{ + LG_ExternalFrame base; + atomic_uint references; + EGLDisplay display; + EGLConfig eglConfig; + EGLContext shareContext; + GLuint texture; + GLsync readySync; +}; + +static uint32_t networkVideoReadU32(const uint8_t * data) +{ + return (uint32_t)data[0] | + (uint32_t)data[1] << 8 | + (uint32_t)data[2] << 16 | + (uint32_t)data[3] << 24; +} + +static uint32_t networkVideoAlign(uint32_t value, uint32_t alignment) +{ + return (value + alignment - 1U) & ~(alignment - 1U); +} + +static void networkVideoSaveCurrent(struct NetworkVideoCurrent * current) +{ + current->display = eglGetCurrentDisplay(); + current->draw = eglGetCurrentSurface(EGL_DRAW); + current->read = eglGetCurrentSurface(EGL_READ); + current->context = eglGetCurrentContext(); + current->api = eglQueryAPI(); +} + +static bool networkVideoMakeCurrent(NetworkVideoDecoder * decoder, + struct NetworkVideoCurrent * previous) +{ + networkVideoSaveCurrent(previous); + if (eglBindAPI(EGL_OPENGL_ES_API) && + eglMakeCurrent(decoder->display, EGL_NO_SURFACE, EGL_NO_SURFACE, + decoder->context)) + return true; + + eglBindAPI(previous->api); + return false; +} + +static void networkVideoRestoreCurrent(NetworkVideoDecoder * decoder, + const struct NetworkVideoCurrent * previous) +{ + if (previous->context != EGL_NO_CONTEXT && + previous->display != EGL_NO_DISPLAY) + { + eglBindAPI(previous->api); + eglMakeCurrent(previous->display, previous->draw, previous->read, + previous->context); + return; + } + + eglMakeCurrent(decoder->display, EGL_NO_SURFACE, EGL_NO_SURFACE, + EGL_NO_CONTEXT); + eglBindAPI(previous->api); +} + +static GLuint networkVideoCompileShader( + const char * const * sources, GLsizei count) +{ + const GLuint shader = glCreateShader(GL_COMPUTE_SHADER); + if (!shader) + return 0; + + glShaderSource(shader, count, sources, NULL); + glCompileShader(shader); + GLint compiled = GL_FALSE; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); + if (compiled == GL_TRUE) + return shader; + + char log[4096]; + GLint length = 0; + glGetShaderInfoLog(shader, sizeof(log), &length, log); + DEBUG_ERROR("Network video compute shader failed to compile: %.*s", + length, log); + glDeleteShader(shader); + return 0; +} + +static GLuint networkVideoCreateProgram( + const char * const * sources, GLsizei count) +{ + const GLuint shader = networkVideoCompileShader(sources, count); + if (!shader) + return 0; + + const GLuint program = glCreateProgram(); + if (!program) + { + glDeleteShader(shader); + return 0; + } + + glAttachShader(program, shader); + glLinkProgram(program); + glDeleteShader(shader); + GLint linked = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linked); + if (linked == GL_TRUE) + return program; + + char log[4096]; + GLint length = 0; + glGetProgramInfoLog(program, sizeof(log), &length, log); + DEBUG_ERROR("Network video compute program failed to link: %.*s", + length, log); + glDeleteProgram(program); + return 0; +} + +static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder) +{ + if (decoder->coefficientTexture) + glDeleteTextures(1, &decoder->coefficientTexture); + glDeleteTextures(NETWORK_VIDEO_COMPONENTS, decoder->planeTextures); + if (decoder->offsetBuffer) + glDeleteBuffers(1, &decoder->offsetBuffer); + if (decoder->payloadBuffer) + glDeleteBuffers(1, &decoder->payloadBuffer); + + decoder->coefficientTexture = 0; + decoder->offsetBuffer = 0; + decoder->payloadBuffer = 0; + decoder->alignedWidth = 0; + decoder->alignedHeight = 0; + decoder->blockCapacity = 0; + decoder->configured = false; + memset(decoder->planeTextures, 0, sizeof(decoder->planeTextures)); + memset(&decoder->config, 0, sizeof(decoder->config)); + memset(decoder->blocks, 0, sizeof(decoder->blocks)); +} + +static void networkVideoDeleteProgramsNL(NetworkVideoDecoder * decoder) +{ + if (decoder->dequantProgram) + glDeleteProgram(decoder->dequantProgram); + if (decoder->idwtProgram) + glDeleteProgram(decoder->idwtProgram); + if (decoder->yuvProgram) + glDeleteProgram(decoder->yuvProgram); + decoder->dequantProgram = 0; + decoder->idwtProgram = 0; + decoder->yuvProgram = 0; +} + +static bool networkVideoBuildLayoutNL(NetworkVideoDecoder * decoder, + const LGNetVideoStreamConfig * config) +{ + decoder->alignedWidth = networkVideoAlign( + config->width, NETWORK_VIDEO_ALIGNMENT); + decoder->alignedHeight = networkVideoAlign( + config->height, NETWORK_VIDEO_ALIGNMENT); + if (decoder->alignedWidth < NETWORK_VIDEO_MIN_SIZE) + decoder->alignedWidth = NETWORK_VIDEO_MIN_SIZE; + if (decoder->alignedHeight < NETWORK_VIDEO_MIN_SIZE) + decoder->alignedHeight = NETWORK_VIDEO_MIN_SIZE; + + uint64_t blockOffset = 0; + for (int level = NETWORK_VIDEO_LEVELS - 1; level >= 0; --level) + { + const uint32_t width = decoder->alignedWidth >> (level + 1); + const uint32_t height = decoder->alignedHeight >> (level + 1); + const uint32_t blockStride = (width + 31U) / 32U; + const uint32_t blockRows = (height + 31U) / 32U; + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + { + if (!level && component && + config->chromaSubsampling == LG_NET_VIDEO_CHROMA_420) + continue; + + const unsigned firstBand = + level == NETWORK_VIDEO_LEVELS - 1 ? 0U : 1U; + for (unsigned band = firstBand; band < NETWORK_VIDEO_BANDS; ++band) + { + struct NetworkVideoBlockMeta * meta = + &decoder->blocks[component][level][band]; + meta->offset = (uint32_t)blockOffset; + meta->stride = blockStride; + meta->count = blockStride * blockRows; + + blockOffset += meta->count; + } + } + } + + if (!blockOffset || blockOffset > LG_NET_VIDEO_MAX_BLOCKS) + return false; + decoder->blockCapacity = (uint32_t)blockOffset; + return true; +} + +static bool networkVideoResourcesSupportedNL( + const NetworkVideoDecoder * decoder, + const LGNetVideoStreamConfig * config) +{ + const uint64_t offsetBytes = + (uint64_t)decoder->blockCapacity * sizeof(uint32_t); + const uint64_t payloadBytes = (uint64_t)config->maxFrameLength + 8U; + const uint32_t dequantGroupsX = + (decoder->alignedWidth / 2U + 31U) / 32U; + const uint32_t dequantGroupsY = + (decoder->alignedHeight / 2U + 31U) / 32U; + const uint32_t idwtGroupsX = + (decoder->alignedHeight / 2U + 15U) / 16U; + const uint32_t idwtGroupsY = + (decoder->alignedWidth / 2U + 15U) / 16U; + + return decoder->maxTextureSize > 0 && + decoder->maxArrayLayers >= + (GLint)(NETWORK_VIDEO_COMPONENTS * NETWORK_VIDEO_BANDS) && + decoder->alignedWidth <= (uint32_t)decoder->maxTextureSize && + decoder->alignedHeight <= (uint32_t)decoder->maxTextureSize && + decoder->maxStorageBlockSize > 0 && + offsetBytes <= (uint64_t)decoder->maxStorageBlockSize && + payloadBytes <= (uint64_t)decoder->maxStorageBlockSize && + dequantGroupsX <= (uint32_t)decoder->maxWorkGroupCount[0] && + dequantGroupsY <= (uint32_t)decoder->maxWorkGroupCount[1] && + idwtGroupsX <= (uint32_t)decoder->maxWorkGroupCount[0] && + idwtGroupsY <= (uint32_t)decoder->maxWorkGroupCount[1]; +} + +static bool networkVideoCreateTexture2D(GLuint * texture, GLenum format, + uint32_t width, uint32_t height, GLenum filter) +{ + glGenTextures(1, texture); + if (!*texture) + return false; + glBindTexture(GL_TEXTURE_2D, *texture); + glTexStorage2D(GL_TEXTURE_2D, 1, format, width, height); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, filter); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, filter); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + if (glGetError() == GL_NO_ERROR) + return true; + + glDeleteTextures(1, texture); + *texture = 0; + return false; +} + +static bool networkVideoAllocateNL(NetworkVideoDecoder * decoder) +{ + while (glGetError() != GL_NO_ERROR) + { + } + + glGenTextures(1, &decoder->coefficientTexture); + glBindTexture(GL_TEXTURE_2D_ARRAY, decoder->coefficientTexture); + glTexStorage3D(GL_TEXTURE_2D_ARRAY, NETWORK_VIDEO_LEVELS, GL_R32F, + decoder->alignedWidth / 2U, decoder->alignedHeight / 2U, + NETWORK_VIDEO_COMPONENTS * NETWORK_VIDEO_BANDS); + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MIN_FILTER, + GL_NEAREST_MIPMAP_NEAREST); + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_S, + GL_MIRRORED_REPEAT); + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_T, + GL_MIRRORED_REPEAT); + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_BASE_LEVEL, 0); + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAX_LEVEL, + NETWORK_VIDEO_LEVELS - 1U); + + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + { + const bool chroma = component != 0 && + decoder->config.chromaSubsampling == LG_NET_VIDEO_CHROMA_420; + const uint32_t width = chroma ? + decoder->alignedWidth / 2U : decoder->alignedWidth; + const uint32_t height = chroma ? + decoder->alignedHeight / 2U : decoder->alignedHeight; + if (!networkVideoCreateTexture2D(&decoder->planeTextures[component], + GL_R32F, width, height, GL_LINEAR)) + return false; + } + + glGenBuffers(1, &decoder->offsetBuffer); + glBindBuffer(GL_SHADER_STORAGE_BUFFER, decoder->offsetBuffer); + glBufferData(GL_SHADER_STORAGE_BUFFER, + (GLsizeiptr)decoder->blockCapacity * sizeof(uint32_t), + NULL, GL_STREAM_DRAW); + glGenBuffers(1, &decoder->payloadBuffer); + return decoder->coefficientTexture && decoder->offsetBuffer && + decoder->payloadBuffer && glGetError() == GL_NO_ERROR; +} + +static void networkVideoExternalRetain(LG_ExternalFrame * base) +{ + struct NetworkVideoExternalFrame * frame = + (struct NetworkVideoExternalFrame *)base; + atomic_fetch_add_explicit( + &frame->references, 1U, memory_order_relaxed); +} + +static void networkVideoExternalDelete( + struct NetworkVideoExternalFrame * frame) +{ + struct NetworkVideoCurrent previous; + networkVideoSaveCurrent(&previous); + EGLContext temporary = EGL_NO_CONTEXT; + bool changed = false; + bool usable = previous.context != EGL_NO_CONTEXT && + previous.context == frame->shareContext && + previous.display == frame->display; + if (!usable) + { + const EGLint attributes[] = + { + EGL_CONTEXT_CLIENT_VERSION, 3, + EGL_NONE, + }; + if (eglBindAPI(EGL_OPENGL_ES_API)) + temporary = eglCreateContext(frame->display, frame->eglConfig, + frame->shareContext, attributes); + if (temporary != EGL_NO_CONTEXT && + eglMakeCurrent(frame->display, EGL_NO_SURFACE, EGL_NO_SURFACE, + temporary)) + { + changed = true; + usable = true; + } + } + + if (usable) + { + if (frame->readySync) + glDeleteSync(frame->readySync); + if (frame->texture) + glDeleteTextures(1, &frame->texture); + } + + if (temporary != EGL_NO_CONTEXT) + { + if (changed) + eglMakeCurrent(frame->display, EGL_NO_SURFACE, EGL_NO_SURFACE, + EGL_NO_CONTEXT); + eglDestroyContext(frame->display, temporary); + } + if (previous.context != EGL_NO_CONTEXT && + previous.display != EGL_NO_DISPLAY) + { + eglBindAPI(previous.api); + eglMakeCurrent(previous.display, previous.draw, previous.read, + previous.context); + } + else + eglBindAPI(previous.api); +} + +static void networkVideoExternalRelease(LG_ExternalFrame * base) +{ + struct NetworkVideoExternalFrame * frame = + (struct NetworkVideoExternalFrame *)base; + if (atomic_fetch_sub_explicit( + &frame->references, 1U, memory_order_acq_rel) != 1U) + return; + + networkVideoExternalDelete(frame); + free(frame); +} + +static bool networkVideoExternalQuery(const LG_ExternalFrame * base, + void * result, size_t resultSize) +{ + const struct NetworkVideoExternalFrame * frame = + (const struct NetworkVideoExternalFrame *)base; + if (!result || resultSize < sizeof(LG_ExternalFrameEGLTexture)) + return false; + + LG_ExternalFrameEGLTexture descriptor = + { + .version = LG_EXTERNAL_FRAME_EGL_TEXTURE_VERSION, + .size = sizeof(descriptor), + .texture = frame->texture, + .target = GL_TEXTURE_2D, + .readySync = (uintptr_t)frame->readySync, + }; + memcpy(result, &descriptor, sizeof(descriptor)); + return true; +} + +static const LG_ExternalFrameOps NETWORK_VIDEO_EXTERNAL_OPS = +{ + .type = LG_EXTERNAL_FRAME_EGL_TEXTURE, + .retain = networkVideoExternalRetain, + .release = networkVideoExternalRelease, + .query = networkVideoExternalQuery, +}; + +bool networkVideoDecoderCreate(NetworkVideoDecoder ** result) +{ + if (!result) + return false; + + NetworkVideoDecoder * decoder = calloc(1, sizeof(*decoder)); + if (!decoder) + return false; + LG_LOCK_INIT(decoder->lock); + *result = decoder; + return true; +} + +void networkVideoDecoderDetach(NetworkVideoDecoder * decoder) +{ + if (!decoder) + return; + + LG_LOCK(decoder->lock); + if (decoder->context != EGL_NO_CONTEXT) + { + struct NetworkVideoCurrent previous; + if (networkVideoMakeCurrent(decoder, &previous)) + { + networkVideoDeleteConfigurationNL(decoder); + networkVideoDeleteProgramsNL(decoder); + networkVideoRestoreCurrent(decoder, &previous); + } + eglDestroyContext(decoder->display, decoder->context); + } + decoder->display = EGL_NO_DISPLAY; + decoder->eglConfig = NULL; + decoder->shareContext = EGL_NO_CONTEXT; + decoder->context = EGL_NO_CONTEXT; + decoder->maxTextureSize = 0; + decoder->maxArrayLayers = 0; + decoder->maxWorkGroupCount[0] = 0; + decoder->maxWorkGroupCount[1] = 0; + decoder->maxStorageBlockSize = 0; + decoder->attached = false; + LG_UNLOCK(decoder->lock); +} + +void networkVideoDecoderDestroy(NetworkVideoDecoder ** result) +{ + if (!result || !*result) + return; + NetworkVideoDecoder * decoder = *result; + networkVideoDecoderDetach(decoder); + LG_LOCK_FREE(decoder->lock); + free(decoder); + *result = NULL; +} + +void networkVideoDecoderReset(NetworkVideoDecoder * decoder) +{ + if (!decoder) + return; + + LG_LOCK(decoder->lock); + if (decoder->context != EGL_NO_CONTEXT) + { + struct NetworkVideoCurrent previous; + if (networkVideoMakeCurrent(decoder, &previous)) + { + networkVideoDeleteConfigurationNL(decoder); + networkVideoRestoreCurrent(decoder, &previous); + } + } + LG_UNLOCK(decoder->lock); +} + +bool networkVideoDecoderAttach(NetworkVideoDecoder * decoder, + const LG_RendererInterop * interop) +{ + const size_t requiredSize = offsetof(LG_RendererInterop, + egl.sharedTextureImport) + sizeof(interop->egl.sharedTextureImport); + if (!decoder || !interop || + interop->version < LG_RENDERER_INTEROP_EGL_TEXTURE_VERSION || + interop->size < requiredSize || + interop->type != LG_RENDERER_INTEROP_EGL || + !interop->egl.display || !interop->egl.config || + !interop->egl.shareContext || !interop->egl.sharedTextureImport) + return false; + + networkVideoDecoderDetach(decoder); + LG_LOCK(decoder->lock); + decoder->display = interop->egl.display; + decoder->eglConfig = interop->egl.config; + decoder->shareContext = interop->egl.shareContext; + const EGLint attributes[] = + { + EGL_CONTEXT_CLIENT_VERSION, 3, + EGL_NONE, + }; + if (!eglBindAPI(EGL_OPENGL_ES_API)) + goto fail; + decoder->context = eglCreateContext(decoder->display, decoder->eglConfig, + decoder->shareContext, attributes); + if (decoder->context == EGL_NO_CONTEXT) + goto fail; + + struct NetworkVideoCurrent previous; + if (!networkVideoMakeCurrent(decoder, &previous)) + goto fail; + + GLint major = 0; + GLint minor = 0; + glGetIntegerv(GL_MAJOR_VERSION, &major); + glGetIntegerv(GL_MINOR_VERSION, &minor); + if (major < 3 || (major == 3 && minor < 1)) + { + DEBUG_ERROR("Network video requires OpenGL ES 3.1, found %d.%d", + major, minor); + networkVideoRestoreCurrent(decoder, &previous); + goto fail; + } + + glGetIntegerv(GL_MAX_TEXTURE_SIZE, &decoder->maxTextureSize); + glGetIntegerv(GL_MAX_ARRAY_TEXTURE_LAYERS, &decoder->maxArrayLayers); + glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_COUNT, 0, + &decoder->maxWorkGroupCount[0]); + glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_COUNT, 1, + &decoder->maxWorkGroupCount[1]); + glGetInteger64v(GL_MAX_SHADER_STORAGE_BLOCK_SIZE, + &decoder->maxStorageBlockSize); + + decoder->dequantProgram = networkVideoCreateProgram( + NETWORK_VIDEO_DEQUANT_SOURCES, + sizeof(NETWORK_VIDEO_DEQUANT_SOURCES) / + sizeof(NETWORK_VIDEO_DEQUANT_SOURCES[0])); + decoder->idwtProgram = networkVideoCreateProgram( + NETWORK_VIDEO_IDWT_SOURCES, + sizeof(NETWORK_VIDEO_IDWT_SOURCES) / + sizeof(NETWORK_VIDEO_IDWT_SOURCES[0])); + decoder->yuvProgram = networkVideoCreateProgram( + NETWORK_VIDEO_YUV_SOURCES, + sizeof(NETWORK_VIDEO_YUV_SOURCES) / + sizeof(NETWORK_VIDEO_YUV_SOURCES[0])); + networkVideoRestoreCurrent(decoder, &previous); + if (!decoder->dequantProgram || !decoder->idwtProgram || + !decoder->yuvProgram) + goto fail; + + decoder->attached = true; + LG_UNLOCK(decoder->lock); + return true; + +fail: + if (decoder->context != EGL_NO_CONTEXT) + { + struct NetworkVideoCurrent cleanup; + if (networkVideoMakeCurrent(decoder, &cleanup)) + { + networkVideoDeleteProgramsNL(decoder); + networkVideoRestoreCurrent(decoder, &cleanup); + } + eglDestroyContext(decoder->display, decoder->context); + } + decoder->context = EGL_NO_CONTEXT; + decoder->display = EGL_NO_DISPLAY; + decoder->eglConfig = NULL; + decoder->shareContext = EGL_NO_CONTEXT; + decoder->maxTextureSize = 0; + decoder->maxArrayLayers = 0; + decoder->maxWorkGroupCount[0] = 0; + decoder->maxWorkGroupCount[1] = 0; + decoder->maxStorageBlockSize = 0; + LG_UNLOCK(decoder->lock); + return false; +} + +NetworkVideoDecodeResult networkVideoDecoderConfigure( + NetworkVideoDecoder * decoder, const LGNetVideoStreamConfig * config) +{ + if (!decoder || !networkPyroWaveProfileValid(config)) + return NETWORK_VIDEO_DECODE_UNSUPPORTED; + + LG_LOCK(decoder->lock); + if (!decoder->attached) + { + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_UNSUPPORTED; + } + + struct NetworkVideoCurrent previous; + if (!networkVideoMakeCurrent(decoder, &previous)) + { + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_ERROR; + } + + networkVideoDeleteConfigurationNL(decoder); + decoder->config = *config; + + const bool valid = networkVideoBuildLayoutNL(decoder, config) && + networkVideoResourcesSupportedNL(decoder, config) && + networkVideoAllocateNL(decoder); + if (!valid) + networkVideoDeleteConfigurationNL(decoder); + else + decoder->configured = true; + networkVideoRestoreCurrent(decoder, &previous); + LG_UNLOCK(decoder->lock); + return valid ? NETWORK_VIDEO_DECODE_OK : NETWORK_VIDEO_DECODE_ERROR; +} + +static bool networkVideoConfigMatches( + const LGNetVideoStreamConfig * left, + const LGNetVideoStreamConfig * right) +{ + return left->streamID == right->streamID && + left->configEpoch == right->configEpoch && + left->codec == right->codec && + left->codecVersion == right->codecVersion && + left->width == right->width && left->height == right->height && + left->pixelFormat == right->pixelFormat && + left->chromaSubsampling == right->chromaSubsampling && + left->colorPrimaries == right->colorPrimaries && + left->colorTransfer == right->colorTransfer && + left->colorMatrix == right->colorMatrix && + left->colorRange == right->colorRange && + left->bitDepth == right->bitDepth && left->flags == right->flags; +} + +static bool networkVideoPayloadOffsetsNL(NetworkVideoDecoder * decoder, + const uint8_t * data, size_t size, uint32_t * offsets, + bool * chromaLeft) +{ + for (uint32_t index = 0; index < decoder->blockCapacity; ++index) + offsets[index] = UINT32_MAX; + + size_t cursor = 0; + bool header = false; + while (cursor + 8U <= size) + { + const uint32_t first = networkVideoReadU32(data + cursor); + const uint32_t second = networkVideoReadU32(data + cursor + 4U); + if (first >> 31) + { + *chromaLeft = (second >> 31) != 0; + header = true; + cursor += 8U; + continue; + } + + const uint32_t words = (first >> 16) & UINT32_C(0x00000FFF); + const uint32_t block = second >> 8; + const size_t bytes = (size_t)words * sizeof(uint32_t); + if (!words || block >= decoder->blockCapacity || bytes > size - cursor) + return false; + if (offsets[block] == UINT32_MAX) + offsets[block] = (uint32_t)(cursor / sizeof(uint32_t)); + cursor += bytes; + } + return header && cursor == size; +} + +static void networkVideoSetUniform2i( + GLuint program, const char * name, GLint x, GLint y) +{ + glUniform2i(glGetUniformLocation(program, name), x, y); +} + +static void networkVideoDequantNL(NetworkVideoDecoder * decoder) +{ + glUseProgram(decoder->dequantProgram); + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, decoder->offsetBuffer); + glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, decoder->payloadBuffer); + + for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level) + { + const uint32_t width = decoder->alignedWidth >> (level + 1U); + const uint32_t height = decoder->alignedHeight >> (level + 1U); + glBindImageTexture(0, decoder->coefficientTexture, level, + GL_TRUE, 0, GL_WRITE_ONLY, GL_R32F); + networkVideoSetUniform2i(decoder->dequantProgram, + "resolution", width, height); + + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + { + if (!level && component && + decoder->config.chromaSubsampling == LG_NET_VIDEO_CHROMA_420) + continue; + const unsigned firstBand = + level == NETWORK_VIDEO_LEVELS - 1U ? 0U : 1U; + for (unsigned band = firstBand; band < NETWORK_VIDEO_BANDS; ++band) + { + const struct NetworkVideoBlockMeta * meta = + &decoder->blocks[component][level][band]; + glUniform1i(glGetUniformLocation(decoder->dequantProgram, + "outputLayer"), component * NETWORK_VIDEO_BANDS + band); + glUniform1i(glGetUniformLocation(decoder->dequantProgram, + "blockOffset"), meta->offset); + glUniform1i(glGetUniformLocation(decoder->dequantProgram, + "blockStride"), meta->stride); + glDispatchCompute((width + 31U) / 32U, (height + 31U) / 32U, 1); + } + } + } + glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT | + GL_TEXTURE_FETCH_BARRIER_BIT); +} + +static void networkVideoIDWTNL(NetworkVideoDecoder * decoder) +{ + glUseProgram(decoder->idwtProgram); + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D_ARRAY, decoder->coefficientTexture); + glUniform1i(glGetUniformLocation(decoder->idwtProgram, + "coefficients"), 0); + + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + { + const unsigned finalLevel = component && + decoder->config.chromaSubsampling == LG_NET_VIDEO_CHROMA_420 ? 1U : 0U; + for (int level = NETWORK_VIDEO_LEVELS - 1; + level >= (int)finalLevel; --level) + { + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_BASE_LEVEL, level); + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAX_LEVEL, level); + const uint32_t inputWidth = decoder->alignedWidth >> (level + 1); + const uint32_t inputHeight = decoder->alignedHeight >> (level + 1); + networkVideoSetUniform2i(decoder->idwtProgram, + "resolution", inputHeight, inputWidth); + glUniform2f(glGetUniformLocation(decoder->idwtProgram, + "invResolution"), 1.0f / inputHeight, 1.0f / inputWidth); + glUniform1i(glGetUniformLocation(decoder->idwtProgram, + "inputLevel"), level); + glUniform1i(glGetUniformLocation(decoder->idwtProgram, + "baseLayer"), component * NETWORK_VIDEO_BANDS); + const bool final = level == (int)finalLevel; + glUniform1i(glGetUniformLocation(decoder->idwtProgram, + "dcShift"), final ? GL_TRUE : GL_FALSE); + + if (final) + { + glBindImageTexture(1, decoder->planeTextures[component], 0, + GL_FALSE, 0, GL_WRITE_ONLY, GL_R32F); + } + else + { + glBindImageTexture(1, decoder->coefficientTexture, level - 1, + GL_FALSE, component * NETWORK_VIDEO_BANDS, + GL_WRITE_ONLY, GL_R32F); + } + glDispatchCompute( + (inputHeight + 15U) / 16U, (inputWidth + 15U) / 16U, 1); + glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT | + GL_TEXTURE_FETCH_BARRIER_BIT); + } + } + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_BASE_LEVEL, 0); + glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAX_LEVEL, + NETWORK_VIDEO_LEVELS - 1U); +} + +static void networkVideoColorRange( + const LGNetVideoStreamConfig * config, + float * lumaOffset, float * lumaScale, + float * chromaMidpoint, float * chromaScale) +{ + float maximum; + float shift; + if (config->pixelFormat == LG_NET_VIDEO_PIXEL_FORMAT_YUV444P16F) + { + maximum = 255.0f; + shift = 1.0f; + *chromaMidpoint = 0.5f; + } + else + { + maximum = (float)((UINT32_C(1) << config->bitDepth) - 1U); + shift = (float)(UINT32_C(1) << (config->bitDepth - 8U)); + *chromaMidpoint = 128.0f * shift / maximum; + } + + if (config->colorRange == LG_NET_COLOR_RANGE_LIMITED) + { + *lumaOffset = 16.0f * shift / maximum; + *lumaScale = maximum / (219.0f * shift); + *chromaScale = maximum / (224.0f * shift); + } + else + { + *lumaOffset = 0.0f; + *lumaScale = 1.0f; + *chromaScale = 1.0f; + } +} + +static struct NetworkVideoExternalFrame * networkVideoConvertNL( + NetworkVideoDecoder * decoder, bool chromaLeft) +{ + GLuint texture = 0; + if (!networkVideoCreateTexture2D(&texture, GL_RGBA16F, + decoder->config.width, decoder->config.height, GL_LINEAR)) + return NULL; + + glUseProgram(decoder->yuvProgram); + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + { + glActiveTexture(GL_TEXTURE0 + component); + glBindTexture(GL_TEXTURE_2D, decoder->planeTextures[component]); + } + glUniform1i(glGetUniformLocation(decoder->yuvProgram, "lumaPlane"), 0); + glUniform1i(glGetUniformLocation(decoder->yuvProgram, "cbPlane"), 1); + glUniform1i(glGetUniformLocation(decoder->yuvProgram, "crPlane"), 2); + networkVideoSetUniform2i(decoder->yuvProgram, "outputSize", + decoder->config.width, decoder->config.height); + glUniform1i(glGetUniformLocation(decoder->yuvProgram, "bt2020"), + decoder->config.colorMatrix == LG_NET_COLOR_MATRIX_BT2020); + glUniform1i(glGetUniformLocation(decoder->yuvProgram, "chromaLeft"), + chromaLeft); + float lumaOffset; + float lumaScale; + float chromaMidpoint; + float chromaScale; + networkVideoColorRange(&decoder->config, + &lumaOffset, &lumaScale, &chromaMidpoint, &chromaScale); + glUniform1f(glGetUniformLocation(decoder->yuvProgram, "lumaOffset"), + lumaOffset); + glUniform1f(glGetUniformLocation(decoder->yuvProgram, "lumaScale"), + lumaScale); + glUniform1f(glGetUniformLocation(decoder->yuvProgram, "chromaMidpoint"), + chromaMidpoint); + glUniform1f(glGetUniformLocation(decoder->yuvProgram, "chromaScale"), + chromaScale); + glBindImageTexture(0, texture, 0, GL_FALSE, 0, + GL_WRITE_ONLY, GL_RGBA16F); + glDispatchCompute((decoder->config.width + 7U) / 8U, + (decoder->config.height + 7U) / 8U, 1); + glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT | + GL_TEXTURE_FETCH_BARRIER_BIT); + + GLsync readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); + glFlush(); + if (!readySync || glGetError() != GL_NO_ERROR) + { + if (readySync) + glDeleteSync(readySync); + glDeleteTextures(1, &texture); + return NULL; + } + + struct NetworkVideoExternalFrame * frame = calloc(1, sizeof(*frame)); + if (!frame) + { + glDeleteSync(readySync); + glDeleteTextures(1, &texture); + return NULL; + } + frame->base.ops = &NETWORK_VIDEO_EXTERNAL_OPS; + frame->display = decoder->display; + frame->eglConfig = decoder->eglConfig; + frame->shareContext = decoder->shareContext; + frame->texture = texture; + frame->readySync = readySync; + atomic_init(&frame->references, 1U); + return frame; +} + +NetworkVideoDecodeResult networkVideoDecoderDecode( + NetworkVideoDecoder * decoder, const LGNetVideoStreamConfig * config, + const void * data, size_t size, NetworkVideoDecodedFrame * result) +{ + if (!decoder || !config || !data || !size || !result) + return NETWORK_VIDEO_DECODE_CORRUPT; + memset(result, 0, sizeof(*result)); + + LG_LOCK(decoder->lock); + if (!decoder->configured || + !networkVideoConfigMatches(&decoder->config, config)) + { + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_UNSUPPORTED; + } + + uint32_t * offsets = malloc( + (size_t)decoder->blockCapacity * sizeof(*offsets)); + bool chromaLeft = false; + if (!offsets || !networkVideoPayloadOffsetsNL(decoder, + data, size, offsets, &chromaLeft)) + { + free(offsets); + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_CORRUPT; + } + + struct NetworkVideoCurrent previous; + if (!networkVideoMakeCurrent(decoder, &previous)) + { + free(offsets); + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_ERROR; + } + + while (glGetError() != GL_NO_ERROR) + { + } + glBindBuffer(GL_SHADER_STORAGE_BUFFER, decoder->offsetBuffer); + glBufferData(GL_SHADER_STORAGE_BUFFER, + (GLsizeiptr)decoder->blockCapacity * sizeof(*offsets), + offsets, GL_STREAM_DRAW); + glBindBuffer(GL_SHADER_STORAGE_BUFFER, decoder->payloadBuffer); + glBufferData(GL_SHADER_STORAGE_BUFFER, size + 8U, NULL, GL_STREAM_DRAW); + glBufferSubData(GL_SHADER_STORAGE_BUFFER, 0, size, data); + free(offsets); + + const uint64_t decodeStart = nanotime(); + networkVideoDequantNL(decoder); + networkVideoIDWTNL(decoder); + struct NetworkVideoExternalFrame * external = + networkVideoConvertNL(decoder, chromaLeft); + const uint64_t decodeTime = nanotime() - decodeStart; + const bool success = external && glGetError() == GL_NO_ERROR; + networkVideoRestoreCurrent(decoder, &previous); + if (!success) + { + if (external) + networkVideoExternalRelease(&external->base); + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_ERROR; + } + + uint32_t formatVersion = (uint32_t)config->configEpoch; + if (!formatVersion) + formatVersion = 1; + + result->externalFrame = &external->base; + result->decodeTimeNs = decodeTime; + result->format = (LG_TransportFrameFormat) + { + .version = formatVersion, + .type = FRAME_TYPE_RGBA16F, + .screenWidth = config->width, + .screenHeight = config->height, + .dataWidth = config->width, + .dataHeight = config->height, + .frameWidth = config->width, + .frameHeight = config->height, + .rotation = FRAME_ROT_0, + .stride = config->width, + .pitch = config->width * 8U, + .hdr = (config->flags & LG_NET_VIDEO_STREAM_HDR) != 0, + .hdrPQ = config->colorTransfer == LG_NET_COLOR_TRANSFER_PQ, + .hdrMetadata = false, + .sdrWhiteLevel = LG_SDR_WHITE_LEVEL_DEFAULT, + }; + result->timing.providerValid = true; + result->timing.prepareTime = decodeTime; + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_OK; +} + +void networkVideoDecoderRelease(NetworkVideoDecodedFrame * frame) +{ + if (!frame) + return; + lgExternalFrameRelease(frame->externalFrame); + memset(frame, 0, sizeof(*frame)); +} diff --git a/client/transports/Network/video/pyrowave.c b/client/transports/Network/video/pyrowave.c new file mode 100644 index 00000000..c70a09e0 --- /dev/null +++ b/client/transports/Network/video/pyrowave.c @@ -0,0 +1,324 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* + * The packet layout and block ordering validated here are adapted from the + * PyroWave bitstream specification and reference parser at commit + * 89f7e47d4abbf650c91fae766728af866c5e32a0. + * + * Copyright (c) 2025 Hans-Kristian Arntzen + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "pyrowave.h" + +#include +#include +#include + +#define NETWORK_PYROWAVE_HEADER_SIZE 8U +#define NETWORK_PYROWAVE_DECOMPOSITION_LEVELS 5U +#define NETWORK_PYROWAVE_ALIGNMENT \ + (1U << NETWORK_PYROWAVE_DECOMPOSITION_LEVELS) +#define NETWORK_PYROWAVE_MINIMUM_SIZE \ + (4U << NETWORK_PYROWAVE_DECOMPOSITION_LEVELS) +#define NETWORK_PYROWAVE_SEQUENCE_MASK 7U + +static uint32_t networkPyroWaveReadU32(const uint8_t * data) +{ + return (uint32_t)data[0] | + (uint32_t)data[1] << 8 | + (uint32_t)data[2] << 16 | + (uint32_t)data[3] << 24; +} + +static uint16_t networkPyroWaveReadU16(const uint8_t * data) +{ + return (uint16_t)((uint16_t)data[0] | (uint16_t)data[1] << 8); +} + +static unsigned networkPyroWaveBitCount(uint32_t value) +{ + unsigned count = 0; + while (value) + { + count += value & 1U; + value >>= 1; + } + return count; +} + +static bool networkPyroWavePacketValid( + const uint8_t * packet, size_t packetSize, uint16_t ballot) +{ + const unsigned groups = networkPyroWaveBitCount(ballot); + const size_t controlBase = NETWORK_PYROWAVE_HEADER_SIZE; + const size_t scaleBase = controlBase + groups * sizeof(uint16_t); + const size_t planeBase = scaleBase + groups; + if (planeBase > packetSize) + return false; + + size_t planeOffset = planeBase; + unsigned signs = 0; + for (unsigned group = 0; group < 16U; ++group) + { + if (!(ballot & (UINT32_C(1) << group))) + continue; + + const unsigned compact = networkPyroWaveBitCount( + ballot & ((UINT32_C(1) << group) - 1U)); + const uint16_t control = networkPyroWaveReadU16( + packet + controlBase + compact * sizeof(uint16_t)); + const uint8_t qBits = packet[scaleBase + compact] & 15U; + + for (unsigned subblock = 0; subblock < 8U; ++subblock) + { + const unsigned planes = qBits + + ((control >> (2U * subblock)) & 3U); + if (planes > packetSize - planeOffset) + return false; + + uint8_t nonzero = 0; + for (unsigned plane = 0; plane < planes; ++plane) + nonzero |= packet[planeOffset++]; + signs += networkPyroWaveBitCount(nonzero); + } + } + + const size_t signBytes = (signs + 7U) / 8U; + if (signBytes > packetSize - planeOffset) + return false; + + /* payload_words rounds the packet to a u32 boundary. No other trailing + * payload is defined by version 1. */ + return packetSize - planeOffset - signBytes < sizeof(uint32_t); +} + +static uint32_t networkPyroWaveAlign(uint32_t value, uint32_t alignment) +{ + return (value + alignment - 1U) & ~(alignment - 1U); +} + +static bool networkPyroWaveBlockCapacity( + const LGNetVideoStreamConfig * config, uint32_t * capacity) +{ + if (!config || !capacity) + return false; + + uint32_t width = networkPyroWaveAlign( + config->width, NETWORK_PYROWAVE_ALIGNMENT); + uint32_t height = networkPyroWaveAlign( + config->height, NETWORK_PYROWAVE_ALIGNMENT); + if (width < NETWORK_PYROWAVE_MINIMUM_SIZE) + width = NETWORK_PYROWAVE_MINIMUM_SIZE; + if (height < NETWORK_PYROWAVE_MINIMUM_SIZE) + height = NETWORK_PYROWAVE_MINIMUM_SIZE; + + uint64_t blocks = 0; + for (int level = NETWORK_PYROWAVE_DECOMPOSITION_LEVELS - 1; + level >= 0; --level) + { + const uint32_t levelWidth = width >> (level + 1); + const uint32_t levelHeight = height >> (level + 1); + const uint32_t blocksWide = (levelWidth + 31U) / 32U; + const uint32_t blocksHigh = (levelHeight + 31U) / 32U; + const uint32_t bands = + level == NETWORK_PYROWAVE_DECOMPOSITION_LEVELS - 1 ? 4U : 3U; + uint32_t components = 3U; + if (!level && config->chromaSubsampling == LG_NET_VIDEO_CHROMA_420) + components = 1U; + + blocks += (uint64_t)blocksWide * blocksHigh * bands * components; + } + + if (!blocks || blocks > LG_NET_VIDEO_MAX_BLOCKS) + return false; + + *capacity = (uint32_t)blocks; + return true; +} + +bool networkPyroWaveProfileValid(const LGNetVideoStreamConfig * config) +{ + if (!config || !lgNetVideoStreamConfigValid(config) || + config->codec != LG_NET_VIDEO_CODEC_PYROWAVE || + config->codecVersion < LG_NET_VIDEO_PYROWAVE_CODEC_VERSION_MIN || + config->codecVersion > LG_NET_VIDEO_PYROWAVE_CODEC_VERSION_MAX) + return false; + + if (config->flags & LG_NET_VIDEO_STREAM_RESERVED_ALPHA) + return false; + + const bool hdr = + (config->flags & LG_NET_VIDEO_STREAM_HDR) != 0; + if (hdr ? + (config->colorPrimaries != LG_NET_COLOR_PRIMARIES_BT2020 || + config->colorTransfer != LG_NET_COLOR_TRANSFER_PQ || + config->colorMatrix != LG_NET_COLOR_MATRIX_BT2020) : + (config->colorPrimaries != LG_NET_COLOR_PRIMARIES_BT709 || + config->colorTransfer != LG_NET_COLOR_TRANSFER_BT709 || + config->colorMatrix != LG_NET_COLOR_MATRIX_BT709)) + return false; + + uint32_t capacity; + return networkPyroWaveBlockCapacity(config, &capacity); +} + +static bool networkPyroWaveSequenceProfile( + const LGNetVideoStreamConfig * config, uint32_t word) +{ + const uint32_t chroma = (word >> 26) & 1U; + const uint32_t primaries = (word >> 27) & 1U; + const uint32_t transfer = (word >> 28) & 1U; + const uint32_t transform = (word >> 29) & 1U; + const uint32_t range = (word >> 30) & 1U; + const uint32_t wireChroma = + config->chromaSubsampling == LG_NET_VIDEO_CHROMA_444 ? 1U : 0U; + const uint32_t wirePrimaries = + config->colorPrimaries == LG_NET_COLOR_PRIMARIES_BT2020 ? 1U : 0U; + const uint32_t wireTransfer = + config->colorTransfer == LG_NET_COLOR_TRANSFER_PQ ? 1U : 0U; + const uint32_t wireTransform = + config->colorMatrix == LG_NET_COLOR_MATRIX_BT2020 ? 1U : 0U; + const uint32_t wireRange = + config->colorRange == LG_NET_COLOR_RANGE_LIMITED ? 1U : 0U; + + return chroma == wireChroma && + primaries == wirePrimaries && + transfer == wireTransfer && + transform == wireTransform && + range == wireRange; +} + +NetworkPyroWaveResult networkPyroWaveValidate( + const LGNetVideoStreamConfig * config, const void * data, size_t size, + uint32_t expectedBlocks, uint8_t * sequence) +{ + if (!networkPyroWaveProfileValid(config)) + return NETWORK_PYROWAVE_UNSUPPORTED; + if (!data || size < NETWORK_PYROWAVE_HEADER_SIZE || !expectedBlocks) + return NETWORK_PYROWAVE_INVALID; + + uint32_t capacity; + if (!networkPyroWaveBlockCapacity(config, &capacity) || + expectedBlocks > capacity) + return NETWORK_PYROWAVE_INVALID; + + const size_t seenSize = ((size_t)capacity + 7U) / 8U; + uint8_t * seen = calloc(1, seenSize); + if (!seen) + return NETWORK_PYROWAVE_NO_MEMORY; + + const uint8_t * bytes = data; + size_t remaining = size; + uint32_t uniqueBlocks = 0; + uint32_t declaredBlocks = 0; + uint8_t frameSequence = 0; + bool haveSequence = false; + bool haveHeader = false; + bool valid = true; + + while (valid && remaining >= NETWORK_PYROWAVE_HEADER_SIZE) + { + const uint32_t first = networkPyroWaveReadU32(bytes); + const uint32_t second = networkPyroWaveReadU32(bytes + 4U); + const uint8_t packetSequence = + (uint8_t)((first >> 28) & NETWORK_PYROWAVE_SEQUENCE_MASK); + const bool extended = (first >> 31) != 0; + + if (haveSequence && packetSequence != frameSequence) + { + valid = false; + break; + } + haveSequence = true; + frameSequence = packetSequence; + + if (extended) + { + const uint32_t code = (second >> 24) & 3U; + const uint32_t width = (first & UINT32_C(0x00003FFF)) + 1U; + const uint32_t height = ((first >> 14) & UINT32_C(0x00003FFF)) + 1U; + const uint32_t blocks = second & UINT32_C(0x00FFFFFF); + if (code || width != config->width || height != config->height || + !blocks || blocks > capacity || + !networkPyroWaveSequenceProfile(config, second) || + (haveHeader && blocks != declaredBlocks)) + { + valid = false; + break; + } + + declaredBlocks = blocks; + haveHeader = true; + bytes += NETWORK_PYROWAVE_HEADER_SIZE; + remaining -= NETWORK_PYROWAVE_HEADER_SIZE; + continue; + } + + const uint32_t ballot = first & UINT32_C(0x0000FFFF); + const uint32_t payloadWords = (first >> 16) & UINT32_C(0x00000FFF); + const uint32_t blockIndex = second >> 8; + const size_t packetSize = (size_t)payloadWords * sizeof(uint32_t); + if (!ballot || payloadWords < 2U || packetSize > remaining || + blockIndex >= capacity || + !networkPyroWavePacketValid(bytes, packetSize, (uint16_t)ballot)) + { + valid = false; + break; + } + + const uint8_t mask = (uint8_t)(1U << (blockIndex & 7U)); + if (!(seen[blockIndex >> 3] & mask)) + { + seen[blockIndex >> 3] |= mask; + ++uniqueBlocks; + } + bytes += packetSize; + remaining -= packetSize; + } + + valid = valid && !remaining && haveHeader && haveSequence && + declaredBlocks == uniqueBlocks && expectedBlocks == uniqueBlocks; + free(seen); + if (!valid) + return NETWORK_PYROWAVE_INVALID; + + if (sequence) + *sequence = frameSequence; + return NETWORK_PYROWAVE_OK; +} diff --git a/client/transports/Network/video/pyrowave.h b/client/transports/Network/video/pyrowave.h new file mode 100644 index 00000000..cb600987 --- /dev/null +++ b/client/transports/Network/video/pyrowave.h @@ -0,0 +1,42 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#pragma once + +#include + +#include +#include +#include + +typedef enum NetworkPyroWaveResult +{ + NETWORK_PYROWAVE_OK, + NETWORK_PYROWAVE_INVALID, + NETWORK_PYROWAVE_UNSUPPORTED, + NETWORK_PYROWAVE_NO_MEMORY, +} +NetworkPyroWaveResult; + +bool networkPyroWaveProfileValid(const LGNetVideoStreamConfig * config); + +NetworkPyroWaveResult networkPyroWaveValidate( + const LGNetVideoStreamConfig * config, const void * data, size_t size, + uint32_t expectedBlocks, uint8_t * sequence); diff --git a/client/transports/Network/video/shaders.h b/client/transports/Network/video/shaders.h new file mode 100644 index 00000000..c21b5096 --- /dev/null +++ b/client/transports/Network/video/shaders.h @@ -0,0 +1,332 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* + * The coefficient and inverse-wavelet shaders are OpenGL ES adaptations of + * PyroWave's Vulkan shaders at commit + * 89f7e47d4abbf650c91fae766728af866c5e32a0. + * + * Copyright (c) 2025 Hans-Kristian Arntzen + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#pragma once + +static const char NETWORK_VIDEO_DEQUANT_SHADER_0[] = + "#version 310 es\n" + "precision highp float;\n" + "precision highp int;\n" + "layout(local_size_x=128) in;\n" + "layout(r32f,binding=0) writeonly uniform highp image2DArray outputImage;\n" + "layout(std430,binding=0) readonly buffer OffsetBuffer { uint offsets[]; };\n" + "layout(std430,binding=1) readonly buffer PayloadBuffer { uint payload[]; };\n" + "uniform ivec2 resolution;\n" + "uniform int outputLayer;\n" + "uniform int blockOffset;\n" + "uniform int blockStride;\n" + "shared uint signOffset;\n" + "shared uint planeOffset[16];\n" + "shared uint significantPrefix[128];\n" + "uint readU8(uint byteOffset) {\n" + " return (payload[byteOffset >> 2u] >> (8u * (byteOffset & 3u))) & 255u;\n" + "}\n" + "uint readU16(uint byteOffset) {\n" + " return readU8(byteOffset) | (readU8(byteOffset + 1u) << 8u);\n" + "}\n" + "ivec2 unswizzle8x8(uint index) {\n" + " uint y = bitfieldExtract(index, 0, 1);\n" + " uint x = bitfieldExtract(index, 1, 2);\n" + " y |= bitfieldExtract(index, 3, 2) << 1;\n" + " x |= bitfieldExtract(index, 5, 1) << 2;\n" + " return ivec2(x, y);\n" + "}\n" + "float quantScale(uint code) { return float(code) / 8.0 + 0.25; }\n" + "float blockScale(uint code) {\n" + " int exponent = 4 - int(code >> 3u);\n" + " return (float(8u + (code & 7u)) / 8.0) * exp2(float(exponent));\n" + "}\n" + "void storeValues(ivec2 coord, mat2x4 values) {\n" + " for (int y=0; y<2; ++y) for (int x=0; x<4; ++x) {\n" + " ivec2 target = coord + ivec2(x,y);\n" + " if (all(lessThan(target,resolution)))\n" + " imageStore(outputImage,ivec3(target,outputLayer),vec4(values[y][x]));\n" + " }\n" + "}\n" + "void main() {\n" + " uint localIndex = gl_LocalInvocationIndex;\n" + " int blockIndex = blockOffset + int(gl_WorkGroupID.y) * blockStride +\n" + " int(gl_WorkGroupID.x);\n" + " uint sourceWord = offsets[blockIndex];\n" + " uint subblock = bitfieldExtract(localIndex,0,3);\n" + " uint blockX = bitfieldExtract(localIndex,3,2);\n" + " uint blockY = bitfieldExtract(localIndex,5,2);\n" + " uint group = blockY * 4u + blockX;\n" + " ivec2 coord = ivec2(gl_WorkGroupID.xy) * 32 +\n" + " 8 * ivec2(blockX,blockY) + unswizzle8x8(subblock << 3u);\n" + " if (sourceWord == 0xffffffffu) {\n" + " storeValues(coord,mat2x4(vec4(0.0),vec4(0.0)));\n" + " return;\n" + " }\n" + " uint sourceByte = sourceWord * 4u;\n" + " uint ballot = payload[sourceWord] & 65535u;\n" + " if (localIndex == 0u) {\n" + " uint cursor = sourceByte + 8u + 3u * uint(bitCount(ballot));\n" + " for (uint index=0u; index<16u; ++index) {\n" + " planeOffset[index] = cursor;\n" + " if (bitfieldExtract(ballot,int(index),1) != 0u) {\n" + " uint compact = uint(bitCount(bitfieldExtract(ballot,0,int(index))));\n" + " uint control = readU16(sourceByte + 8u + 2u * compact);\n" + " uint qbits = readU8(sourceByte + 8u +\n" + " 2u * uint(bitCount(ballot)) + compact) & 15u;\n" + " uint low = control & 0x5555u;\n" + " uint high = control & 0xaaaau;\n" + " high |= high >> 1u;\n" + " cursor += uint(bitCount(low) + bitCount(high)) + 8u * qbits;\n" + " }\n" + " }\n" + " signOffset = cursor * 8u;\n" + " }\n" + " barrier();\n"; + +static const char NETWORK_VIDEO_DEQUANT_SHADER_1[] = + " mat2x4 values = mat2x4(vec4(0.0),vec4(0.0));\n" + " uint significant = 0u;\n" + " if (bitfieldExtract(ballot,int(group),1) != 0u) {\n" + " uint compact = uint(bitCount(bitfieldExtract(ballot,0,int(group))));\n" + " uint control = readU16(sourceByte + 8u + 2u * compact);\n" + " uint scaleCode = readU8(sourceByte + 8u +\n" + " 2u * uint(bitCount(ballot)) + compact);\n" + " uint planes = (scaleCode & 15u) +\n" + " bitfieldExtract(control,int(2u * subblock),2);\n" + " int magnitude[8];\n" + " for (int i=0; i<8; ++i) magnitude[i]=0;\n" + " uint cursor = planeOffset[group];\n" + " for (int plane=int(planes)-1; plane>=0; --plane) {\n" + " uint bits = readU8(cursor++);\n" + " for (int i=0; i<8; ++i)\n" + " magnitude[i] = (magnitude[i] << 1) | int((bits >> uint(i)) & 1u);\n" + " }\n" + " float scale = blockScale(payload[sourceWord+1u] & 255u) *\n" + " quantScale(scaleCode >> 4u);\n" + " for (int x=0; x<4; ++x) for (int y=0; y<2; ++y) {\n" + " int index = x * 2 + y;\n" + " float value = float(magnitude[index]);\n" + " if (value != 0.0) { value += 0.5; ++significant; }\n" + " values[y][x] = value * scale;\n" + " }\n" + " }\n" + " significantPrefix[localIndex] = significant;\n" + " barrier();\n" + " if (localIndex == 0u) {\n" + " uint prefix = 0u;\n" + " for (uint index=0u; index<128u; ++index) {\n" + " uint count = significantPrefix[index];\n" + " significantPrefix[index] = prefix;\n" + " prefix += count;\n" + " }\n" + " }\n" + " barrier();\n" + " uint bitCursor = signOffset + significantPrefix[localIndex];\n" + " uint word = payload[bitCursor >> 5u];\n" + " uint upper = payload[(bitCursor >> 5u) + 1u];\n" + " uint shift = bitCursor & 31u;\n" + " if (shift != 0u) word = (word >> shift) | (upper << (32u-shift));\n" + " uint signIndex = 0u;\n" + " for (int x=0; x<4; ++x) for (int y=0; y<2; ++y) {\n" + " if (values[y][x] != 0.0) {\n" + " if (((word >> signIndex) & 1u) != 0u) values[y][x] = -values[y][x];\n" + " ++signIndex;\n" + " }\n" + " }\n" + " storeValues(coord,values);\n" + "}\n"; + +static const char NETWORK_VIDEO_IDWT_SHADER_0[] = + "#version 310 es\n" + "precision highp float;\n" + "precision highp int;\n" + "layout(local_size_x=64) in;\n" + "layout(binding=0) uniform highp sampler2DArray coefficients;\n" + "layout(r32f,binding=1) writeonly uniform highp image2D outputImage;\n" + "uniform ivec2 resolution;\n" + "uniform vec2 invResolution;\n" + "uniform int inputLevel;\n" + "uniform int baseLayer;\n" + "uniform bool dcShift;\n" + "const int APRON=4, HALF_APRON=2, BLOCK=32, HALF_BLOCK=16;\n" + "const float ALPHA=-1.586134342059924;\n" + "const float BETA=-0.052980118572961;\n" + "const float GAMMA=0.882911075530934;\n" + "const float DELTA=0.443506852043971;\n" + "const float K=1.230174104914001;\n" + "const float INV_K=1.0/1.230174104914001;\n" + "shared vec2 sharedBlock[20][41];\n" + "ivec2 unswizzle8x8(uint index) {\n" + " uint y=bitfieldExtract(index,0,1), x=bitfieldExtract(index,1,2);\n" + " y|=bitfieldExtract(index,3,2)<<1; x|=bitfieldExtract(index,5,1)<<2;\n" + " return ivec2(x,y);\n" + "}\n" + "bvec2 and2(bvec2 a,bvec2 b) { return bvec2(a.x&&b.x,a.y&&b.y); }\n" + "vec2 mirrorUV(ivec2 coord,bool evenX,bool evenY) {\n" + " coord-=ivec2(and2(bvec2(evenX,evenY),lessThan(coord,ivec2(0))));\n" + " coord+=1;\n" + " coord+=ivec2(and2(bvec2(!evenX,!evenY),greaterThanEqual(coord,resolution)));\n" + " return (vec2(coord)*invResolution).yx;\n" + "}\n" + "vec4 gatherBand(ivec2 coord,bool evenX,bool evenY,int band) {\n" + " return textureGather(coefficients,vec3(mirrorUV(coord,evenX,evenY),\n" + " float(baseLayer+band)),0);\n" + "}\n" + "void store4(ivec2 coord,vec4 a,vec4 b,vec4 c,vec4 d) {\n" + " sharedBlock[coord.y][2*coord.x+0]=vec2(a.x,c.x);\n" + " sharedBlock[coord.y][2*coord.x+1]=vec2(b.x,d.x);\n" + " sharedBlock[coord.y][2*coord.x+2]=vec2(a.y,c.y);\n" + " sharedBlock[coord.y][2*coord.x+3]=vec2(b.y,d.y);\n" + " sharedBlock[coord.y+1][2*coord.x+0]=vec2(a.z,c.z);\n" + " sharedBlock[coord.y+1][2*coord.x+1]=vec2(b.z,d.z);\n" + " sharedBlock[coord.y+1][2*coord.x+2]=vec2(a.w,c.w);\n" + " sharedBlock[coord.y+1][2*coord.x+3]=vec2(b.w,d.w);\n" + "}\n" + "void loadApron(uint index) {\n" + " ivec2 base=ivec2(gl_WorkGroupID.xy)*HALF_BLOCK-HALF_APRON;\n" + " ivec2 local=2*unswizzle8x8(index); ivec2 coord=base+local;\n" + " store4(local,gatherBand(coord,true,true,0).wxzy,\n" + " gatherBand(coord,false,true,2).wxzy,\n" + " gatherBand(coord,true,false,1).wxzy,\n" + " gatherBand(coord,false,false,3).wxzy);\n" + " ivec2 horizontal=ivec2(HALF_BLOCK+2*int(index%2u),2*int(index/2u));\n" + " if (horizontal.y>1)+(i-2); sharedBlock[y][2*local.y]=vec2(a.x,b.x);\n" + " sharedBlock[y][2*local.y+1]=vec2(a.y,b.y);}\n" + "}\n" + "void inverse4x2(uint index,bool activeLane,int yOffset) {\n" + " vec2 values[12]; ivec2 local=ivec2(4*int(index%8u),int(index/8u)+yOffset);\n" + " if(activeLane){for(int i=0;i<12;i+=2){values[i]=sharedBlock[local.y][local.x+i]*K;\n" + " values[i+1]=sharedBlock[local.y][local.x+i+1]*INV_K;}\n" + " for(int i=2;i<11;i+=2)values[i]-=DELTA*(values[i-1]+values[i+1]);\n" + " for(int i=3;i<10;i+=2)values[i]-=GAMMA*(values[i-1]+values[i+1]);\n" + " for(int i=4;i<9;i+=2)values[i]-=BETA*(values[i-1]+values[i+1]);\n" + " for(int i=5;i<8;i+=2)values[i]-=ALPHA*(values[i-1]+values[i+1]);}\n" + " barrier();\n" + " if(activeLane)for(int i=2;i<4;++i){vec2 a=values[2*i],b=values[2*i+1];\n" + " int y=(local.x>>1)+(i-2); sharedBlock[y][2*local.y]=vec2(a.x,b.x);\n" + " sharedBlock[y][2*local.y+1]=vec2(a.y,b.y);}\n" + "}\n" + "void main(){uint index=gl_LocalInvocationIndex;loadApron(index);\n" + " inverse8x2(index);inverse4x2(index,index<32u,HALF_BLOCK);barrier();\n" + " inverse8x2(index);barrier();ivec2 local=unswizzle8x8(index);\n" + " ivec2 outputSize=imageSize(outputImage);\n" + " for(int y=local.y;y + @@ -180,6 +181,7 @@ + diff --git a/idd/LGIdd/LGIdd.vcxproj.filters b/idd/LGIdd/LGIdd.vcxproj.filters index b471c256..94000a9c 100644 --- a/idd/LGIdd/LGIdd.vcxproj.filters +++ b/idd/LGIdd/LGIdd.vcxproj.filters @@ -301,6 +301,9 @@ Transport\Network + + Transport\Network + Transport\Network @@ -498,6 +501,9 @@ Transport\Network + + Transport\Network + Transport\Network diff --git a/idd/LGIdd/transport/network/CNetworkServer.cpp b/idd/LGIdd/transport/network/CNetworkServer.cpp index d746af4e..89aba78c 100644 --- a/idd/LGIdd/transport/network/CNetworkServer.cpp +++ b/idd/LGIdd/transport/network/CNetworkServer.cpp @@ -439,10 +439,12 @@ struct CNetworkServer::Impl }; Impl(CNetworkTransport& transport, CNetworkAudio& audio, - CNetworkClipboard& clipboard, const NetworkSettings& settings) : + CNetworkClipboard& clipboard, CNetworkVideo& video, + const NetworkSettings& settings) : transport(transport), audio(audio), clipboard(clipboard), + video(video), settings(settings) { } @@ -463,6 +465,7 @@ struct CNetworkServer::Impl CNetworkTransport& transport; CNetworkAudio& audio; CNetworkClipboard& clipboard; + CNetworkVideo& video; NetworkSettings settings; const QUIC_API_TABLE * api = nullptr; HQUIC registration = nullptr; @@ -536,6 +539,15 @@ struct CNetworkServer::Impl }; } + static NetworkVideoPeer VideoPeer(const Connection& connection) + { + return + { + connection.client, + connection.generation, + }; + } + bool AudioPeerInfo(const NetworkAudioPeer& peer, NetworkAudioPeerInfo& info) { @@ -679,6 +691,87 @@ struct CNetworkServer::Impl return NetworkClipboardSendResult::FAILED; } + bool VideoPeerInfo(const NetworkVideoPeer& peer, + NetworkVideoPeerInfo& info) + { + CSRWSharedLock guard(lock); + for (Connection * connection : connections) + { + if (connection->client != peer.client || + connection->generation != peer.generation) + continue; + + CSRWSharedLock connectionGuard(connection->lock); + const uint16_t serviceMask = UINT16_C(1) << LG_NET_SERVICE_VIDEO; + if (connection->closing || + connection->handshake != HandshakeState::READY || + !(connection->selectedServices & serviceMask)) + return false; + + const LGNetCapability& capability = + connection->selectedCapabilities[LG_NET_SERVICE_VIDEO]; + info.serviceVersion = capability.versionMin; + info.capabilityFlags = capability.flags; + info.maxReliablePayload = capability.maxPayloadLength; + info.maxDatagramPayload = capability.maxDatagramPayload; + info.maxBulkPayload = + capability.flags & LG_NET_CAPABILITY_BULK ? + capability.maxPayloadLength : 0; + return true; + } + return false; + } + + NetworkVideoSendResult SendVideo( + const NetworkVideoPeer& peer, NetworkVideoDelivery delivery, + uint16_t messageType, LGNetEnvelopeFlags flags, + uint64_t componentEpoch, uint64_t requestID, + const void * payload, uint32_t payloadLength) + { + CSRWSharedLock guard(lock); + for (Connection * connection : connections) + { + if (connection->client != peer.client || + connection->generation != peer.generation) + continue; + + SendResult result = SendResult::FAILED; + switch (delivery) + { + case NetworkVideoDelivery::RELIABLE: + result = TrySendPacket(*connection, LG_NET_SERVICE_VIDEO, + messageType, flags, componentEpoch, requestID, + payload, payloadLength); + break; + + case NetworkVideoDelivery::DATAGRAM: + result = TrySendDatagram(*connection, LG_NET_SERVICE_VIDEO, + messageType, flags, componentEpoch, requestID, + payload, payloadLength, true); + break; + + case NetworkVideoDelivery::BULK: + result = TrySendBulk(*connection, LG_NET_SERVICE_VIDEO, + messageType, flags, componentEpoch, requestID, + payload, payloadLength); + break; + } + + switch (result) + { + case SendResult::SENT: + return NetworkVideoSendResult::ACCEPTED; + + case SendResult::BLOCKED: + return NetworkVideoSendResult::BLOCKED; + + case SendResult::FAILED: + return NetworkVideoSendResult::FAILED; + } + } + return NetworkVideoSendResult::FAILED; + } + void Shutdown(Connection& connection, QUIC_UINT62 error) { HQUIC handle = nullptr; @@ -1048,6 +1141,31 @@ struct CNetworkServer::Impl ClipboardPeer(connection), LG_NET_SERVICE_FILE, bulk); } + void NotifyVideoSendReady(Connection& connection, Delivery delivery) + { + NetworkVideoDelivery videoDelivery = NetworkVideoDelivery::RELIABLE; + switch (delivery) + { + case Delivery::RELIABLE: + videoDelivery = NetworkVideoDelivery::RELIABLE; + break; + + case Delivery::DATAGRAM: + videoDelivery = NetworkVideoDelivery::DATAGRAM; + break; + + case Delivery::BULK: + videoDelivery = NetworkVideoDelivery::BULK; + break; + + default: + return; + } + if (!ConsumeSendReady(connection, LG_NET_SERVICE_VIDEO, delivery)) + return; + video.SendReady(VideoPeer(connection), videoDelivery); + } + SendResult TrySendPacket(Connection& connection, LGNetService service, uint16_t messageType, LGNetEnvelopeFlags flags, uint64_t componentEpoch, uint64_t requestID, const void * payload, @@ -1894,6 +2012,7 @@ retry: bool activated = false; bool audioSelected = false; bool clipboardSelected = false; + bool videoSelected = false; { CSRWExclusiveLock guard(connection.lock); activated = !connection.closing && @@ -1909,15 +2028,17 @@ retry: connection.clientNonce, sizeof(connection.clientNonce)); NetworkAuthCrypto::SecureZero( connection.serverNonce, sizeof(connection.serverNonce)); - connection.handshake = HandshakeState::READY; - audioSelected = (connection.selectedServices & + connection.handshake = HandshakeState::READY; + audioSelected = (connection.selectedServices & (UINT16_C(1) << LG_NET_SERVICE_AUDIO)) != 0; const uint16_t clipboardServices = (UINT16_C(1) << LG_NET_SERVICE_CLIPBOARD) | (UINT16_C(1) << LG_NET_SERVICE_FILE); - clipboardSelected = + clipboardSelected = (connection.selectedServices & clipboardServices) == clipboardServices; + videoSelected = (connection.selectedServices & + (UINT16_C(1) << LG_NET_SERVICE_VIDEO)) != 0; } } if (!activated) @@ -1930,6 +2051,8 @@ retry: if (clipboardSelected && !clipboard.PeerConnected(ClipboardPeer(connection))) return false; + if (videoSelected && !video.PeerConnected(VideoPeer(connection))) + return false; DEBUG_INFO("Network client %u completed the control handshake", connection.client); return true; @@ -3421,6 +3544,9 @@ retry: const bool audioSubscribe = packet.envelope.service == LG_NET_SERVICE_AUDIO && packet.envelope.messageType == LG_NET_AUDIO_MESSAGE_SUBSCRIBE; + const bool videoSubscribe = + packet.envelope.service == LG_NET_SERVICE_VIDEO && + packet.envelope.messageType == LG_NET_VIDEO_MESSAGE_SUBSCRIBE; const bool audioResponse = packet.envelope.service == LG_NET_SERVICE_AUDIO && packet.envelope.messageType == LG_NET_AUDIO_MESSAGE_STATE_ACK; @@ -3429,7 +3555,7 @@ retry: packet.envelope.service == LG_NET_SERVICE_FILE; LGNetEnvelopeFlags expectedFlags = LG_NET_ENVELOPE_RELIABLE | LG_NET_ENVELOPE_FINAL; - if (audioSubscribe) + if (audioSubscribe || videoSubscribe) expectedFlags |= LG_NET_ENVELOPE_ACK_REQUIRED; if (cursorStatus || audioResponse) expectedFlags |= LG_NET_ENVELOPE_RESPONSE; @@ -3544,6 +3670,11 @@ retry: return audio.HandlePacket(AudioPeer(connection), packet, false) != NetworkAudioPacketResult::INVALID; + case LG_NET_SERVICE_VIDEO: + return video.HandlePacket(VideoPeer(connection), packet, + NetworkVideoDelivery::RELIABLE) != + NetworkVideoPacketResult::INVALID; + case LG_NET_SERVICE_CLIPBOARD: case LG_NET_SERVICE_FILE: return clipboard.HandlePacket( @@ -3621,6 +3752,10 @@ retry: if (packet.envelope.service == LG_NET_SERVICE_AUDIO) return audio.HandlePacket(AudioPeer(connection), packet, true) != NetworkAudioPacketResult::INVALID; + if (packet.envelope.service == LG_NET_SERVICE_VIDEO) + return video.HandlePacket(VideoPeer(connection), packet, + NetworkVideoDelivery::DATAGRAM) != + NetworkVideoPacketResult::INVALID; return false; } @@ -3994,6 +4129,7 @@ retry: static_cast(service)); session = std::move(connection.session); } + video.PeerDisconnected(VideoPeer(connection)); clipboard.PeerDisconnected(ClipboardPeer(connection)); audio.PeerDisconnected(AudioPeer(connection)); if (session) @@ -4156,6 +4292,7 @@ retry: ReadyServicesLocked(connection, Delivery::DATAGRAM); } owner.NotifyAudioSendReady(connection, true); + owner.NotifyVideoSendReady(connection, Delivery::DATAGRAM); return QUIC_STATUS_SUCCESS; } @@ -4170,6 +4307,7 @@ retry: ReleaseSendLocked(connection, *send); } owner.NotifyAudioSendReady(connection, true); + owner.NotifyVideoSendReady(connection, Delivery::DATAGRAM); } return QUIC_STATUS_SUCCESS; @@ -4215,6 +4353,7 @@ retry: ReleaseSendLocked(connection, *send); } owner.NotifyAudioSendReady(connection, false); + owner.NotifyVideoSendReady(connection, Delivery::RELIABLE); owner.NotifyClipboardSendReady(connection, Delivery::RELIABLE); return QUIC_STATUS_SUCCESS; } @@ -4352,7 +4491,10 @@ retry: stream.streamID = event->START_COMPLETE.ID; stream.bound = true; if (stream.openingReturned) + { ready = ReadyServicesLocked(connection, Delivery::BULK); + connection.bulkSend.writableServices |= ready; + } } } if (QUIC_FAILED(status)) @@ -4367,7 +4509,10 @@ retry: send.release(); if (ready) + { + owner.NotifyVideoSendReady(connection, Delivery::BULK); owner.NotifyClipboardSendReady(connection, Delivery::BULK); + } return QUIC_STATUS_SUCCESS; } @@ -4385,6 +4530,7 @@ retry: CSRWExclusiveLock guard(connection.lock); ReleaseSendLocked(connection, *send); } + owner.NotifyVideoSendReady(connection, Delivery::BULK); owner.NotifyClipboardSendReady(connection, Delivery::BULK); return QUIC_STATUS_SUCCESS; } @@ -4438,6 +4584,7 @@ retry: { const uint64_t now = GetTickCount64(); audio.Process(now); + video.Process(now); clipboard.Process(now); CSRWSharedLock guard(lock); if (stopping || !api) @@ -4666,8 +4813,8 @@ retry: CNetworkServer::CNetworkServer(CNetworkTransport& transport, CNetworkAudio& audio, CNetworkClipboard& clipboard, - const NetworkSettings& settings) : - m_impl(new Impl(transport, audio, clipboard, settings)) + CNetworkVideo& video, const NetworkSettings& settings) : + m_impl(new Impl(transport, audio, clipboard, video, settings)) { } @@ -4725,3 +4872,21 @@ NetworkClipboardSendResult CNetworkServer::SendClipboard( componentEpoch, requestID, payload, payloadLength) : NetworkClipboardSendResult::FAILED; } + +bool CNetworkServer::VideoPeerInfo( + const NetworkVideoPeer& peer, NetworkVideoPeerInfo& info) +{ + return m_impl && m_impl->VideoPeerInfo(peer, info); +} + +NetworkVideoSendResult CNetworkServer::SendVideo( + const NetworkVideoPeer& peer, NetworkVideoDelivery delivery, + uint16_t messageType, LGNetEnvelopeFlags flags, + uint64_t componentEpoch, uint64_t requestID, + const void * payload, uint32_t payloadLength) +{ + return m_impl ? + m_impl->SendVideo(peer, delivery, messageType, flags, + componentEpoch, requestID, payload, payloadLength) : + NetworkVideoSendResult::FAILED; +} diff --git a/idd/LGIdd/transport/network/CNetworkServer.h b/idd/LGIdd/transport/network/CNetworkServer.h index 4aa4153b..f607cda2 100644 --- a/idd/LGIdd/transport/network/CNetworkServer.h +++ b/idd/LGIdd/transport/network/CNetworkServer.h @@ -22,6 +22,7 @@ #include "transport/network/CNetworkAudio.h" #include "transport/network/CNetworkClipboard.h" +#include "transport/network/CNetworkVideo.h" #include @@ -30,7 +31,8 @@ struct NetworkSettings; class CNetworkServer final : public INetworkAudioSender, - public INetworkClipboardSender + public INetworkClipboardSender, + public INetworkVideoSender { private: struct Impl; @@ -38,7 +40,8 @@ private: public: CNetworkServer(CNetworkTransport& transport, CNetworkAudio& audio, - CNetworkClipboard& clipboard, const NetworkSettings& settings); + CNetworkClipboard& clipboard, CNetworkVideo& video, + const NetworkSettings& settings); ~CNetworkServer(); CNetworkServer(const CNetworkServer&) = delete; @@ -63,4 +66,12 @@ public: uint16_t messageType, LGNetEnvelopeFlags flags, uint64_t componentEpoch, uint64_t requestID, const void * payload, uint32_t payloadLength) override; + + bool VideoPeerInfo(const NetworkVideoPeer& peer, + NetworkVideoPeerInfo& info) override; + NetworkVideoSendResult SendVideo( + const NetworkVideoPeer& peer, NetworkVideoDelivery delivery, + uint16_t messageType, LGNetEnvelopeFlags flags, + uint64_t componentEpoch, uint64_t requestID, + const void * payload, uint32_t payloadLength) override; }; diff --git a/idd/LGIdd/transport/network/CNetworkTransport.cpp b/idd/LGIdd/transport/network/CNetworkTransport.cpp index 3a9159ae..8612f4d7 100644 --- a/idd/LGIdd/transport/network/CNetworkTransport.cpp +++ b/idd/LGIdd/transport/network/CNetworkTransport.cpp @@ -77,7 +77,8 @@ bool CNetworkTransport::Initialize() try { m_server.reset( - new CNetworkServer(*this, m_audio, m_clipboard, m_settings)); + new CNetworkServer( + *this, m_audio, m_clipboard, m_video, m_settings)); } catch (...) { @@ -97,11 +98,25 @@ bool CNetworkTransport::Initialize() m_server.reset(); return false; } + if (!m_video.AttachSender(*m_server)) + { + DEBUG_ERROR("Failed to attach the network video sender"); + m_clipboard.DetachSender(); + m_audio.DetachSender(); + m_server.reset(); + return false; + } + if ((m_config.services & TRANSPORT_SERVICE_FRAME) && !m_video.Start()) + { + DEBUG_ERROR("Failed to start the network video endpoint"); + m_video.DetachSender(); + m_clipboard.DetachSender(); + m_audio.DetachSender(); + m_server.reset(); + return false; + } m_state = State::INITIALIZED; - if (m_config.services & TRANSPORT_SERVICE_FRAME) - DEBUG_WARN("Network video is unavailable until the GPU codec and " - "texture sink are attached"); return true; } @@ -171,6 +186,7 @@ void CNetworkTransport::Stop() if (server) server->Stop(); + m_video.DetachSender(); m_clipboard.DetachSender(); m_audio.DetachSender(); @@ -183,6 +199,7 @@ void CNetworkTransport::Stop() session->Close(); m_input.Stop(); + m_video.Stop(); m_audio.Stop(); m_clipboard.Stop(); m_control.SetControlEvents(nullptr, {}); @@ -208,33 +225,42 @@ void CNetworkTransport::RecoveryStatus(const SourceKey& source, const FrameProfile * CNetworkTransport::Profiles(unsigned& count) const { - count = 0; - return nullptr; + if (!(m_config.services & TRANSPORT_SERVICE_FRAME)) + { + count = 0; + return nullptr; + } + return m_video.Profiles(count); } CfgResult CNetworkTransport::Probe(const FrameCfg& cfg) const { - (void)cfg; - return CfgResult::REJECTED; + return (m_config.services & TRANSPORT_SERVICE_FRAME) ? + m_video.Probe(cfg) : CfgResult::REJECTED; } CfgResult CNetworkTransport::Prepare(const FrameCfg& cfg) { - (void)cfg; - return CfgResult::REJECTED; + return (m_config.services & TRANSPORT_SERVICE_FRAME) ? + m_video.Prepare(cfg) : CfgResult::REJECTED; } void CNetworkTransport::Commit() { + if (m_config.services & TRANSPORT_SERVICE_FRAME) + m_video.Commit(); } void CNetworkTransport::Abort() { + if (m_config.services & TRANSPORT_SERVICE_FRAME) + m_video.Abort(); } std::shared_ptr CNetworkTransport::GetFrameCaps() const { - return std::shared_ptr(); + return (m_config.services & TRANSPORT_SERVICE_FRAME) ? + m_video.GetFrameCaps() : std::shared_ptr(); } DirectFrameBufferMemory CNetworkTransport::GetDirectMemory() const @@ -264,6 +290,11 @@ IInputSource * CNetworkTransport::Input() return (m_config.services & TRANSPORT_SERVICE_INPUT) ? &m_input : nullptr; } +ITexSink * CNetworkTransport::TexSink() +{ + return (m_config.services & TRANSPORT_SERVICE_FRAME) ? &m_video : nullptr; +} + bool CNetworkTransport::ServiceEnabled(LGNetService service) const { switch (service) diff --git a/idd/LGIdd/transport/network/CNetworkTransport.h b/idd/LGIdd/transport/network/CNetworkTransport.h index 0fc8b087..11882a16 100644 --- a/idd/LGIdd/transport/network/CNetworkTransport.h +++ b/idd/LGIdd/transport/network/CNetworkTransport.h @@ -25,6 +25,7 @@ #include "transport/network/CNetworkAudio.h" #include "transport/network/CNetworkClipboard.h" #include "transport/network/CNetworkSession.h" +#include "transport/network/CNetworkVideo.h" #include "transport/network/NetworkSettings.h" #include @@ -32,9 +33,8 @@ class CNetworkServer; -// This provider owns transport-independent lifecycle and session state. It -// deliberately advertises no frame capabilities and no texture sink until a -// GPU codec backend is attached; it must not be mistaken for working media. +// This provider owns the transport-independent lifecycle, media endpoints, +// and authenticated session state used by the QUIC server. class CNetworkTransport final : public ITransport { private: @@ -53,6 +53,7 @@ private: NetworkSettings m_settings; CNetworkAudio m_audio; CNetworkClipboard m_clipboard; + CNetworkVideo m_video; std::unique_ptr m_server; mutable CSRWLock m_lifecycleLock; @@ -97,6 +98,7 @@ public: IClipboardSource * Clipboard() override; IControlSink * Control() override; IInputSource * Input() override; + ITexSink * TexSink() override; const NetworkSettings& Settings() const { return m_settings; } bool ServiceEnabled(LGNetService service) const; diff --git a/idd/LGIdd/transport/network/CNetworkVideo.cpp b/idd/LGIdd/transport/network/CNetworkVideo.cpp new file mode 100644 index 00000000..206b5456 --- /dev/null +++ b/idd/LGIdd/transport/network/CNetworkVideo.cpp @@ -0,0 +1,2531 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +/* + * The PyroWave packet encoder, CDF 9/7 transform, and quantization logic in + * this file derive from PyroWave at commit + * 89f7e47d4abbf650c91fae766728af866c5e32a0. + * + * Copyright (c) 2025 Hans-Kristian Arntzen + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "transport/network/CNetworkVideo.h" +#include "transport/network/NetworkSettings.h" + +#include "CDebug.h" +#include "CSRWLock.h" +#include "d3d/CD3D12CommandQueue.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Microsoft::WRL; + +namespace +{ + constexpr uint32_t STREAM_ID = 1; + constexpr uint32_t MAX_WIDTH = 8192; + constexpr uint32_t MAX_HEIGHT = 8192; + constexpr uint32_t MAX_FRAME_LENGTH = LG_NET_VIDEO_MAX_FRAME_LENGTH; + constexpr uint32_t MAX_BLOCKS = 262144; + constexpr uint32_t MAX_PENDING_FRAMES = 2; + constexpr uint32_t MAX_SEND_ITEMS = 8192; + constexpr size_t MAX_SEND_BYTES = 96U * 1024U * 1024U; + constexpr uint32_t MAX_FRAME_LATENCY_MS = 50; + constexpr uint32_t PYROWAVE_LEVELS = 5; + constexpr uint32_t PYROWAVE_ALIGNMENT = 1U << PYROWAVE_LEVELS; + constexpr uint32_t PYROWAVE_MIN_DIMENSION = 4U << PYROWAVE_LEVELS; + constexpr uint8_t PYROWAVE_QUANT_CODE = 104; + constexpr uint8_t PYROWAVE_QSCALE_CODE = 6; + constexpr float PYROWAVE_QUANT_SCALE = 512.0f; + + constexpr LGNetEnvelopeFlags DELIVERY_MASK = + LG_NET_ENVELOPE_RESPONSE | LG_NET_ENVELOPE_FINAL | + LG_NET_ENVELOPE_RELIABLE | LG_NET_ENVELOPE_DATAGRAM | + LG_NET_ENVELOPE_ACK_REQUIRED | LG_NET_ENVELOPE_ERROR | + LG_NET_ENVELOPE_EARLY_DATA; + + const FrameProfile VIDEO_PROFILES[] = + { + { FrameStorage::D3D12_TEXTURE, FramePixel::BGRA8, FrameSignal::SRGB }, + { FrameStorage::D3D12_TEXTURE, FramePixel::RGBA8, FrameSignal::SRGB }, + { FrameStorage::D3D12_TEXTURE, FramePixel::RGB10A2, FrameSignal::PQ_BT2020 }, + }; + + class NetworkVideoFrameCaps final : public FrameCaps + { + public: + bool CanUseMode(const FrameMode& mode, + uint32_t * requiredSizeMiB = nullptr) const override + { + if (requiredSizeMiB) + *requiredSizeMiB = 0; + return mode.width && mode.height && mode.refresh100uHz && + mode.width <= MAX_WIDTH && mode.height <= MAX_HEIGHT; + } + }; + + bool SamePeer(const NetworkVideoPeer& left, const NetworkVideoPeer& right) + { + return left.client == right.client && + left.generation == right.generation; + } + + bool SameCfg(const FrameCfg& left, const FrameCfg& right) + { + return Frame::Same(left, right); + } + + bool SupportedProfile(const FrameProfile& profile) + { + for (const FrameProfile& supported : VIDEO_PROFILES) + if (Frame::Same(profile, supported)) + return true; + return false; + } + + uint32_t Align(uint32_t value, uint32_t alignment) + { + return (value + alignment - 1U) & ~(alignment - 1U); + } + + uint64_t Next64(uint64_t value) + { + ++value; + return value ? value : 1; + } + + uint64_t MonotonicNs() + { + const uint64_t now = GetTickCount64() * UINT64_C(1000000); + return now ? now : 1; + } + + LGNetEnvelopeFlags DeliveryFlag(NetworkVideoDelivery delivery) + { + switch (delivery) + { + case NetworkVideoDelivery::RELIABLE: + return LG_NET_ENVELOPE_RELIABLE; + + case NetworkVideoDelivery::DATAGRAM: + return LG_NET_ENVELOPE_DATAGRAM; + + case NetworkVideoDelivery::BULK: + return 0; + } + return 0; + } + + unsigned DeliveryIndex(NetworkVideoDelivery delivery) + { + switch (delivery) + { + case NetworkVideoDelivery::RELIABLE: + return 0; + + case NetworkVideoDelivery::DATAGRAM: + return 1; + + case NetworkVideoDelivery::BULK: + return 2; + } + return 0; + } + + uint8_t DeliveryMask(NetworkVideoDelivery delivery) + { + return static_cast(1U << DeliveryIndex(delivery)); + } + + bool DeliveryFlags(const LGNetEnvelope& envelope, + NetworkVideoDelivery delivery, LGNetEnvelopeFlags expected) + { + return (envelope.flags & DELIVERY_MASK) == + (expected | DeliveryFlag(delivery)); + } + + bool DeliverySupported(const NetworkVideoPeerInfo& info, + NetworkVideoDelivery delivery) + { + switch (delivery) + { + case NetworkVideoDelivery::RELIABLE: + return info.capabilityFlags & LG_NET_CAPABILITY_RELIABLE; + + case NetworkVideoDelivery::DATAGRAM: + return info.capabilityFlags & LG_NET_CAPABILITY_DATAGRAM; + + case NetworkVideoDelivery::BULK: + return info.capabilityFlags & LG_NET_CAPABILITY_BULK; + } + return false; + } + + uint32_t DeliveryPayload(const NetworkVideoPeerInfo& info, + NetworkVideoDelivery delivery) + { + switch (delivery) + { + case NetworkVideoDelivery::RELIABLE: + return info.maxReliablePayload; + + case NetworkVideoDelivery::DATAGRAM: + return info.maxDatagramPayload; + + case NetworkVideoDelivery::BULK: + return info.maxBulkPayload; + } + return 0; + } + + void AppendLE16(std::vector& data, uint16_t value) + { + data.push_back(static_cast(value)); + data.push_back(static_cast(value >> 8)); + } + + void AppendLE32(std::vector& data, uint32_t value) + { + data.push_back(static_cast(value)); + data.push_back(static_cast(value >> 8)); + data.push_back(static_cast(value >> 16)); + data.push_back(static_cast(value >> 24)); + } + + void StoreLE32(uint8_t * data, uint32_t value) + { + data[0] = static_cast(value); + data[1] = static_cast(value >> 8); + data[2] = static_cast(value >> 16); + data[3] = static_cast(value >> 24); + } + + unsigned MagnitudeBits(uint16_t value) + { + unsigned bits = 0; + while (value) + { + ++bits; + value >>= 1; + } + return bits; + } + + bool Multiply(size_t left, size_t right, size_t& result) + { + if (right && left > std::numeric_limits::max() / right) + return false; + result = left * right; + return true; + } + + bool Add(size_t left, size_t right, size_t& result) + { + if (left > std::numeric_limits::max() - right) + return false; + result = left + right; + return true; + } + + DXGI_FORMAT ProfileFormat(FramePixel pixel) + { + switch (pixel) + { + case FramePixel::BGRA8: + return DXGI_FORMAT_B8G8R8A8_UNORM; + + case FramePixel::RGBA8: + return DXGI_FORMAT_R8G8B8A8_UNORM; + + case FramePixel::RGB10A2: + return DXGI_FORMAT_R10G10B10A2_UNORM; + + case FramePixel::RGBA16F: + return DXGI_FORMAT_R16G16B16A16_FLOAT; + } + return DXGI_FORMAT_UNKNOWN; + } + + // The CDF 9/7 lifting constants and pass order below are derived from + // PyroWave commit 89f7e47d4abbf650c91fae766728af866c5e32a0. + // Copyright (c) 2025 Hans-Kristian Arntzen, provided under the MIT license. + constexpr char ENCODER_SHADER[] = R"hlsl( +Texture2D Source : register(t0); +RWStructuredBuffer Wave : register(u0); +RWStructuredBuffer Scratch : register(u1); +RWStructuredBuffer Quant : register(u2); + +cbuffer Parameters : register(b0) +{ + uint4 Dim0; + uint4 Op0; + uint4 Op1; + float4 Values; +}; + +uint mirrorCoord(int coord, uint length) +{ + if (coord < 0) + return uint(-coord); + if (uint(coord) >= length) + return 2u * length - 2u - uint(coord); + return uint(coord); +} + +float3 rgbToYCbCr(float3 rgb, bool bt2020) +{ + const float kr = bt2020 ? 0.2627 : 0.2126; + const float kb = bt2020 ? 0.0593 : 0.0722; + const float kg = 1.0 - kr - kb; + const float y = dot(rgb, float3(kr, kg, kb)); + return float3(y, (rgb.b - y) / (2.0 * (1.0 - kb)), + (rgb.r - y) / (2.0 * (1.0 - kr))); +} + +float3 loadYCbCr(uint2 coord, bool bt2020) +{ + coord.x = min(coord.x, Dim0.x - 1u); + coord.y = min(coord.y, Dim0.y - 1u); + return rgbToYCbCr(Source.Load(int3(coord, 0)).rgb, bt2020); +} + +[numthreads(8, 8, 1)] +void ColorMain(uint3 tid : SV_DispatchThreadID) +{ + const uint component = Op0.z; + if (tid.x >= Dim0.z || tid.y >= Dim0.w) + return; + + const bool bt2020 = Op1.z != 0u; + float3 yuv; + if (Op0.w != 0u && component != 0u) + { + const uint2 src = tid.xy * 2u; + yuv = loadYCbCr(src, bt2020); + yuv += loadYCbCr(src + uint2(1u, 0u), bt2020); + yuv += loadYCbCr(src + uint2(0u, 1u), bt2020); + yuv += loadYCbCr(src + uint2(1u, 1u), bt2020); + yuv *= 0.25; + } + else + yuv = loadYCbCr(tid.xy, bt2020); + + float value; + if (component == 0u) + value = (16.0 / 255.0 + yuv.x * (219.0 / 255.0)) - 0.5; + else + value = yuv[component] * (224.0 / 255.0); + Wave[tid.y * Dim0.z + tid.x] = value; +} + +[numthreads(8, 8, 1)] +void LiftMain(uint3 tid : SV_DispatchThreadID) +{ + if (tid.x >= Op0.x || tid.y >= Op0.y) + return; + + const bool vertical = Op0.z != 0u; + const uint coord = vertical ? tid.y : tid.x; + if ((coord & 1u) != Op1.x) + return; + + const uint length = vertical ? Op0.y : Op0.x; + const uint before = mirrorCoord(int(coord) - 1, length); + const uint after = mirrorCoord(int(coord) + 1, length); + + const uint centerIndex = tid.y * Dim0.z + tid.x; + const uint beforeIndex = vertical ? before * Dim0.z + tid.x : + tid.y * Dim0.z + before; + const uint afterIndex = vertical ? after * Dim0.z + tid.x : + tid.y * Dim0.z + after; + if (vertical) + Scratch[centerIndex] += Values.x * + (Scratch[beforeIndex] + Scratch[afterIndex]); + else + Wave[centerIndex] += Values.x * + (Wave[beforeIndex] + Wave[afterIndex]); +} + +[numthreads(8, 8, 1)] +void HorizontalSplitMain(uint3 tid : SV_DispatchThreadID) +{ + if (tid.x >= Op0.x || tid.y >= Op0.y) + return; + const uint half = Op0.x >> 1; + const bool high = tid.x >= half; + const uint srcX = high ? 2u * (tid.x - half) + 1u : 2u * tid.x; + Scratch[tid.y * Dim0.z + tid.x] = + Wave[tid.y * Dim0.z + srcX] * (high ? Values.z : Values.y); +} + +[numthreads(8, 8, 1)] +void VerticalSplitMain(uint3 tid : SV_DispatchThreadID) +{ + if (tid.x >= Op0.x || tid.y >= Op0.y) + return; + const uint half = Op0.y >> 1; + const bool high = tid.y >= half; + const uint srcY = high ? 2u * (tid.y - half) + 1u : 2u * tid.y; + Wave[tid.y * Dim0.z + tid.x] = + Scratch[srcY * Dim0.z + tid.x] * (high ? Values.z : Values.y); +} + +int quantize(float value) +{ + const float magnitude = floor(abs(value) * Values.w); + const int result = int(min(magnitude, 32767.0)); + return value < 0.0 ? -result : result; +} + +[numthreads(64, 1, 1)] +void QuantMain(uint3 tid : SV_DispatchThreadID) +{ + const uint first = tid.x * 2u; + if (first >= Op1.y) + return; + const int low = quantize(Wave[first]); + const int high = first + 1u < Op1.y ? quantize(Wave[first + 1u]) : 0; + Quant[tid.x] = (uint(low) & 0xffffu) | ((uint(high) & 0xffffu) << 16u); +} +)hlsl"; + + struct EncodedBlock + { + uint32_t offset = 0; + uint32_t length = 0; + }; + + struct EncodedFrame + { + uint64_t configEpoch = 0; + uint64_t frameID = 0; + uint64_t captureTimeNs = 0; + uint32_t deadlineMs = 0; + std::vector data; + std::vector blocks; + }; + + struct ShaderParameters + { + uint32_t dim0 [4] = {}; + uint32_t op0 [4] = {}; + uint32_t op1 [4] = {}; + float values[4] = {}; + }; + + static_assert(sizeof(ShaderParameters) == 16U * sizeof(uint32_t), + "Unexpected shader parameter layout"); + + class PyroWaveD3D12Encoder + { + private: + enum Pipeline + { + PIPELINE_COLOR, + PIPELINE_LIFT, + PIPELINE_HORIZONTAL_SPLIT, + PIPELINE_VERTICAL_SPLIT, + PIPELINE_QUANT, + PIPELINE_COUNT, + }; + + struct Plane + { + uint32_t width = 0; + uint32_t height = 0; + uint32_t levels = 0; + uint64_t readbackOffset = 0; + uint64_t readbackBytes = 0; + }; + + ComPtr m_device; + CD3D12CommandQueue m_queue; + ComPtr m_rootSignature; + ComPtr m_pipelines[PIPELINE_COUNT]; + ComPtr m_descriptors; + ComPtr m_wave; + ComPtr m_scratch; + ComPtr m_quant; + ComPtr m_readback; + D12Sync m_sync; + TexLease m_lease; + FrameIn m_frame; + FrameCfg m_cfg; + Plane m_planes[3] = {}; + LGNetVideoChromaSubsampling m_chroma = 0; + uint64_t m_readbackBytes = 0; + uint32_t m_sequence = 0; + bool m_ready = false; + bool m_inFlight = false; + + static bool CreateBuffer(ID3D12Device3 * device, uint64_t bytes, + D3D12_HEAP_TYPE heapType, D3D12_RESOURCE_STATES state, + D3D12_RESOURCE_FLAGS flags, ComPtr& resource) + { + if (!device || !bytes) + return false; + + D3D12_HEAP_PROPERTIES heap = {}; + + heap.Type = heapType; + heap.CPUPageProperty = D3D12_CPU_PAGE_PROPERTY_UNKNOWN; + heap.MemoryPoolPreference = D3D12_MEMORY_POOL_UNKNOWN; + heap.CreationNodeMask = 1; + heap.VisibleNodeMask = 1; + + D3D12_RESOURCE_DESC desc = {}; + + desc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + desc.Width = bytes; + desc.Height = 1; + desc.DepthOrArraySize = 1; + desc.MipLevels = 1; + desc.SampleDesc.Count = 1; + desc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + desc.Flags = flags; + + const HRESULT hr = device->CreateCommittedResource(&heap, + D3D12_HEAP_FLAG_CREATE_NOT_ZEROED, &desc, state, nullptr, + IID_PPV_ARGS(&resource)); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to create network video buffer"); + return false; + } + return true; + } + + static bool Compile(const char * entry, ComPtr& shader) + { + ComPtr errors; + const HRESULT hr = D3DCompile(ENCODER_SHADER, + sizeof(ENCODER_SHADER) - 1U, "NetworkPyroWave", nullptr, nullptr, + entry, "cs_5_0", D3DCOMPILE_OPTIMIZATION_LEVEL3, 0, + &shader, &errors); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, "Failed to compile the network video shader"); + if (errors) + DEBUG_ERROR("%s", + static_cast(errors->GetBufferPointer())); + return false; + } + return true; + } + + bool CreatePipelines() + { + D3D12_DESCRIPTOR_RANGE ranges[4] = {}; + + ranges[0].RangeType = + D3D12_DESCRIPTOR_RANGE_TYPE_SRV; + ranges[0].NumDescriptors = 1; + ranges[0].BaseShaderRegister = 0; + ranges[0].OffsetInDescriptorsFromTableStart = 0; + for (unsigned i = 1; i < 4; ++i) + { + ranges[i].RangeType = + D3D12_DESCRIPTOR_RANGE_TYPE_UAV; + ranges[i].NumDescriptors = 1; + ranges[i].BaseShaderRegister = i - 1; + ranges[i].OffsetInDescriptorsFromTableStart = i; + } + + D3D12_ROOT_PARAMETER parameters[2] = {}; + + parameters[0].ParameterType = + D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE; + parameters[0].ShaderVisibility = + D3D12_SHADER_VISIBILITY_ALL; + parameters[0].DescriptorTable.NumDescriptorRanges = 4; + parameters[0].DescriptorTable.pDescriptorRanges = ranges; + parameters[1].ParameterType = + D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS; + parameters[1].ShaderVisibility = + D3D12_SHADER_VISIBILITY_ALL; + parameters[1].Constants.ShaderRegister = 0; + parameters[1].Constants.RegisterSpace = 0; + parameters[1].Constants.Num32BitValues = + sizeof(ShaderParameters) / sizeof(uint32_t); + + D3D12_ROOT_SIGNATURE_DESC signature = {}; + + signature.NumParameters = 2; + signature.pParameters = parameters; + signature.Flags = D3D12_ROOT_SIGNATURE_FLAG_NONE; + + ComPtr blob; + ComPtr errors; + HRESULT hr = D3D12SerializeRootSignature(&signature, + D3D_ROOT_SIGNATURE_VERSION_1, &blob, &errors); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, + "Failed to serialize the network video root signature"); + if (errors) + DEBUG_ERROR("%s", + static_cast(errors->GetBufferPointer())); + return false; + } + + hr = m_device->CreateRootSignature(0, blob->GetBufferPointer(), + blob->GetBufferSize(), IID_PPV_ARGS(&m_rootSignature)); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, + "Failed to create the network video root signature"); + return false; + } + + static const char * ENTRIES[PIPELINE_COUNT] = + { + "ColorMain", + "LiftMain", + "HorizontalSplitMain", + "VerticalSplitMain", + "QuantMain", + }; + for (unsigned i = 0; i < PIPELINE_COUNT; ++i) + { + ComPtr shader; + if (!Compile(ENTRIES[i], shader)) + return false; + + D3D12_COMPUTE_PIPELINE_STATE_DESC desc = {}; + + desc.pRootSignature = m_rootSignature.Get(); + desc.CS.pShaderBytecode = shader->GetBufferPointer(); + desc.CS.BytecodeLength = shader->GetBufferSize(); + hr = m_device->CreateComputePipelineState( + &desc, IID_PPV_ARGS(&m_pipelines[i])); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, + "Failed to create a network video compute pipeline"); + return false; + } + } + + D3D12_DESCRIPTOR_HEAP_DESC heap = {}; + + heap.Type = D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV; + heap.NumDescriptors = 4; + heap.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE; + hr = m_device->CreateDescriptorHeap( + &heap, IID_PPV_ARGS(&m_descriptors)); + if (FAILED(hr)) + { + DEBUG_ERROR_HR(hr, + "Failed to create the network video descriptor heap"); + return false; + } + return true; + } + + D3D12_CPU_DESCRIPTOR_HANDLE Descriptor(unsigned index) const + { + D3D12_CPU_DESCRIPTOR_HANDLE result = + m_descriptors->GetCPUDescriptorHandleForHeapStart(); + result.ptr += static_cast(index) * + m_device->GetDescriptorHandleIncrementSize( + D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV); + return result; + } + + void CreateDescriptors(ID3D12Resource * source) + { + D3D12_SHADER_RESOURCE_VIEW_DESC srv = {}; + + srv.Format = ProfileFormat(m_cfg.profile.pixel); + srv.ViewDimension = D3D12_SRV_DIMENSION_TEXTURE2D; + srv.Shader4ComponentMapping = D3D12_DEFAULT_SHADER_4_COMPONENT_MAPPING; + srv.Texture2D.MipLevels = 1; + m_device->CreateShaderResourceView(source, &srv, Descriptor(0)); + + ID3D12Resource * resources[] = + { + m_wave.Get(), + m_scratch.Get(), + m_quant.Get(), + }; + const uint64_t elements[] = + { + m_wave->GetDesc().Width / sizeof(float), + m_scratch->GetDesc().Width / sizeof(float), + m_quant->GetDesc().Width / sizeof(uint32_t), + }; + for (unsigned i = 0; i < 3; ++i) + { + D3D12_UNORDERED_ACCESS_VIEW_DESC uav = {}; + + uav.Format = DXGI_FORMAT_UNKNOWN; + uav.ViewDimension = D3D12_UAV_DIMENSION_BUFFER; + uav.Buffer.NumElements = static_cast(elements[i]); + uav.Buffer.StructureByteStride = sizeof(uint32_t); + m_device->CreateUnorderedAccessView( + resources[i], nullptr, &uav, Descriptor(i + 1)); + } + } + + static void UAVBarrier(ID3D12GraphicsCommandList * list, + ID3D12Resource * resource) + { + D3D12_RESOURCE_BARRIER barrier = {}; + + barrier.Type = D3D12_RESOURCE_BARRIER_TYPE_UAV; + barrier.UAV.pResource = resource; + list->ResourceBarrier(1, &barrier); + } + + static void Transition(ID3D12GraphicsCommandList * list, + ID3D12Resource * resource, D3D12_RESOURCE_STATES before, + D3D12_RESOURCE_STATES after) + { + D3D12_RESOURCE_BARRIER barrier = {}; + + barrier.Type = + D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + barrier.Transition.pResource = resource; + barrier.Transition.StateBefore = before; + barrier.Transition.StateAfter = after; + barrier.Transition.Subresource = + D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + list->ResourceBarrier(1, &barrier); + } + + void Dispatch(ID3D12GraphicsCommandList * list, Pipeline pipeline, + const ShaderParameters& parameters, uint32_t x, uint32_t y) + { + list->SetPipelineState(m_pipelines[pipeline].Get()); + list->SetComputeRoot32BitConstants(1, + sizeof(parameters) / sizeof(uint32_t), ¶meters, 0); + list->Dispatch(x, y, 1); + } + + bool Configure(const FrameCfg& cfg, + LGNetVideoChromaSubsampling chroma, ID3D12Resource * source) + { + if (!source || !SupportedProfile(cfg.profile) || !cfg.width || + !cfg.height || cfg.width > MAX_WIDTH || cfg.height > MAX_HEIGHT) + return false; + + ComPtr device; + HRESULT hr = source->GetDevice(IID_PPV_ARGS(&device)); + if (FAILED(hr) || !device) + return false; + + const uint32_t alignedWidth = std::max( + Align(cfg.width, PYROWAVE_ALIGNMENT), PYROWAVE_MIN_DIMENSION); + const uint32_t alignedHeight = std::max( + Align(cfg.height, PYROWAVE_ALIGNMENT), PYROWAVE_MIN_DIMENSION); + if (alignedWidth > MAX_WIDTH || alignedHeight > MAX_HEIGHT) + return false; + + Plane planes[3] = {}; + uint64_t readbackBytes = 0; + uint64_t maxPixels = 0; + for (unsigned i = 0; i < 3; ++i) + { + const bool subsampled = chroma == LG_NET_VIDEO_CHROMA_420 && i; + planes[i].width = subsampled ? alignedWidth / 2U : alignedWidth; + planes[i].height = subsampled ? alignedHeight / 2U : alignedHeight; + planes[i].levels = subsampled ? + PYROWAVE_LEVELS - 1U : PYROWAVE_LEVELS; + const uint64_t pixels = + static_cast(planes[i].width) * planes[i].height; + planes[i].readbackOffset = readbackBytes; + planes[i].readbackBytes = ((pixels + 1U) / 2U) * sizeof(uint32_t); + if (readbackBytes > UINT64_MAX - planes[i].readbackBytes) + return false; + readbackBytes += planes[i].readbackBytes; + maxPixels = std::max(maxPixels, pixels); + } + if (!maxPixels || maxPixels > UINT_MAX || !readbackBytes) + return false; + + Reset(); + m_device = device; + m_cfg = cfg; + m_chroma = chroma; + std::copy(std::begin(planes), std::end(planes), std::begin(m_planes)); + m_readbackBytes = readbackBytes; + + if (!m_queue.Init(m_device.Get(), D3D12_COMMAND_LIST_TYPE_COMPUTE, + L"Network Video", CD3D12CommandSlot::FAST, 1) || + !CreatePipelines() || + !CreateBuffer(m_device.Get(), maxPixels * sizeof(float), + D3D12_HEAP_TYPE_DEFAULT, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, m_wave) || + !CreateBuffer(m_device.Get(), maxPixels * sizeof(float), + D3D12_HEAP_TYPE_DEFAULT, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, m_scratch) || + !CreateBuffer(m_device.Get(), + ((maxPixels + 1U) / 2U) * sizeof(uint32_t), + D3D12_HEAP_TYPE_DEFAULT, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, m_quant) || + !CreateBuffer(m_device.Get(), readbackBytes, + D3D12_HEAP_TYPE_READBACK, + D3D12_RESOURCE_STATE_COPY_DEST, + D3D12_RESOURCE_FLAG_NONE, m_readback)) + { + Reset(); + return false; + } + + m_ready = true; + return true; + } + + bool SameDevice(ID3D12Resource * source) const + { + if (!m_device || !source) + return false; + ComPtr device; + return SUCCEEDED(source->GetDevice(IID_PPV_ARGS(&device))) && + device.Get() == m_device.Get(); + } + + void RecordPlane(ID3D12GraphicsCommandList * list, unsigned component) + { + static constexpr float LIFT [4] = + { + -1.586134342059924f, + -0.052980118572961f, + 0.882911075530934f, + 0.443506852043971f, + }; + static constexpr uint32_t PHASE[4] = { 1, 0, 1, 0 }; + static constexpr float K = 1.230174104914001f; + static constexpr float INV_K = 1.0f / K; + + const Plane& plane = m_planes[component]; + + ShaderParameters parameters = {}; + parameters.dim0[0] = m_cfg.width; + parameters.dim0[1] = m_cfg.height; + parameters.dim0[2] = plane.width; + parameters.dim0[3] = plane.height; + parameters.op0[2] = component; + parameters.op0[3] = m_chroma == LG_NET_VIDEO_CHROMA_420; + parameters.op1[2] = m_cfg.profile.signal == FrameSignal::PQ_BT2020; + parameters.values[1] = INV_K; + parameters.values[2] = K; + parameters.values[3] = PYROWAVE_QUANT_SCALE; + Dispatch(list, PIPELINE_COLOR, parameters, + (plane.width + 7U) / 8U, (plane.height + 7U) / 8U); + UAVBarrier(list, m_wave.Get()); + + uint32_t activeWidth = plane.width; + uint32_t activeHeight = plane.height; + for (unsigned level = 0; level < plane.levels; ++level) + { + parameters.op0[0] = activeWidth; + parameters.op0[1] = activeHeight; + parameters.op0[2] = 0; + for (unsigned pass = 0; pass < 4; ++pass) + { + parameters.op1[0] = PHASE[pass]; + parameters.values[0] = LIFT[pass]; + Dispatch(list, PIPELINE_LIFT, parameters, + (activeWidth + 7U) / 8U, (activeHeight + 7U) / 8U); + UAVBarrier(list, m_wave.Get()); + } + + Dispatch(list, PIPELINE_HORIZONTAL_SPLIT, parameters, + (activeWidth + 7U) / 8U, (activeHeight + 7U) / 8U); + UAVBarrier(list, m_scratch.Get()); + + parameters.op0[2] = 1; + for (unsigned pass = 0; pass < 4; ++pass) + { + parameters.op1[0] = PHASE[pass]; + parameters.values[0] = LIFT[pass]; + Dispatch(list, PIPELINE_LIFT, parameters, + (activeWidth + 7U) / 8U, (activeHeight + 7U) / 8U); + UAVBarrier(list, m_scratch.Get()); + } + + Dispatch(list, PIPELINE_VERTICAL_SPLIT, parameters, + (activeWidth + 7U) / 8U, (activeHeight + 7U) / 8U); + UAVBarrier(list, m_wave.Get()); + activeWidth >>= 1; + activeHeight >>= 1; + } + + const uint32_t pixels = plane.width * plane.height; + + parameters.op1[1] = pixels; + Dispatch(list, PIPELINE_QUANT, parameters, + ((pixels + 1U) / 2U + 63U) / 64U, 1); + UAVBarrier(list, m_quant.Get()); + Transition(list, m_quant.Get(), D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_STATE_COPY_SOURCE); + list->CopyBufferRegion(m_readback.Get(), plane.readbackOffset, + m_quant.Get(), 0, plane.readbackBytes); + Transition(list, m_quant.Get(), D3D12_RESOURCE_STATE_COPY_SOURCE, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + } + + int16_t Coefficient( + const uint8_t * data, unsigned component, uint32_t x, uint32_t y) const + { + const Plane& plane = m_planes[component]; + const uint64_t index = static_cast(y) * plane.width + x; + const uint64_t wordIndex = index / 2U; + const uint8_t * source = data + plane.readbackOffset + + wordIndex * sizeof(uint32_t); + const uint32_t word = + static_cast(source[0]) | + static_cast(source[1]) << 8 | + static_cast(source[2]) << 16 | + static_cast(source[3]) << 24; + return static_cast(index & 1U ? word >> 16 : word); + } + + static void CoefficientCoord(unsigned linear, uint32_t& x, uint32_t& y) + { + y = (linear & 1U) | (((linear >> 3U) & 3U) << 1U); + x = ((linear >> 1U) & 3U) | (((linear >> 5U) & 1U) << 2U); + } + + bool AppendBlock(const uint8_t * coefficients, unsigned component, + uint32_t bandX, uint32_t bandY, uint32_t bandWidth, + uint32_t bandHeight, uint32_t blockX, uint32_t blockY, + uint32_t blockIndex, EncodedFrame& output) + { + std::array, 16> values = {}; + uint16_t ballot = 0; + for (unsigned block = 0; block < 16; ++block) + { + const uint32_t block8X = block & 3U; + const uint32_t block8Y = block >> 2U; + + bool nonzero = false; + for (unsigned linear = 0; linear < 64; ++linear) + { + uint32_t localX; + uint32_t localY; + CoefficientCoord(linear, localX, localY); + const uint32_t x = blockX * 32U + block8X * 8U + localX; + const uint32_t y = blockY * 32U + block8Y * 8U + localY; + if (x < bandWidth && y < bandHeight) + values[block][linear] = Coefficient(coefficients, component, + bandX + x, bandY + y); + nonzero |= values[block][linear] != 0; + } + if (nonzero) + ballot |= static_cast(1U << block); + } + if (!ballot) + return true; + + std::vector controls; + std::vector scales; + std::vector magnitude; + std::vector signs; + controls.reserve(16); + scales.reserve(16); + for (unsigned block = 0; block < 16; ++block) + { + if (!(ballot & (1U << block))) + continue; + unsigned maxPlanes = 0; + unsigned planes[8] = {}; + for (unsigned subblock = 0; subblock < 8; ++subblock) + for (unsigned pixel = 0; pixel < 8; ++pixel) + { + const int value = values[block][subblock * 8U + pixel]; + const uint16_t absolute = static_cast( + value < 0 ? -value : value); + planes[subblock] = std::max( + planes[subblock], MagnitudeBits(absolute)); + maxPlanes = std::max(maxPlanes, planes[subblock]); + } + + const unsigned qualityPlanes = maxPlanes > 3U ? maxPlanes - 3U : 0; + uint16_t control = 0; + for (unsigned subblock = 0; subblock < 8; ++subblock) + { + const unsigned coded = planes[subblock] > qualityPlanes ? + planes[subblock] - qualityPlanes : 0; + control |= static_cast(std::min(coded, 3U) << + (subblock * 2U)); + } + controls.push_back(control); + scales.push_back(static_cast( + (PYROWAVE_QSCALE_CODE << 4U) | qualityPlanes)); + + for (unsigned subblock = 0; subblock < 8; ++subblock) + { + const unsigned coded = + ((control >> (subblock * 2U)) & 3U) + qualityPlanes; + for (unsigned plane = 0; plane < coded; ++plane) + { + const unsigned shift = coded - plane - 1U; + uint8_t byte = 0; + for (unsigned pixel = 0; pixel < 8; ++pixel) + { + const int value = values[block][subblock * 8U + pixel]; + const uint16_t absolute = static_cast( + value < 0 ? -value : value); + byte |= static_cast( + ((absolute >> shift) & 1U) << pixel); + } + magnitude.push_back(byte); + } + } + } + + uint8_t signByte = 0; + unsigned signBit = 0; + for (unsigned block = 0, compact = 0; block < 16; ++block) + { + if (!(ballot & (1U << block))) + continue; + const uint16_t control = controls[compact]; + const unsigned qualityPlanes = scales[compact] & 0xfU; + ++compact; + for (unsigned subblock = 0; subblock < 8; ++subblock) + { + const unsigned coded = + ((control >> (subblock * 2U)) & 3U) + qualityPlanes; + if (!coded) + continue; + for (unsigned pixel = 0; pixel < 8; ++pixel) + { + const int value = values[block][subblock * 8U + pixel]; + if (!value) + continue; + if (value < 0) + signByte |= static_cast(1U << signBit); + if (++signBit == 8U) + { + signs.push_back(signByte); + signByte = 0; + signBit = 0; + } + } + } + } + if (signBit) + signs.push_back(signByte); + + const size_t start = output.data.size(); + output.data.resize(start + 8U, 0); + for (uint16_t control : controls) + AppendLE16(output.data, control); + output.data.insert(output.data.end(), scales.begin(), scales.end()); + output.data.insert( + output.data.end(), magnitude.begin(), magnitude.end()); + output.data.insert(output.data.end(), signs.begin(), signs.end()); + while (output.data.size() & 3U) + output.data.push_back(0); + + const size_t bytes = output.data.size() - start; + const size_t words = bytes / sizeof(uint32_t); + if (words > 0xfffU || blockIndex > 0xffffffU) + return false; + const uint32_t word0 = ballot | + static_cast(words << 16U) | + (m_sequence << 28U); + const uint32_t word1 = PYROWAVE_QUANT_CODE | (blockIndex << 8U); + StoreLE32(output.data.data() + start, word0); + StoreLE32(output.data.data() + start + 4U, word1); + output.blocks.push_back( + { static_cast(start), static_cast(bytes) }); + return output.data.size() <= MAX_FRAME_LENGTH && + output.blocks.size() <= MAX_BLOCKS; + } + + bool Packetize(const uint8_t * coefficients, EncodedFrame& output) + { + output.data.assign(8, 0); + output.blocks.clear(); + uint32_t blockIndex = 0; + for (int level = static_cast(PYROWAVE_LEVELS) - 1; + level >= 0; --level) + for (unsigned component = 0; component < 3; ++component) + { + if (m_chroma == LG_NET_VIDEO_CHROMA_420 && component && !level) + continue; + const uint32_t localLevel = + m_chroma == LG_NET_VIDEO_CHROMA_420 && component ? + static_cast(level - 1) : + static_cast(level); + const uint32_t bandWidth = + m_planes[component].width >> (localLevel + 1U); + const uint32_t bandHeight = + m_planes[component].height >> (localLevel + 1U); + const unsigned firstBand = + level == static_cast(PYROWAVE_LEVELS) - 1 ? 0 : 1; + for (unsigned band = firstBand; band < 4; ++band) + { + const uint32_t bandX = band & 1U ? bandWidth : 0; + const uint32_t bandY = band & 2U ? bandHeight : 0; + const uint32_t blocksX = (bandWidth + 31U) / 32U; + const uint32_t blocksY = (bandHeight + 31U) / 32U; + for (uint32_t y = 0; y < blocksY; ++y) + for (uint32_t x = 0; x < blocksX; ++x) + { + if (blockIndex >= MAX_BLOCKS || + !AppendBlock(coefficients, component, bandX, bandY, + bandWidth, bandHeight, x, y, blockIndex, output)) + return false; + ++blockIndex; + } + } + } + + if (blockIndex > MAX_BLOCKS || output.blocks.empty() || + output.data.size() > MAX_FRAME_LENGTH) + return false; + + const uint32_t word0 = (m_cfg.width - 1U) | + ((m_cfg.height - 1U) << 14U) | (m_sequence << 28U) | + (1U << 31U); + const bool hdr = + m_cfg.profile.signal == FrameSignal::PQ_BT2020; + const uint32_t word1 = + static_cast(output.blocks.size()) | + ((m_chroma == LG_NET_VIDEO_CHROMA_444 ? 1U : 0U) << 26U) | + (static_cast(hdr) << 27U) | + (static_cast(hdr) << 28U) | + (static_cast(hdr) << 29U) | + (1U << 30U); + StoreLE32(output.data.data(), word0); + StoreLE32(output.data.data() + 4U, word1); + output.blocks[0].offset = 0; + output.blocks[0].length += 8U; + + return true; + } + + public: + ~PyroWaveD3D12Encoder() + { + Reset(); + } + + bool Busy() const + { + return m_inFlight; + } + + bool Submit(FrameIn frame, TexLease lease, const FrameCfg& cfg, + LGNetVideoChromaSubsampling chroma) + { + if (m_inFlight || !lease || !lease.Get() || + lease->State() != D3D12_RESOURCE_STATE_COMMON) + return false; + ID3D12Resource * source = lease->Get(); + if (!m_ready || !SameCfg(m_cfg, cfg) || m_chroma != chroma || + !SameDevice(source)) + if (!Configure(cfg, chroma, source)) + return false; + + CD3D12CommandSlot * command = m_queue.Try(0); + if (!command) + return false; + if (!lease->Wait(*command)) + { + command->Cancel(); + return false; + } + + CreateDescriptors(source); + ID3D12DescriptorHeap * heaps[] = { m_descriptors.Get() }; + + ComPtr list = command->GetGfxList(); + list->SetDescriptorHeaps(1, heaps); + list->SetComputeRootSignature(m_rootSignature.Get()); + list->SetComputeRootDescriptorTable( + 0, m_descriptors->GetGPUDescriptorHandleForHeapStart()); + for (unsigned component = 0; component < 3; ++component) + RecordPlane(list.Get(), component); + if (!command->Execute(&m_sync)) + return false; + + m_frame = std::move(frame); + m_lease = std::move(lease); + m_inFlight = true; + return true; + } + + D12SyncState State() const + { + return m_inFlight ? m_sync.State() : D12SyncState::FAILED; + } + + bool Complete(uint64_t configEpoch, uint64_t frameID, + EncodedFrame& output) + { + if (!m_inFlight || m_sync.State() != D12SyncState::READY) + return false; + + const D3D12_RANGE readRange = + { + 0, + static_cast(m_readbackBytes), + }; + void * mapped = nullptr; + const HRESULT hr = m_readback->Map(0, &readRange, &mapped); + const uint8_t * data = static_cast(mapped); + if (FAILED(hr) || !data) + { + m_inFlight = false; + m_lease.Reset(); + return false; + } + + output = {}; + + const bool result = Packetize(data, output); + + const D3D12_RANGE writeRange = { 0, 0 }; + m_readback->Unmap(0, &writeRange); + if (result) + { + output.configEpoch = configEpoch; + output.frameID = frameID; + output.captureTimeNs = MonotonicNs(); + output.deadlineMs = MAX_FRAME_LATENCY_MS; + } + m_sequence = (m_sequence + 1U) & 7U; + m_inFlight = false; + m_lease.Reset(); + m_sync = {}; + return result; + } + + void Reset() + { + m_queue.WaitForIdle(); + m_inFlight = false; + m_ready = false; + m_lease.Reset(); + m_sync = {}; + m_readback.Reset(); + m_quant.Reset(); + m_scratch.Reset(); + m_wave.Reset(); + m_descriptors.Reset(); + for (ComPtr& pipeline : m_pipelines) + pipeline.Reset(); + m_rootSignature.Reset(); + m_queue.DeInit(); + m_device.Reset(); + m_cfg = {}; + m_chroma = 0; + m_readbackBytes = 0; + std::fill(std::begin(m_planes), std::end(m_planes), Plane {}); + } + }; +} + +struct CNetworkVideo::Impl +{ + struct PeerState + { + NetworkVideoPeer peer = {}; + NetworkVideoPeerInfo info = {}; + LGNetVideoSubscribe subscription = {}; + LGNetVideoFeedback feedback = {}; + uint64_t serial = 0; + uint64_t componentEpoch = 0; + uint64_t statusSequence = 0; + uint64_t lastFrameID = 0; + uint64_t lastCompleteID = 0; + uint64_t lastCaptureTimeNs = 0; + uint64_t sendReadyGeneration[3] = {}; + uint32_t droppedFrames = 0; + uint8_t blockedDeliveries = 0; + bool subscribed = false; + bool configured = false; + bool congested = false; + bool keyframePending = false; + bool sendFailed = false; + }; + + struct PendingFrame + { + FrameIn frame; + TexLease lease; + FrameCfg cfg; + uint64_t configEpoch = 0; + }; + + struct SendItem + { + uint64_t serial = 0; + NetworkVideoPeer peer = {}; + NetworkVideoDelivery delivery = + NetworkVideoDelivery::RELIABLE; + uint16_t messageType = 0; + LGNetEnvelopeFlags flags = + LG_NET_ENVELOPE_FINAL; + uint64_t componentEpoch = 0; + uint64_t requestID = 0; + uint64_t frameID = 0; + std::shared_ptr> payload; + std::shared_ptr encoded; + uint32_t fragmentIndex = 0; + uint32_t fragmentCount = 0; + uint32_t blockIndex = 0; + uint32_t blockOffset = 0; + uint32_t payloadLimit = 0; + }; + + CSRWLock m_processLock; + CSRWLock m_senderCallbackLock; + CSRWLock m_lock; + + INetworkVideoSender * m_sender = nullptr; + bool m_started = false; + bool m_stopped = false; + bool m_failed = false; + bool m_changing = false; + + FrameCfg m_activeCfg = {}; + FrameCfg m_pendingCfg = {}; + bool m_active = false; + bool m_pending = false; + + uint64_t m_configEpoch = 0; + uint64_t m_nextFrameID = 0; + uint64_t m_nextPeerSerial = 0; + uint64_t m_nextSendSerial = 0; + + std::vector m_peers; + std::deque m_frames; + std::deque m_sends; + size_t m_sendBytes = 0; + + PyroWaveD3D12Encoder m_encoder; + + PeerState * FindPeerNL(const NetworkVideoPeer& peer) + { + const auto found = std::find_if(m_peers.begin(), m_peers.end(), + [&peer](const PeerState& current) + { + return SamePeer(current.peer, peer); + }); + return found == m_peers.end() ? nullptr : &*found; + } + + const PeerState * FindPeerNL(const NetworkVideoPeer& peer) const + { + const auto found = std::find_if(m_peers.begin(), m_peers.end(), + [&peer](const PeerState& current) + { + return SamePeer(current.peer, peer); + }); + return found == m_peers.end() ? nullptr : &*found; + } + + bool HasSubscribersNL() const + { + for (const PeerState& peer : m_peers) + if (peer.subscribed) + return true; + return false; + } + + LGNetVideoChromaSubsampling ChromaNL() const + { + if (!m_active) + return 0; + if (m_activeCfg.profile.signal == FrameSignal::PQ_BT2020 || + (m_activeCfg.width & 1U) || (m_activeCfg.height & 1U)) + return LG_NET_VIDEO_CHROMA_444; + return LG_NET_VIDEO_CHROMA_420; + } + + LGNetVideoStreamConfig StreamConfigNL() const + { + LGNetVideoStreamConfig config = {}; + if (!m_active || !m_configEpoch) + return config; + const bool hdr = + m_activeCfg.profile.signal == FrameSignal::PQ_BT2020; + const LGNetVideoChromaSubsampling chroma = ChromaNL(); + config.streamID = STREAM_ID; + config.codec = LG_NET_VIDEO_CODEC_PYROWAVE; + config.codecVersion = + LG_NET_VIDEO_PYROWAVE_CODEC_VERSION_CURRENT; + config.configEpoch = m_configEpoch; + config.width = m_activeCfg.width; + config.height = m_activeCfg.height; + config.pixelFormat = chroma == LG_NET_VIDEO_CHROMA_420 ? + LG_NET_VIDEO_PIXEL_FORMAT_YUV420P10 : + LG_NET_VIDEO_PIXEL_FORMAT_YUV444P10; + config.chromaSubsampling = chroma; + config.colorPrimaries = hdr ? + LG_NET_COLOR_PRIMARIES_BT2020 : LG_NET_COLOR_PRIMARIES_BT709; + config.colorTransfer = hdr ? + LG_NET_COLOR_TRANSFER_PQ : LG_NET_COLOR_TRANSFER_BT709; + config.colorMatrix = hdr ? + LG_NET_COLOR_MATRIX_BT2020 : LG_NET_COLOR_MATRIX_BT709; + config.colorRange = LG_NET_COLOR_RANGE_LIMITED; + config.planeCount = 3; + config.bitDepth = 10; + config.flags = LG_NET_VIDEO_STREAM_GPU_PLANES_ONLY; + config.maxFrameLength = MAX_FRAME_LENGTH; + config.maxFragmentLength = LG_NET_VIDEO_MAX_FRAGMENT_LENGTH; + config.maxFrameLatencyMs = MAX_FRAME_LATENCY_MS; + if (hdr) + config.flags |= LG_NET_VIDEO_STREAM_HDR; + return config; + } + + bool PeerCanStreamNL(const PeerState& peer) const + { + if (!peer.subscribed || !m_active || !m_configEpoch) + return false; + const LGNetVideoSubscribe& request = peer.subscription; + const bool hdr = + m_activeCfg.profile.signal == FrameSignal::PQ_BT2020; + return (!request.preferredCodec || + request.preferredCodec == LG_NET_VIDEO_CODEC_PYROWAVE) && + request.maxWidth >= m_activeCfg.width && + request.maxHeight >= m_activeCfg.height && + request.maxFrameLength >= 8U && + (!(request.flags & LG_NET_VIDEO_SUBSCRIBE_REQUIRE_HDR) || hdr); + } + + bool QueueNL(const NetworkVideoPeer& peer, + NetworkVideoDelivery delivery, uint16_t messageType, + LGNetEnvelopeFlags flags, uint64_t componentEpoch, uint64_t requestID, + uint64_t frameID, std::shared_ptr> payload) + { + if (!payload || payload->size() > UINT32_MAX || + m_sends.size() >= MAX_SEND_ITEMS || + payload->size() > MAX_SEND_BYTES - + std::min(m_sendBytes, MAX_SEND_BYTES)) + return false; + try + { + SendItem item; + const size_t bytes = payload->size(); + item.serial = Next64(m_nextSendSerial); + item.peer = peer; + item.delivery = delivery; + item.messageType = messageType; + item.flags = flags; + item.componentEpoch = componentEpoch; + item.requestID = requestID; + item.frameID = frameID; + item.payload = std::move(payload); + const uint64_t serial = item.serial; + m_sends.push_back(std::move(item)); + m_nextSendSerial = serial; + m_sendBytes += bytes; + } + catch (...) + { + return false; + } + return true; + } + + static bool EncodedQueueBytes( + const EncodedFrame& encoded, size_t& bytes) + { + size_t blockBytes; + return Multiply(encoded.blocks.size(), sizeof(EncodedBlock), blockBytes) && + Add(encoded.data.size(), blockBytes, bytes); + } + + bool EncodedQueuedNL(const EncodedFrame * encoded, + uint64_t excludingSerial = 0) const + { + return std::any_of(m_sends.begin(), m_sends.end(), + [encoded, excludingSerial](const SendItem& item) + { + return item.serial != excludingSerial && + item.encoded.get() == encoded; + }); + } + + void ReleaseSendAccountingNL(const SendItem& item) + { + if (item.payload) + { + m_sendBytes -= item.payload->size(); + return; + } + if (!item.encoded || EncodedQueuedNL(item.encoded.get(), item.serial)) + return; + + size_t bytes; + if (EncodedQueueBytes(*item.encoded, bytes)) + m_sendBytes -= bytes; + } + + template + bool QueueFixedNL(const NetworkVideoPeer& peer, + NetworkVideoDelivery delivery, uint16_t messageType, + LGNetEnvelopeFlags flags, uint64_t componentEpoch, uint64_t requestID, + uint64_t frameID, const T& value, size_t size) + { + std::shared_ptr> payload; + try + { + payload = std::make_shared>(size); + } + catch (...) + { + return false; + } + if (!Encode(payload->data(), payload->size(), &value)) + return false; + return QueueNL(peer, delivery, messageType, flags, componentEpoch, + requestID, frameID, std::move(payload)); + } + + bool QueueConfigNL(PeerState& peer, uint64_t requestID = 0) + { + if (m_failed) + { + peer.configured = false; + return QueueStatusNL( + peer, LG_NET_VIDEO_STATE_ERROR, 6, requestID); + } + if (!PeerCanStreamNL(peer)) + { + peer.configured = false; + return QueueStatusNL( + peer, LG_NET_VIDEO_STATE_ERROR, 1, requestID); + } + LGNetVideoStreamConfig config = StreamConfigNL(); + config.maxFrameLength = std::min( + config.maxFrameLength, peer.subscription.maxFrameLength); + config.maxFragmentLength = std::min( + config.maxFragmentLength, config.maxFrameLength); + config.maxFrameLatencyMs = std::min( + config.maxFrameLatencyMs, peer.subscription.maxFrameLatencyMs); + if ((peer.subscription.flags & + LG_NET_VIDEO_SUBSCRIBE_ALLOW_DATAGRAMS) && + DeliverySupported(peer.info, NetworkVideoDelivery::DATAGRAM)) + config.flags |= LG_NET_VIDEO_STREAM_DATAGRAMS; + if (!lgNetVideoStreamConfigValid(&config) || + !QueueFixedNL(peer.peer, + NetworkVideoDelivery::RELIABLE, + LG_NET_VIDEO_MESSAGE_STREAM_CONFIG, + LG_NET_ENVELOPE_FINAL | (requestID ? + LG_NET_ENVELOPE_RESPONSE : 0), + m_configEpoch, requestID, 0, config, + LG_NET_VIDEO_STREAM_CONFIG_WIRE_SIZE)) + return false; + peer.componentEpoch = m_configEpoch; + peer.configured = true; + peer.lastCaptureTimeNs = 0; + peer.sendFailed = false; + return QueueStatusNL(peer, LG_NET_VIDEO_STATE_CONFIGURED, 0); + } + + bool QueueStatusNL(PeerState& peer, LGNetVideoState state, + uint32_t detail, uint64_t requestID = 0) + { + LGNetVideoStatus status = {}; + status.streamID = STREAM_ID; + status.state = state; + status.configEpoch = peer.componentEpoch; + status.lastFrameID = peer.lastFrameID; + status.statusSequence = Next64(peer.statusSequence); + if (peer.keyframePending) + status.flags |= LG_NET_VIDEO_STATUS_KEYFRAME_PENDING; + if (peer.congested) + status.flags |= LG_NET_VIDEO_STATUS_CONGESTED; + if (m_changing) + status.flags |= LG_NET_VIDEO_STATUS_RECONFIGURING; + status.detail = detail; + + peer.statusSequence = status.statusSequence; + return QueueFixedNL( + peer.peer, NetworkVideoDelivery::RELIABLE, + LG_NET_VIDEO_MESSAGE_STATUS, + LG_NET_ENVELOPE_FINAL | (requestID ? + LG_NET_ENVELOPE_RESPONSE : 0), + peer.componentEpoch, requestID, 0, status, + LG_NET_VIDEO_STATUS_WIRE_SIZE); + } + + void DropPeerFramesNL(PeerState& peer) + { + for (auto item = m_sends.begin(); item != m_sends.end();) + if (item->frameID && SamePeer(item->peer, peer.peer)) + { + ReleaseSendAccountingNL(*item); + item = m_sends.erase(item); + } + else + ++item; + } + + void DropPeerSendsNL(PeerState& peer) + { + for (auto item = m_sends.begin(); item != m_sends.end();) + if (SamePeer(item->peer, peer.peer)) + { + ReleaseSendAccountingNL(*item); + item = m_sends.erase(item); + } + else + ++item; + } + + bool QueueScheduleNL(PeerState& peer, const EncodedFrame& frame) + { + const uint32_t deadlineMs = std::min( + frame.deadlineMs, peer.subscription.maxFrameLatencyMs); + const uint64_t presentationNs = frame.captureTimeNs + + static_cast(deadlineMs) * UINT64_C(1000000); + + LGNetVideoSchedule schedule = {}; + schedule.streamID = STREAM_ID; + schedule.flags = LG_NET_VIDEO_SCHEDULE_PRESENT | + LG_NET_VIDEO_SCHEDULE_DROP_IF_LATE; + schedule.configEpoch = frame.configEpoch; + schedule.frameID = frame.frameID; + schedule.captureTimestampNs = frame.captureTimeNs; + schedule.presentationTimestampNs = presentationNs; + schedule.deadlineTimestampNs = presentationNs; + return QueueFixedNL(peer.peer, + NetworkVideoDelivery::RELIABLE, LG_NET_VIDEO_MESSAGE_SCHEDULE, + LG_NET_ENVELOPE_FINAL, frame.configEpoch, 0, frame.frameID, + schedule, LG_NET_VIDEO_SCHEDULE_WIRE_SIZE); + } + + bool QueueWholeFrameNL(PeerState& peer, const EncodedFrame& encoded, + NetworkVideoDelivery delivery) + { + const uint32_t deadlineMs = std::min( + encoded.deadlineMs, peer.subscription.maxFrameLatencyMs); + + LGNetVideoFrame frame = {}; + frame.streamID = STREAM_ID; + frame.codec = LG_NET_VIDEO_CODEC_PYROWAVE; + frame.flags = LG_NET_VIDEO_FRAME_KEYFRAME; + frame.configEpoch = encoded.configEpoch; + frame.frameID = encoded.frameID; + frame.captureTimestampNs = encoded.captureTimeNs; + frame.presentationTimestampNs = encoded.captureTimeNs + + static_cast(deadlineMs) * UINT64_C(1000000); + frame.encodedLength = static_cast(encoded.data.size()); + frame.blockCount = static_cast(encoded.blocks.size()); + frame.fragmentCount = 1; + frame.deadlineMs = deadlineMs; + frame.data = encoded.data.data(); + + const size_t size = lgNetVideoFrameSize(&frame); + if (!size || size > UINT32_MAX) + return false; + + std::shared_ptr> payload; + try + { + payload = std::make_shared>(size); + } + catch (...) + { + return false; + } + if (!lgNetVideoFrameEncode( + payload->data(), payload->size(), &frame)) + return false; + return QueueNL(peer.peer, delivery, LG_NET_VIDEO_MESSAGE_FRAME, + LG_NET_ENVELOPE_FINAL, encoded.configEpoch, 0, encoded.frameID, + std::move(payload)); + } + + bool QueueFragmentsNL(PeerState& peer, + std::shared_ptr encoded, + NetworkVideoDelivery delivery, uint32_t maxPayload) + { + if (!encoded || maxPayload <= LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE) + return false; + const uint32_t payloadLimit = std::min( + maxPayload - LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE, + static_cast(LG_NET_VIDEO_MAX_FRAGMENT_LENGTH)); + if (!payloadLimit) + return false; + + uint64_t fragmentCount64 = 0; + for (const EncodedBlock& block : encoded->blocks) + fragmentCount64 += + (static_cast(block.length) + payloadLimit - 1U) / + payloadLimit; + if (!fragmentCount64 || fragmentCount64 > LG_NET_VIDEO_MAX_FRAGMENTS || + fragmentCount64 > UINT32_MAX) + return false; + const uint32_t fragmentCount = static_cast(fragmentCount64); + + const bool newFrame = !EncodedQueuedNL(encoded.get()); + size_t bytes = 0; + if ((newFrame && !EncodedQueueBytes(*encoded, bytes)) || + m_sends.size() >= MAX_SEND_ITEMS || + bytes > MAX_SEND_BYTES - std::min(m_sendBytes, MAX_SEND_BYTES)) + return false; + try + { + SendItem item; + item.serial = Next64(m_nextSendSerial); + item.peer = peer.peer; + item.delivery = delivery; + item.messageType = LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT; + item.flags = LG_NET_ENVELOPE_FINAL; + item.componentEpoch = encoded->configEpoch; + item.frameID = encoded->frameID; + item.encoded = std::move(encoded); + item.fragmentCount = fragmentCount; + item.payloadLimit = payloadLimit; + const uint64_t serial = item.serial; + m_sends.push_back(std::move(item)); + m_nextSendSerial = serial; + m_sendBytes += bytes; + } + catch (...) + { + return false; + } + return true; + } + + bool BuildFragmentPayload( + const SendItem& item, std::vector& payload) const + { + if (!item.encoded || !item.payloadLimit || + item.blockIndex >= item.encoded->blocks.size() || + item.fragmentIndex >= item.fragmentCount) + return false; + + const EncodedBlock& block = item.encoded->blocks[item.blockIndex]; + if (item.blockOffset >= block.length) + return false; + const uint32_t length = std::min( + item.payloadLimit, block.length - item.blockOffset); + + LGNetVideoFragment fragment = {}; + fragment.streamID = STREAM_ID; + fragment.configEpoch = item.encoded->configEpoch; + fragment.frameID = item.encoded->frameID; + fragment.frameOffset = block.offset + item.blockOffset; + fragment.frameLength = + static_cast(item.encoded->data.size()); + fragment.blockIndex = item.blockIndex; + fragment.blockOffset = item.blockOffset; + fragment.blockLength = block.length; + fragment.blockCount = + static_cast(item.encoded->blocks.size()); + fragment.fragmentIndex = item.fragmentIndex; + fragment.fragmentCount = item.fragmentCount; + fragment.payloadLength = length; + fragment.payload = + item.encoded->data.data() + fragment.frameOffset; + if (!item.blockOffset) + fragment.flags |= LG_NET_VIDEO_FRAGMENT_BLOCK_START; + if (item.blockOffset + length == block.length) + fragment.flags |= LG_NET_VIDEO_FRAGMENT_BLOCK_END; + if (fragment.frameOffset + length == item.encoded->data.size()) + fragment.flags |= LG_NET_VIDEO_FRAGMENT_FRAME_END; + + const size_t size = lgNetVideoFragmentSize(&fragment); + if (!size) + return false; + try + { + payload.resize(size); + } + catch (...) + { + return false; + } + return lgNetVideoFragmentEncode( + payload.data(), payload.size(), &fragment); + } + + bool AdvanceFragmentNL(SendItem& item) const + { + if (!item.encoded || item.blockIndex >= item.encoded->blocks.size()) + return false; + const EncodedBlock& block = item.encoded->blocks[item.blockIndex]; + const uint32_t length = std::min( + item.payloadLimit, block.length - item.blockOffset); + item.blockOffset += length; + ++item.fragmentIndex; + if (item.blockOffset == block.length) + { + ++item.blockIndex; + item.blockOffset = 0; + } + return item.fragmentIndex == item.fragmentCount; + } + + bool QueueFrameNL(PeerState& peer, + const std::shared_ptr& frame) + { + if (!frame || !peer.configured || + peer.componentEpoch != frame->configEpoch || + frame->data.empty() || frame->data.size() > + peer.subscription.maxFrameLength) + return false; + + DropPeerFramesNL(peer); + if (!QueueScheduleNL(peer, *frame)) + return false; + + const bool datagrams = + (peer.subscription.flags & LG_NET_VIDEO_SUBSCRIBE_ALLOW_DATAGRAMS) && + DeliverySupported(peer.info, NetworkVideoDelivery::DATAGRAM); + const bool bulk = + DeliverySupported(peer.info, NetworkVideoDelivery::BULK); + const uint32_t fullSize = static_cast(frame->data.size()) + + LG_NET_VIDEO_FRAME_HEADER_WIRE_SIZE; + + NetworkVideoDelivery delivery = NetworkVideoDelivery::RELIABLE; + bool result; + if (bulk) + { + delivery = NetworkVideoDelivery::BULK; + result = fullSize <= peer.info.maxBulkPayload ? + QueueWholeFrameNL(peer, *frame, delivery) : + QueueFragmentsNL( + peer, frame, delivery, DeliveryPayload(peer.info, delivery)); + } + else if (!datagrams && fullSize <= peer.info.maxReliablePayload) + result = QueueWholeFrameNL( + peer, *frame, NetworkVideoDelivery::RELIABLE); + else + { + if (datagrams) + delivery = NetworkVideoDelivery::DATAGRAM; + const uint32_t maxPayload = DeliveryPayload(peer.info, delivery); + result = DeliverySupported(peer.info, delivery) && + QueueFragmentsNL(peer, frame, delivery, maxPayload); + } + + if (!result) + { + DropPeerFramesNL(peer); + ++peer.droppedFrames; + peer.congested = true; + return false; + } + peer.lastFrameID = frame->frameID; + peer.lastCaptureTimeNs = frame->captureTimeNs; + peer.keyframePending = false; + return true; + } + + static bool PeerFrameDueNL( + const PeerState& peer, const EncodedFrame& frame) + { + if (!peer.lastCaptureTimeNs || + frame.captureTimeNs <= peer.lastCaptureTimeNs) + return true; + const uint64_t period = UINT64_C(1000000000) / + peer.subscription.maxFrameRate; + return !period || + frame.captureTimeNs - peer.lastCaptureTimeNs >= period; + } + + void PublishFrame(EncodedFrame&& frame) + { + std::shared_ptr encoded; + try + { + encoded = std::make_shared(std::move(frame)); + } + catch (...) + { + return; + } + + CSRWExclusiveLock lock(m_lock); + if (!m_started || m_stopped || !m_active || + encoded->configEpoch != m_configEpoch) + return; + for (PeerState& peer : m_peers) + if (peer.subscribed && peer.configured && + PeerFrameDueNL(peer, *encoded)) + { + if (!QueueFrameNL(peer, encoded)) + QueueStatusNL(peer, LG_NET_VIDEO_STATE_STALLED, 2); + else + QueueStatusNL(peer, LG_NET_VIDEO_STATE_STREAMING, 0); + } + } + + bool SubmitPending() + { + PendingFrame pending; + { + CSRWExclusiveLock lock(m_lock); + if (m_encoder.Busy() || m_frames.empty() || !m_started || + m_stopped || !m_active || !HasSubscribersNL()) + return false; + pending = std::move(m_frames.front()); + m_frames.pop_front(); + if (pending.configEpoch != m_configEpoch || + !SameCfg(pending.cfg, m_activeCfg)) + return true; + } + + if (m_encoder.Submit(std::move(pending.frame), + std::move(pending.lease), pending.cfg, + pending.cfg.profile.signal == FrameSignal::PQ_BT2020 || + (pending.cfg.width & 1U) || (pending.cfg.height & 1U) ? + LG_NET_VIDEO_CHROMA_444 : LG_NET_VIDEO_CHROMA_420)) + return true; + + CSRWExclusiveLock lock(m_lock); + m_failed = true; + for (PeerState& peer : m_peers) + if (peer.subscribed) + QueueStatusNL(peer, LG_NET_VIDEO_STATE_ERROR, 3); + return false; + } + + void Flush(size_t limit = MAX_SEND_ITEMS) + { + CSRWSharedLock callbackLock(m_senderCallbackLock); + { + CSRWSharedLock lock(m_lock); + if (m_sends.empty()) + return; + } + size_t skipped = 0; + for (size_t count = 0; count < limit; ++count) + { + SendItem item; + INetworkVideoSender * sender = nullptr; + uint64_t readyGeneration = 0; + { + CSRWExclusiveLock lock(m_lock); + if (m_sends.empty() || !m_sender) + return; + PeerState * peer = FindPeerNL(m_sends.front().peer); + if (!peer) + { + ReleaseSendAccountingNL(m_sends.front()); + m_sends.pop_front(); + continue; + } + if (peer->blockedDeliveries & + DeliveryMask(m_sends.front().delivery)) + { + m_sends.push_back(std::move(m_sends.front())); + m_sends.pop_front(); + if (++skipped >= m_sends.size()) + return; + continue; + } + item = m_sends.front(); + sender = m_sender; + readyGeneration = peer->sendReadyGeneration[ + DeliveryIndex(item.delivery)]; + } + skipped = 0; + + std::vector fragmentPayload; + const bool prepared = item.payload || + BuildFragmentPayload(item, fragmentPayload); + const void * payload = item.payload ? + item.payload->data() : fragmentPayload.data(); + const size_t payloadSize = item.payload ? + item.payload->size() : fragmentPayload.size(); + const NetworkVideoSendResult result = prepared ? + sender->SendVideo(item.peer, item.delivery, item.messageType, + item.flags, item.componentEpoch, item.requestID, payload, + static_cast(payloadSize)) : + NetworkVideoSendResult::FAILED; + + CSRWExclusiveLock lock(m_lock); + if (m_sends.empty() || m_sends.front().serial != item.serial) + continue; + PeerState * peer = FindPeerNL(item.peer); + if (result == NetworkVideoSendResult::BLOCKED) + { + if (peer && peer->sendReadyGeneration[ + DeliveryIndex(item.delivery)] == readyGeneration) + { + peer->congested = true; + peer->blockedDeliveries |= DeliveryMask(item.delivery); + m_sends.push_back(std::move(m_sends.front())); + m_sends.pop_front(); + } + continue; + } + if (result == NetworkVideoSendResult::FAILED) + { + if (peer) + { + const bool notify = !peer->sendFailed; + + peer->configured = false; + peer->congested = true; + peer->sendFailed = true; + DropPeerSendsNL(*peer); + if (notify) + QueueStatusNL(*peer, LG_NET_VIDEO_STATE_ERROR, 7); + } + else + { + ReleaseSendAccountingNL(m_sends.front()); + m_sends.pop_front(); + } + return; + } + const bool complete = !m_sends.front().encoded || + AdvanceFragmentNL(m_sends.front()); + if (complete) + { + ReleaseSendAccountingNL(m_sends.front()); + m_sends.pop_front(); + } + if (peer && item.frameID) + peer->congested = false; + } + } +}; + +CNetworkVideo::CNetworkVideo() : + m_impl(new (std::nothrow) Impl()) +{ +} + +CNetworkVideo::~CNetworkVideo() +{ + if (!m_impl) + return; + Stop(); + DetachSender(); +} + +bool CNetworkVideo::AttachSender(INetworkVideoSender& sender) +{ + if (!m_impl) + return false; + CSRWExclusiveLock callbackLock(m_impl->m_senderCallbackLock); + CSRWExclusiveLock lock(m_impl->m_lock); + if (m_impl->m_sender) + return m_impl->m_sender == &sender; + m_impl->m_sender = &sender; + return true; +} + +void CNetworkVideo::DetachSender() +{ + if (!m_impl) + return; + CSRWExclusiveLock processLock(m_impl->m_processLock); + CSRWExclusiveLock callbackLock(m_impl->m_senderCallbackLock); + CSRWExclusiveLock lock(m_impl->m_lock); + m_impl->m_sender = nullptr; + m_impl->m_sendBytes = 0; + m_impl->m_sends.clear(); + m_impl->m_peers.clear(); +} + +bool CNetworkVideo::Start() +{ + if (!m_impl) + return false; + CSRWExclusiveLock lock(m_impl->m_lock); + if (m_impl->m_stopped || m_impl->m_failed) + return false; + m_impl->m_started = true; + return true; +} + +void CNetworkVideo::Stop() +{ + if (!m_impl) + return; + CSRWExclusiveLock processLock(m_impl->m_processLock); + { + CSRWExclusiveLock lock(m_impl->m_lock); + if (m_impl->m_stopped) + return; + m_impl->m_stopped = true; + m_impl->m_started = false; + m_impl->m_active = false; + m_impl->m_pending = false; + m_impl->m_changing = false; + m_impl->m_sendBytes = 0; + m_impl->m_frames.clear(); + m_impl->m_sends.clear(); + for (Impl::PeerState& peer : m_impl->m_peers) + { + peer.configured = false; + peer.componentEpoch = 0; + } + } + m_impl->m_encoder.Reset(); +} + +bool CNetworkVideo::PeerConnected(const NetworkVideoPeer& key) +{ + if (!m_impl || !key.client || !key.generation) + return false; + + NetworkVideoPeerInfo info; + { + CSRWSharedLock callbackLock(m_impl->m_senderCallbackLock); + INetworkVideoSender * sender; + { + CSRWSharedLock lock(m_impl->m_lock); + sender = m_impl->m_sender; + } + if (!sender || !sender->VideoPeerInfo(key, info)) + return false; + } + + if (info.serviceVersion < LG_NET_VIDEO_VERSION_MIN || + info.serviceVersion > LG_NET_VIDEO_VERSION_CURRENT || + !(info.capabilityFlags & LG_NET_CAPABILITY_RELIABLE) || + info.maxReliablePayload < LG_NET_VIDEO_STATUS_WIRE_SIZE) + return false; + + CSRWExclusiveLock lock(m_impl->m_lock); + Impl::PeerState * existing = m_impl->FindPeerNL(key); + if (existing) + { + existing->info = info; + return true; + } + if (m_impl->m_peers.size() >= NETWORK_MAX_CLIENTS || m_impl->m_stopped) + return false; + try + { + Impl::PeerState peer; + peer.peer = key; + peer.info = info; + peer.serial = Next64(m_impl->m_nextPeerSerial); + + m_impl->m_nextPeerSerial = peer.serial; + m_impl->m_peers.push_back(peer); + } + catch (...) + { + return false; + } + return true; +} + +void CNetworkVideo::PeerDisconnected(const NetworkVideoPeer& key) +{ + if (!m_impl) + return; + CSRWExclusiveLock lock(m_impl->m_lock); + for (auto item = m_impl->m_sends.begin(); item != m_impl->m_sends.end();) + if (SamePeer(item->peer, key)) + { + m_impl->ReleaseSendAccountingNL(*item); + item = m_impl->m_sends.erase(item); + } + else + ++item; + const auto peer = std::find_if(m_impl->m_peers.begin(), + m_impl->m_peers.end(), [&key](const Impl::PeerState& current) + { + return SamePeer(current.peer, key); + }); + if (peer != m_impl->m_peers.end()) + m_impl->m_peers.erase(peer); + if (!m_impl->HasSubscribersNL()) + m_impl->m_frames.clear(); +} + +NetworkVideoPacketResult CNetworkVideo::HandlePacket( + const NetworkVideoPeer& key, const LGNetPacketView& packet, + NetworkVideoDelivery delivery) +{ + if (!m_impl || packet.envelope.service != LG_NET_SERVICE_VIDEO || + packet.envelope.serviceVersion < LG_NET_VIDEO_VERSION_MIN || + packet.envelope.serviceVersion > LG_NET_VIDEO_VERSION_CURRENT || + packet.payloadLength != packet.envelope.payloadLength) + return NetworkVideoPacketResult::INVALID; + + CSRWExclusiveLock lock(m_impl->m_lock); + Impl::PeerState * peer = m_impl->FindPeerNL(key); + if (!peer || peer->info.serviceVersion != packet.envelope.serviceVersion) + return NetworkVideoPacketResult::IGNORED; + if (!DeliverySupported(peer->info, delivery)) + return NetworkVideoPacketResult::INVALID; + + switch (packet.envelope.messageType) + { + case LG_NET_VIDEO_MESSAGE_SUBSCRIBE: + { + if (delivery != NetworkVideoDelivery::RELIABLE || + packet.envelope.componentEpoch || !packet.envelope.requestID || + !DeliveryFlags(packet.envelope, delivery, + LG_NET_ENVELOPE_ACK_REQUIRED | LG_NET_ENVELOPE_FINAL)) + return NetworkVideoPacketResult::INVALID; + LGNetVideoSubscribe subscribe = {}; + if (lgNetVideoSubscribeDecode(&subscribe, packet.payload, + packet.payloadLength) != LG_NET_PARSE_OK || + subscribe.maxWidth > MAX_WIDTH || + subscribe.maxHeight > MAX_HEIGHT || + subscribe.maxFrameLength > MAX_FRAME_LENGTH || + (subscribe.preferredCodec && + subscribe.preferredCodec != LG_NET_VIDEO_CODEC_PYROWAVE)) + return NetworkVideoPacketResult::INVALID; + + m_impl->DropPeerSendsNL(*peer); + peer->subscription = subscribe; + peer->subscribed = true; + peer->configured = false; + peer->componentEpoch = m_impl->m_active ? + m_impl->m_configEpoch : 0; + peer->congested = false; + peer->keyframePending = true; + peer->sendFailed = false; + if (m_impl->m_active) + { + if (!m_impl->QueueConfigNL( + *peer, packet.envelope.requestID)) + return NetworkVideoPacketResult::INVALID; + } + else if (!m_impl->QueueStatusNL( + *peer, LG_NET_VIDEO_STATE_SUBSCRIBED, 0, + packet.envelope.requestID)) + return NetworkVideoPacketResult::INVALID; + break; + } + + case LG_NET_VIDEO_MESSAGE_UNSUBSCRIBE: + { + if (delivery != NetworkVideoDelivery::RELIABLE || + !DeliveryFlags(packet.envelope, delivery, + LG_NET_ENVELOPE_FINAL)) + return NetworkVideoPacketResult::INVALID; + LGNetVideoControl control = {}; + if (lgNetVideoControlDecode(&control, packet.payload, + packet.payloadLength) != LG_NET_PARSE_OK || + control.streamID != STREAM_ID || + (control.configEpoch && + control.configEpoch != peer->componentEpoch)) + return NetworkVideoPacketResult::INVALID; + m_impl->DropPeerSendsNL(*peer); + peer->subscribed = false; + peer->configured = false; + peer->componentEpoch = 0; + peer->keyframePending = false; + break; + } + + case LG_NET_VIDEO_MESSAGE_KEYFRAME_REQUEST: + { + if (delivery == NetworkVideoDelivery::BULK || + !DeliveryFlags(packet.envelope, delivery, + LG_NET_ENVELOPE_FINAL)) + return NetworkVideoPacketResult::INVALID; + LGNetVideoControl control = {}; + if (lgNetVideoControlDecode(&control, packet.payload, + packet.payloadLength) != LG_NET_PARSE_OK || + control.streamID != STREAM_ID || !peer->subscribed || + (control.configEpoch && + control.configEpoch != peer->componentEpoch)) + return NetworkVideoPacketResult::INVALID; + peer->keyframePending = true; + break; + } + + case LG_NET_VIDEO_MESSAGE_FEEDBACK: + { + if (delivery == NetworkVideoDelivery::BULK || + !DeliveryFlags(packet.envelope, delivery, + LG_NET_ENVELOPE_FINAL)) + return NetworkVideoPacketResult::INVALID; + LGNetVideoFeedback feedback = {}; + if (lgNetVideoFeedbackDecode(&feedback, packet.payload, + packet.payloadLength) != LG_NET_PARSE_OK || + feedback.streamID != STREAM_ID || !peer->subscribed || + packet.envelope.componentEpoch != peer->componentEpoch || + feedback.highestSeenFrameID > peer->lastFrameID) + return NetworkVideoPacketResult::INVALID; + peer->feedback = feedback; + peer->lastCompleteID = feedback.lastCompleteFrameID; + peer->congested = + feedback.flags & LG_NET_VIDEO_FEEDBACK_CONGESTED; + if (feedback.flags & (LG_NET_VIDEO_FEEDBACK_REQUEST_KEYFRAME | + LG_NET_VIDEO_FEEDBACK_DECODER_STALLED)) + peer->keyframePending = true; + break; + } + + default: + return NetworkVideoPacketResult::INVALID; + } + + lock.Unlock(); + CSRWExclusiveLock processLock = + CSRWExclusiveLock::Try(m_impl->m_processLock); + if (processLock) + m_impl->Flush(32); + return NetworkVideoPacketResult::HANDLED; +} + +void CNetworkVideo::Process(uint64_t nowMs) +{ + (void)nowMs; + if (!m_impl) + return; + CSRWExclusiveLock processLock(m_impl->m_processLock); + + if (m_impl->m_encoder.Busy()) + { + const D12SyncState state = m_impl->m_encoder.State(); + if (state == D12SyncState::READY) + { + uint64_t configEpoch; + uint64_t frameID; + { + CSRWExclusiveLock lock(m_impl->m_lock); + configEpoch = m_impl->m_configEpoch; + frameID = Next64(m_impl->m_nextFrameID); + + m_impl->m_nextFrameID = frameID; + } + EncodedFrame frame; + if (m_impl->m_encoder.Complete(configEpoch, frameID, frame)) + m_impl->PublishFrame(std::move(frame)); + else + { + CSRWExclusiveLock lock(m_impl->m_lock); + m_impl->m_failed = true; + for (Impl::PeerState& peer : m_impl->m_peers) + if (peer.subscribed) + m_impl->QueueStatusNL( + peer, LG_NET_VIDEO_STATE_ERROR, 4); + } + } + else if (state == D12SyncState::FAILED) + { + m_impl->m_encoder.Reset(); + CSRWExclusiveLock lock(m_impl->m_lock); + m_impl->m_failed = true; + for (Impl::PeerState& peer : m_impl->m_peers) + if (peer.subscribed) + m_impl->QueueStatusNL(peer, LG_NET_VIDEO_STATE_ERROR, 5); + } + } + + if (!m_impl->m_encoder.Busy()) + m_impl->SubmitPending(); + m_impl->Flush(); +} + +void CNetworkVideo::SendReady( + const NetworkVideoPeer& key, NetworkVideoDelivery delivery) +{ + if (!m_impl) + return; + { + CSRWExclusiveLock lock(m_impl->m_lock); + Impl::PeerState * peer = m_impl->FindPeerNL(key); + if (!peer || !DeliverySupported(peer->info, delivery)) + return; + const unsigned index = DeliveryIndex(delivery); + peer->congested = false; + peer->blockedDeliveries &= + static_cast(~DeliveryMask(delivery)); + peer->sendReadyGeneration[index] = + Next64(peer->sendReadyGeneration[index]); + } + CSRWExclusiveLock processLock = + CSRWExclusiveLock::Try(m_impl->m_processLock); + if (processLock) + m_impl->Flush(); +} + +const FrameProfile * CNetworkVideo::Profiles(unsigned& count) const +{ + count = ARRAYSIZE(VIDEO_PROFILES); + return VIDEO_PROFILES; +} + +std::shared_ptr CNetworkVideo::GetFrameCaps() const +{ + static const std::shared_ptr caps( + new (std::nothrow) NetworkVideoFrameCaps()); + return caps; +} + +CfgResult CNetworkVideo::Probe(const FrameCfg& cfg) const +{ + if (!m_impl) + return CfgResult::FAILED; + CSRWSharedLock lock(m_impl->m_lock); + if (m_impl->m_stopped || m_impl->m_failed) + return CfgResult::FAILED; + if (cfg.mode != GpuMode::HARDWARE || !cfg.width || !cfg.height || + cfg.width > MAX_WIDTH || cfg.height > MAX_HEIGHT || + !SupportedProfile(cfg.profile)) + return CfgResult::NEXT; + const uint32_t alignedWidth = std::max( + Align(cfg.width, PYROWAVE_ALIGNMENT), PYROWAVE_MIN_DIMENSION); + const uint32_t alignedHeight = std::max( + Align(cfg.height, PYROWAVE_ALIGNMENT), PYROWAVE_MIN_DIMENSION); + if (alignedWidth > MAX_WIDTH || alignedHeight > MAX_HEIGHT) + return CfgResult::NEXT; + return CfgResult::ACCEPTED; +} + +CfgResult CNetworkVideo::Prepare(const FrameCfg& cfg) +{ + const CfgResult result = Probe(cfg); + if (result != CfgResult::ACCEPTED) + return result; + CSRWExclusiveLock processLock(m_impl->m_processLock); + CSRWExclusiveLock lock(m_impl->m_lock); + if (m_impl->m_changing || m_impl->m_stopped) + return CfgResult::RETRY; + m_impl->m_pendingCfg = cfg; + m_impl->m_pending = true; + m_impl->m_changing = true; + return CfgResult::ACCEPTED; +} + +void CNetworkVideo::Commit() +{ + if (!m_impl) + return; + CSRWExclusiveLock processLock(m_impl->m_processLock); + { + CSRWExclusiveLock lock(m_impl->m_lock); + if (!m_impl->m_pending || !m_impl->m_changing || m_impl->m_stopped) + return; + m_impl->m_activeCfg = m_impl->m_pendingCfg; + m_impl->m_pendingCfg = {}; + m_impl->m_active = true; + m_impl->m_pending = false; + m_impl->m_changing = false; + m_impl->m_configEpoch = Next64(m_impl->m_configEpoch); + m_impl->m_frames.clear(); + for (Impl::PeerState& peer : m_impl->m_peers) + if (peer.subscribed) + { + peer.configured = false; + peer.componentEpoch = m_impl->m_configEpoch; + peer.keyframePending = true; + m_impl->DropPeerSendsNL(peer); + m_impl->QueueConfigNL(peer); + } + } + m_impl->m_encoder.Reset(); +} + +void CNetworkVideo::Abort() +{ + if (!m_impl) + return; + CSRWExclusiveLock lock(m_impl->m_lock); + if (!m_impl->m_pending || !m_impl->m_changing) + return; + m_impl->m_pendingCfg = {}; + m_impl->m_pending = false; + m_impl->m_changing = false; +} + +PushResult CNetworkVideo::Push(FrameIn frame, TexLease lease) noexcept +{ + if (!m_impl || !lease || !lease.Get()) + return PushResult::REJECTED; + CSRWExclusiveLock lock = CSRWExclusiveLock::Try(m_impl->m_lock); + if (!lock) + return PushResult::BUSY; + if (!m_impl->m_started || m_impl->m_stopped || m_impl->m_failed || + !m_impl->m_active || m_impl->m_changing || + !m_impl->HasSubscribersNL()) + return PushResult::BUSY; + + const CFrameTex * texture = lease.Get(); + const D3D12_RESOURCE_DESC desc = texture->Get()->GetDesc(); + if (frame.graph != texture->graph || frame.desc.node != texture->node || + !Frame::Same(frame.desc.profile, m_impl->m_activeCfg.profile) || + !Frame::Same(texture->profile, m_impl->m_activeCfg.profile) || + desc.Dimension != D3D12_RESOURCE_DIMENSION_TEXTURE2D || + desc.Width != m_impl->m_activeCfg.width || + desc.Height != m_impl->m_activeCfg.height || + desc.Format != ProfileFormat(m_impl->m_activeCfg.profile.pixel) || + texture->State() != D3D12_RESOURCE_STATE_COMMON) + return PushResult::STALE; + + try + { + if (m_impl->m_frames.size() >= MAX_PENDING_FRAMES) + m_impl->m_frames.pop_front(); + Impl::PendingFrame pending; + pending.frame = std::move(frame); + pending.lease = std::move(lease); + pending.cfg = m_impl->m_activeCfg; + pending.configEpoch = m_impl->m_configEpoch; + m_impl->m_frames.push_back(std::move(pending)); + } + catch (...) + { + return PushResult::FAILED; + } + return PushResult::ACCEPTED; +} + +PushResult CNetworkVideo::Push(FrameIn frame, D11Lease lease) noexcept +{ + (void)frame; + (void)lease; + return PushResult::REJECTED; +} diff --git a/idd/LGIdd/transport/network/CNetworkVideo.h b/idd/LGIdd/transport/network/CNetworkVideo.h new file mode 100644 index 00000000..4f304b4e --- /dev/null +++ b/idd/LGIdd/transport/network/CNetworkVideo.h @@ -0,0 +1,118 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#pragma once + +#include "transport/FrameCaps.h" +#include "transport/ITexSink.h" + +#include +#include + +#include +#include + +struct NetworkVideoPeer +{ + uint32_t client = 0; + uint32_t generation = 0; +}; + +struct NetworkVideoPeerInfo +{ + uint16_t serviceVersion = 0; + LGNetCapabilityFlags capabilityFlags = 0; + uint32_t maxReliablePayload = 0; + uint32_t maxDatagramPayload = 0; + uint32_t maxBulkPayload = 0; +}; + +enum class NetworkVideoDelivery +{ + RELIABLE, + DATAGRAM, + BULK, +}; + +enum class NetworkVideoSendResult +{ + ACCEPTED, + BLOCKED, + FAILED, +}; + +enum class NetworkVideoPacketResult +{ + HANDLED, + IGNORED, + INVALID, +}; + +class INetworkVideoSender +{ +public: + virtual ~INetworkVideoSender() = default; + + virtual bool VideoPeerInfo( + const NetworkVideoPeer& peer, NetworkVideoPeerInfo& info) = 0; + virtual NetworkVideoSendResult SendVideo( + const NetworkVideoPeer& peer, NetworkVideoDelivery delivery, + uint16_t messageType, LGNetEnvelopeFlags flags, + uint64_t componentEpoch, uint64_t requestID, + const void * payload, uint32_t payloadLength) = 0; +}; + +class CNetworkVideo final : public ITexSink +{ +private: + struct Impl; + std::unique_ptr m_impl; + +public: + CNetworkVideo(); + ~CNetworkVideo() override; + + CNetworkVideo(const CNetworkVideo&) = delete; + CNetworkVideo& operator=(const CNetworkVideo&) = delete; + + bool AttachSender(INetworkVideoSender& sender); + void DetachSender(); + + bool Start(); + void Stop(); + void Process(uint64_t nowMs); + + bool PeerConnected(const NetworkVideoPeer& peer); + void PeerDisconnected(const NetworkVideoPeer& peer); + NetworkVideoPacketResult HandlePacket(const NetworkVideoPeer& peer, + const LGNetPacketView& packet, NetworkVideoDelivery delivery); + void SendReady( + const NetworkVideoPeer& peer, NetworkVideoDelivery delivery); + + const FrameProfile * Profiles(unsigned& count) const; + std::shared_ptr GetFrameCaps() const; + CfgResult Probe(const FrameCfg& cfg) const; + CfgResult Prepare(const FrameCfg& cfg); + void Commit(); + void Abort(); + + PushResult Push(FrameIn frame, TexLease lease) noexcept override; + PushResult Push(FrameIn frame, D11Lease lease) noexcept override; +}; diff --git a/repos/LGProtocol b/repos/LGProtocol index a2e35d67..82f05ec6 160000 --- a/repos/LGProtocol +++ b/repos/LGProtocol @@ -1 +1 @@ -Subproject commit a2e35d67ce0baea91202ed483b58ed0ca1ad50c5 +Subproject commit 82f05ec6645f000b460ed6182da106b5c571b5a8