From ea24c288adfb4ca51981d7966d0ee1187d303409 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Thu, 8 Oct 2026 23:56:14 +1100 Subject: [PATCH] [client] network: bound video decoder backlog Cap decoded snapshots and queued GPU work so a slow presentation path cannot accumulate frames indefinitely. Coalesce deferred damage and retry the latest image when a texture returns, including while JIT rendering is idle. --- client/transports/Network/video.c | 217 ++++++- client/transports/Network/video/decoder.h | 8 +- client/transports/Network/video/decoder_egl.c | 546 ++++++++++++++---- 3 files changed, 632 insertions(+), 139 deletions(-) diff --git a/client/transports/Network/video.c b/client/transports/Network/video.c index 8d44506f..f71c86d1 100644 --- a/client/transports/Network/video.c +++ b/client/transports/Network/video.c @@ -143,6 +143,21 @@ struct NetworkVideoEncoded LGNetVideoSchedule schedule; }; +struct NetworkVideoDeferred +{ + uint64_t decodeGeneration; + uint64_t configEpoch; + uint64_t frameID; + uint64_t epoch; + uint64_t timestamp; + uint64_t captureTimestampNs; + uint64_t presentationTimestampNs; + uint64_t receiveTimeNs; + uint32_t formatVersion; + bool complete; + bool valid; +}; + struct NetworkVideo { NetworkConnection * connection; @@ -165,6 +180,7 @@ struct NetworkVideo bool waitCancelled; bool decodeStopping; bool decodePending; + bool framePollPending; uint16_t serviceVersion; uint64_t subscriberID; @@ -192,6 +208,7 @@ struct NetworkVideo struct NetworkVideoAssembly assembly; struct NetworkVideoEncoded pendingDecode; + struct NetworkVideoDeferred deferred; struct NetworkVideoScheduleEntry schedules[NETWORK_VIDEO_SCHEDULE_CACHE]; struct NetworkVideoOutput * output; struct NetworkVideoOutput * leases; @@ -354,6 +371,39 @@ static void networkVideoClearEncoded(struct NetworkVideoEncoded * encoded) memset(encoded, 0, sizeof(*encoded)); } +static void networkVideoDeferNL(NetworkVideo * video, + const struct NetworkVideoEncoded * encoded) +{ + video->deferred = (struct NetworkVideoDeferred) + { + .decodeGeneration = encoded->generation, + .configEpoch = encoded->config.configEpoch, + .frameID = encoded->frameID, + .epoch = video->status.epoch, + .timestamp = encoded->presentationTimestampNs, + .captureTimestampNs = encoded->captureTimestampNs, + .presentationTimestampNs = encoded->presentationTimestampNs, + .receiveTimeNs = encoded->receiveTimeNs, + .formatVersion = video->localConfigGeneration, + .complete = encoded->complete, + .valid = true, + }; + video->framePollPending = true; +} + +static bool networkVideoDeferredCurrentNL( + const NetworkVideo * video, const struct NetworkVideoDeferred * deferred) +{ + return deferred->valid && video->deferred.valid && + deferred->decodeGeneration == video->deferred.decodeGeneration && + deferred->configEpoch == video->deferred.configEpoch && + deferred->frameID == video->deferred.frameID && + deferred->epoch == video->deferred.epoch && + video->decodeGeneration == deferred->decodeGeneration && + video->config.configEpoch == deferred->configEpoch && + video->status.epoch == deferred->epoch; +} + static void networkVideoInvalidateDecodeNL(NetworkVideo * video, struct NetworkVideoEncoded * pending) { @@ -361,6 +411,8 @@ static void networkVideoInvalidateDecodeNL(NetworkVideo * video, *pending = video->pendingDecode; video->pendingDecode = (struct NetworkVideoEncoded){0}; video->decodePending = false; + video->framePollPending = false; + memset(&video->deferred, 0, sizeof(video->deferred)); } static struct NetworkVideoOutput * networkVideoFindLeaseNL( @@ -1102,7 +1154,10 @@ static void networkVideoDecodeFailure(NetworkVideo * video, video->wireConfigured && video->config.configEpoch == encoded->config.configEpoch) { - video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + video->lostFrames = networkVideoSaturatingIncrement( + video->lostFrames); + video->framePollPending = false; + memset(&video->deferred, 0, sizeof(video->deferred)); const LGNetVideoFeedbackFlags flags = result == NETWORK_VIDEO_DECODE_ERROR ? @@ -1141,24 +1196,18 @@ static void networkVideoDecode(NetworkVideo * video, return; } - struct NetworkVideoOutput * output = calloc(1, sizeof(*output)); - if (!output) - { - free(encoded->ownedData); - networkVideoDecodeFailure( - video, encoded, NETWORK_VIDEO_DECODE_ERROR); - return; - } - NetworkVideoDecodedFrame decoded = {0}; const NetworkVideoDecodeResult result = networkVideoDecoderDecode( video->decoder, &encoded->config, encoded->data, encoded->length, encoded->blockCount, encoded->update, encoded->complete, encoded->baseFrameID, encoded->frameID, &decoded); free(encoded->ownedData); - if (result == NETWORK_VIDEO_DECODE_NO_FRAME) + encoded->ownedData = NULL; + encoded->data = NULL; + if (result == NETWORK_VIDEO_DECODE_NO_FRAME || + result == NETWORK_VIDEO_DECODE_DEFERRED) { - free(output); + bool pollFrame = false; LG_LOCK(video->stateLock); if (video->decodeGeneration == encoded->generation && video->wireConfigured && @@ -1169,18 +1218,33 @@ static void networkVideoDecode(NetworkVideo * video, if (encoded->complete) video->lastCompleteFrameID = encoded->frameID; video->lastDecodedFrameID = encoded->frameID; + if (result == NETWORK_VIDEO_DECODE_DEFERRED) + { + networkVideoDeferNL(video, encoded); + pollFrame = true; + } networkVideoQueueFeedbackNL(video, 0); } LG_UNLOCK(video->stateLock); + if (pollFrame) + lgSignalEvent(video->frameEvent); return; } if (result != NETWORK_VIDEO_DECODE_OK) { - free(output); networkVideoDecodeFailure(video, encoded, result); return; } + struct NetworkVideoOutput * output = calloc(1, sizeof(*output)); + if (!output) + { + networkVideoDecoderRelease(&decoded); + networkVideoDecodeFailure( + video, encoded, NETWORK_VIDEO_DECODE_ERROR); + return; + } + struct NetworkVideoOutput * replaced = NULL; output->decoded = decoded; output->frameID = encoded->frameID; @@ -1209,6 +1273,8 @@ static void networkVideoDecode(NetworkVideo * video, if (encoded->complete) video->lastCompleteFrameID = output->frameID; video->lastDecodedFrameID = output->frameID; + video->framePollPending = false; + memset(&video->deferred, 0, sizeof(video->deferred)); output->decoded.format.version = video->localConfigGeneration; output->epoch = video->status.epoch; video->output = output; @@ -1218,6 +1284,101 @@ static void networkVideoDecode(NetworkVideo * video, lgSignalEvent(video->frameEvent); } +static bool networkVideoPresentDeferred(NetworkVideo * video, + const struct NetworkVideoDeferred * deferred) +{ + LG_LOCK(video->stateLock); + const bool current = !video->destroying && !video->decodeStopping && + video->connected && video->wireConfigured && video->rendererAttached && + video->decoderReady && networkVideoDeferredCurrentNL(video, deferred); + LG_UNLOCK(video->stateLock); + if (!current) + return false; + + NetworkVideoDecodedFrame decoded = {0}; + const NetworkVideoDecodeResult result = + networkVideoDecoderPresentPending(video->decoder, &decoded); + if (result == NETWORK_VIDEO_DECODE_DEFERRED) + return false; + if (result == NETWORK_VIDEO_DECODE_NO_FRAME) + { + LG_LOCK(video->stateLock); + if (networkVideoDeferredCurrentNL(video, deferred)) + { + video->framePollPending = false; + memset(&video->deferred, 0, sizeof(video->deferred)); + } + LG_UNLOCK(video->stateLock); + return true; + } + + if (result != NETWORK_VIDEO_DECODE_OK) + { + struct NetworkVideoEncoded encoded = {0}; + encoded.config.configEpoch = deferred->configEpoch; + encoded.generation = deferred->decodeGeneration; + encoded.frameID = deferred->frameID; + encoded.complete = deferred->complete; + LG_LOCK(video->stateLock); + if (networkVideoDeferredCurrentNL(video, deferred)) + { + video->framePollPending = false; + memset(&video->deferred, 0, sizeof(video->deferred)); + } + LG_UNLOCK(video->stateLock); + networkVideoDecodeFailure(video, &encoded, result); + return true; + } + + struct NetworkVideoOutput * output = calloc(1, sizeof(*output)); + if (!output) + { + struct NetworkVideoEncoded encoded = {0}; + encoded.config.configEpoch = deferred->configEpoch; + encoded.generation = deferred->decodeGeneration; + encoded.frameID = deferred->frameID; + encoded.complete = deferred->complete; + networkVideoDecoderRelease(&decoded); + networkVideoDecodeFailure( + video, &encoded, NETWORK_VIDEO_DECODE_ERROR); + return true; + } + + struct NetworkVideoOutput * replaced = NULL; + output->decoded = decoded; + output->frameID = deferred->frameID; + output->timestamp = deferred->timestamp; + output->captureTimestampNs = deferred->captureTimestampNs; + output->presentationTimestampNs = deferred->presentationTimestampNs; + output->receiveTimeNs = deferred->receiveTimeNs; + LG_LOCK(video->stateLock); + if (!networkVideoDeferredCurrentNL(video, deferred) || + output->frameID != video->lastDecodedFrameID || + !video->rendererAttached || !video->decoderReady) + { + LG_UNLOCK(video->stateLock); + networkVideoFreeOutput(&output); + return true; + } + + if (video->output) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + replaced = video->output; + networkVideoMergeDamage(&output->decoded, &replaced->decoded); + } + output->decoded.format.version = deferred->formatVersion; + output->epoch = deferred->epoch; + video->output = output; + video->framePollPending = false; + memset(&video->deferred, 0, sizeof(video->deferred)); + networkVideoQueueFeedbackNL(video, 0); + LG_UNLOCK(video->stateLock); + networkVideoFreeOutput(&replaced); + lgSignalEvent(video->frameEvent); + return true; +} + static void networkVideoQueueDecode(NetworkVideo * video, struct NetworkVideoEncoded * encoded) { @@ -1283,7 +1444,8 @@ static int networkVideoDecodeThread(void * opaque) lgWaitEvent(video->decodeEvent, TIMEOUT_INFINITE); for (;;) { - struct NetworkVideoEncoded encoded = {0}; + struct NetworkVideoEncoded encoded = {0}; + struct NetworkVideoDeferred deferred = {0}; LG_LOCK(video->stateLock); if (video->decodeStopping) { @@ -1300,11 +1462,18 @@ static int networkVideoDecodeThread(void * opaque) video->pendingDecode = (struct NetworkVideoEncoded){0}; video->decodePending = false; } + else if (video->deferred.valid) + deferred = video->deferred; LG_UNLOCK(video->stateLock); - if (!encoded.ownedData) + if (encoded.ownedData) + { + networkVideoDecode(video, &encoded); + continue; + } + if (!deferred.valid || + !networkVideoPresentDeferred(video, &deferred)) break; - networkVideoDecode(video, &encoded); } } } @@ -1960,6 +2129,7 @@ void networkVideoProcess(NetworkVideo * video) const uint64_t now = nanotime(); struct NetworkVideoAssembly retiredAssembly = {0}; struct NetworkVideoEncoded recovered = {0}; + bool pollFrame = false; LG_LOCK(video->stateLock); if (video->destroying) { @@ -1978,10 +2148,17 @@ void networkVideoProcess(NetworkVideo * video) networkVideoSendSubscribeNL(video, now); networkVideoSendKeyframeNL(video, now); networkVideoSendFeedbackNL(video, now); + if (video->deferred.valid && !video->framePollPending) + { + video->framePollPending = true; + pollFrame = true; + } LG_UNLOCK(video->stateLock); networkVideoClearAssembly(&retiredAssembly); if (recovered.ownedData) networkVideoQueueDecode(video, &recovered); + if (pollFrame) + lgSignalEvent(video->frameEvent); } static bool networkVideoTimer(void * opaque) @@ -2032,7 +2209,7 @@ bool networkVideoCreate(NetworkVideo ** result, video->frameEvent = lgCreateEvent(true, 0); video->decodeEvent = lgCreateEvent(true, 0); if (!video->frameEvent || !video->decodeEvent || - !networkVideoDecoderCreate(&video->decoder) || + !networkVideoDecoderCreate(&video->decoder, video->decodeEvent) || !lgCreateThread("networkDecode", networkVideoDecodeThread, video, &video->decodeThread)) { @@ -2324,6 +2501,12 @@ LG_TransportStatus networkVideoNextFrame(NetworkVideo * video, bool useDMA, frame->releaseHandle = output->handle; ready = true; } + else if (video->framePollPending) + { + video->framePollPending = false; + status = LG_TRANSPORT_TIMEOUT; + ready = true; + } } LG_UNLOCK(video->stateLock); networkVideoFreeOutputList(released); diff --git a/client/transports/Network/video/decoder.h b/client/transports/Network/video/decoder.h index e81b4715..8051152d 100644 --- a/client/transports/Network/video/decoder.h +++ b/client/transports/Network/video/decoder.h @@ -24,6 +24,8 @@ #include "interface/renderer.h" #include "interface/transport.h" +#include "common/event.h" + #include #include @@ -36,6 +38,7 @@ typedef enum NetworkVideoDecodeResult { NETWORK_VIDEO_DECODE_OK, NETWORK_VIDEO_DECODE_NO_FRAME, + NETWORK_VIDEO_DECODE_DEFERRED, NETWORK_VIDEO_DECODE_UNSUPPORTED, NETWORK_VIDEO_DECODE_CORRUPT, NETWORK_VIDEO_DECODE_ERROR, @@ -53,7 +56,8 @@ typedef struct NetworkVideoDecodedFrame } NetworkVideoDecodedFrame; -bool networkVideoDecoderCreate(NetworkVideoDecoder ** decoder); +bool networkVideoDecoderCreate(NetworkVideoDecoder ** decoder, + LGEvent * presentEvent); void networkVideoDecoderDestroy(NetworkVideoDecoder ** decoder); bool networkVideoDecoderAttach(NetworkVideoDecoder * decoder, @@ -68,5 +72,7 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( const void * data, size_t size, uint32_t blockCount, bool update, bool complete, uint64_t baseFrameID, uint64_t frameID, NetworkVideoDecodedFrame * frame); +NetworkVideoDecodeResult networkVideoDecoderPresentPending( + NetworkVideoDecoder * decoder, NetworkVideoDecodedFrame * frame); void networkVideoDecoderRelease(NetworkVideoDecodedFrame * frame); diff --git a/client/transports/Network/video/decoder_egl.c b/client/transports/Network/video/decoder_egl.c index e438a18a..11d389c4 100644 --- a/client/transports/Network/video/decoder_egl.c +++ b/client/transports/Network/video/decoder_egl.c @@ -50,6 +50,7 @@ #include "shaders.h" #include "common/debug.h" +#include "common/event.h" #include "common/locking.h" #include "common/time.h" @@ -67,6 +68,7 @@ #define NETWORK_VIDEO_ALIGNMENT (1U << NETWORK_VIDEO_LEVELS) #define NETWORK_VIDEO_MIN_SIZE (4U << NETWORK_VIDEO_LEVELS) #define NETWORK_VIDEO_UPLOADS 3U +#define NETWORK_VIDEO_POOL_MIN_SIZE 2U #define NETWORK_VIDEO_POOL_SIZE 3U #define NETWORK_VIDEO_DAMAGE_HISTORY 64U #define NETWORK_VIDEO_POOL_BYTE_LIMIT \ @@ -131,11 +133,13 @@ struct NetworkVideoTexturePool LG_Lock lock; atomic_uint references; + LGEvent * presentEvent; EGLDisplay display; EGLConfig eglConfig; EGLContext shareContext; struct NetworkVideoTexture textures[NETWORK_VIDEO_POOL_SIZE]; unsigned textureCount; + unsigned textureAllocated; unsigned textureLimit; uint32_t width; uint32_t height; @@ -219,6 +223,8 @@ struct NetworkVideoDecoder uint8_t * outputDirty; struct NetworkVideoDamageHistory damageHistory[NETWORK_VIDEO_DAMAGE_HISTORY]; + LGNetVideoDamage pendingDamage; + LGEvent * presentEvent; LGNetVideoStreamConfig config; struct NetworkVideoBlockMeta @@ -232,6 +238,7 @@ struct NetworkVideoDecoder uint32_t outputTileHeight; uint64_t referenceFrameID; uint64_t outputGeneration; + uint64_t pendingDecodeTimeNs; unsigned uploadIndex; GLint maxTextureSize; GLint maxArrayLayers; @@ -240,6 +247,7 @@ struct NetworkVideoDecoder bool attached; bool configured; bool referenceValid; + bool pendingChromaLeft; }; struct NetworkVideoExternalFrame @@ -257,6 +265,12 @@ static uint32_t networkVideoAlign(uint32_t value, uint32_t alignment) return (value + alignment - 1U) & ~(alignment - 1U); } +static uint64_t networkVideoSaturatingAdd( + uint64_t left, uint64_t right) +{ + return left > UINT64_MAX - right ? UINT64_MAX : left + right; +} + static uint32_t networkVideoReadU32(const uint8_t * data) { return (uint32_t)data[0] | @@ -384,21 +398,24 @@ static struct NetworkVideoTexturePool * networkVideoTexturePoolCreate( if (!pool) return NULL; - const uint64_t width = decoder->config.width; - const uint64_t height = decoder->config.height; - const uint64_t textureBytes = width * height * NETWORK_VIDEO_PIXEL_BYTES; - const uint64_t textureLimit = textureBytes ? + const uint64_t width = decoder->config.width; + const uint64_t height = decoder->config.height; + const uint64_t textureBytes = width * height * NETWORK_VIDEO_PIXEL_BYTES; + const uint64_t textureLimit = textureBytes ? NETWORK_VIDEO_POOL_BYTE_LIMIT / textureBytes : 0U; + const uint64_t boundedLimit = textureLimit < NETWORK_VIDEO_POOL_MIN_SIZE ? + NETWORK_VIDEO_POOL_MIN_SIZE : textureLimit; LG_LOCK_INIT(pool->lock); atomic_init(&pool->references, 1U); + pool->presentEvent = decoder->presentEvent; pool->display = decoder->display; pool->eglConfig = decoder->eglConfig; pool->shareContext = decoder->shareContext; pool->width = decoder->config.width; pool->height = decoder->config.height; - pool->textureLimit = textureLimit < NETWORK_VIDEO_POOL_SIZE ? - (unsigned)textureLimit : NETWORK_VIDEO_POOL_SIZE; + pool->textureLimit = boundedLimit < NETWORK_VIDEO_POOL_SIZE ? + (unsigned)boundedLimit : NETWORK_VIDEO_POOL_SIZE; return pool; } @@ -408,6 +425,17 @@ static void networkVideoTexturePoolRetain( atomic_fetch_add_explicit(&pool->references, 1U, memory_order_relaxed); } +static void networkVideoTexturePoolDetachEvent( + struct NetworkVideoTexturePool * pool) +{ + if (!pool) + return; + + LG_LOCK(pool->lock); + pool->presentEvent = NULL; + LG_UNLOCK(pool->lock); +} + static void networkVideoTexturePoolRelease( struct NetworkVideoTexturePool * pool, bool current) { @@ -452,7 +480,7 @@ static bool networkVideoTexturePoolReturn( bool retained = false; LG_LOCK(pool->lock); - if (pool->textureCount < pool->textureLimit) + if (pool->textureCount < pool->textureAllocated) { struct NetworkVideoTexture * pooled = &pool->textures[pool->textureCount++]; @@ -460,6 +488,8 @@ static bool networkVideoTexturePoolReturn( pooled->readySync = readySync; pooled->generation = generation; retained = true; + if (pool->presentEvent) + lgSignalEvent(pool->presentEvent); } LG_UNLOCK(pool->lock); return retained; @@ -549,6 +579,7 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder) if (upload->payloadBuffer) glDeleteBuffers(1, &upload->payloadBuffer); } + networkVideoTexturePoolDetachEvent(decoder->texturePool); networkVideoTexturePoolRelease(decoder->texturePool, true); networkVideoFreePacketStates( decoder->packetStates, decoder->blockCapacity); @@ -563,23 +594,26 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder) memset(decoder->uploads, 0, sizeof(decoder->uploads)); memset(decoder->damageHistory, 0, sizeof(decoder->damageHistory)); - decoder->texturePool = NULL; - decoder->offsets = NULL; - decoder->packetStates = NULL; - decoder->pendingPackets = NULL; - decoder->dirtyBlocks = NULL; - decoder->outputDirty = NULL; - decoder->canonicalTexture = 0; - decoder->alignedWidth = 0; - decoder->alignedHeight = 0; - decoder->blockCapacity = 0; - decoder->outputTileWidth = 0; - decoder->outputTileHeight = 0; - decoder->referenceFrameID = 0; - decoder->outputGeneration = 0; - decoder->uploadIndex = 0; - decoder->configured = false; - decoder->referenceValid = false; + memset(&decoder->pendingDamage, 0, sizeof(decoder->pendingDamage)); + decoder->texturePool = NULL; + decoder->offsets = NULL; + decoder->packetStates = NULL; + decoder->pendingPackets = NULL; + decoder->dirtyBlocks = NULL; + decoder->outputDirty = NULL; + decoder->canonicalTexture = 0; + decoder->alignedWidth = 0; + decoder->alignedHeight = 0; + decoder->blockCapacity = 0; + decoder->outputTileWidth = 0; + decoder->outputTileHeight = 0; + decoder->referenceFrameID = 0; + decoder->outputGeneration = 0; + decoder->pendingDecodeTimeNs = 0; + decoder->uploadIndex = 0; + decoder->configured = false; + decoder->referenceValid = false; + decoder->pendingChromaLeft = false; memset(decoder->coefficientTextures, 0, sizeof(decoder->coefficientTextures)); memset(decoder->planeTextures, 0, sizeof(decoder->planeTextures)); @@ -800,10 +834,32 @@ static bool networkVideoCreateTexture2D(GLuint * texture, GLenum format, return false; } +static bool networkVideoTexturePoolAllocateNL( + struct NetworkVideoTexturePool * pool) +{ + while (pool->textureAllocated < NETWORK_VIDEO_POOL_MIN_SIZE) + { + struct NetworkVideoTexture * pooled = + &pool->textures[pool->textureCount]; + if (!networkVideoCreateTexture2D(&pooled->texture, GL_RGBA16F, + pool->width, pool->height, GL_LINEAR)) + { + while (glGetError() != GL_NO_ERROR) + { + } + return false; + } + ++pool->textureCount; + ++pool->textureAllocated; + } + return true; +} + static struct NetworkVideoTexture networkVideoTexturePoolAcquire( struct NetworkVideoTexturePool * pool) { - struct NetworkVideoTexture pooled = { 0 }; + struct NetworkVideoTexture pooled = { 0 }; + bool allocate = false; LG_LOCK(pool->lock); if (pool->textureCount) { @@ -811,15 +867,26 @@ static struct NetworkVideoTexture networkVideoTexturePoolAcquire( memset(&pool->textures[pool->textureCount], 0, sizeof(pool->textures[pool->textureCount])); } + else if (pool->textureAllocated < pool->textureLimit) + { + ++pool->textureAllocated; + allocate = true; + } LG_UNLOCK(pool->lock); + + if (allocate && !networkVideoCreateTexture2D(&pooled.texture, GL_RGBA16F, + pool->width, pool->height, GL_LINEAR)) + { + while (glGetError() != GL_NO_ERROR) + { + } + LG_LOCK(pool->lock); + --pool->textureAllocated; + pool->textureLimit = pool->textureAllocated; + LG_UNLOCK(pool->lock); + } if (pooled.readySync) glDeleteSync(pooled.readySync); - if (pooled.texture) - return pooled; - - if (!networkVideoCreateTexture2D(&pooled.texture, GL_RGBA16F, - pool->width, pool->height, GL_LINEAR)) - pooled.texture = 0; return pooled; } @@ -888,7 +955,8 @@ static bool networkVideoAllocateNL(NetworkVideoDecoder * decoder) outputTileCount, sizeof(*decoder->outputDirty)); if (!decoder->offsets || !decoder->packetStates || !decoder->pendingPackets || !decoder->dirtyBlocks || - !decoder->texturePool || !decoder->outputDirty) + !decoder->texturePool || !decoder->outputDirty || + !networkVideoTexturePoolAllocateNL(decoder->texturePool)) return false; for (unsigned component = 0; @@ -997,16 +1065,18 @@ static const LG_ExternalFrameOps NETWORK_VIDEO_EXTERNAL_OPS = .query = networkVideoExternalQuery, }; -bool networkVideoDecoderCreate(NetworkVideoDecoder ** result) +bool networkVideoDecoderCreate(NetworkVideoDecoder ** result, + LGEvent * presentEvent) { - if (!result) + if (!result || !presentEvent) return false; NetworkVideoDecoder * decoder = calloc(1, sizeof(*decoder)); if (!decoder) return false; LG_LOCK_INIT(decoder->lock); - *result = decoder; + decoder->presentEvent = presentEvent; + *result = decoder; return true; } @@ -1027,6 +1097,12 @@ void networkVideoDecoderDetach(NetworkVideoDecoder * decoder) } eglDestroyContext(decoder->display, decoder->context); } + if (decoder->texturePool) + { + networkVideoTexturePoolDetachEvent(decoder->texturePool); + networkVideoTexturePoolRelease(decoder->texturePool, false); + decoder->texturePool = NULL; + } decoder->display = EGL_NO_DISPLAY; decoder->eglConfig = NULL; decoder->shareContext = EGL_NO_CONTEXT; @@ -1037,8 +1113,11 @@ void networkVideoDecoderDetach(NetworkVideoDecoder * decoder) decoder->maxWorkGroupCount[1] = 0; decoder->maxStorageBlockSize = 0; decoder->referenceFrameID = 0; + decoder->pendingDecodeTimeNs = 0; decoder->attached = false; decoder->referenceValid = false; + decoder->pendingChromaLeft = false; + memset(&decoder->pendingDamage, 0, sizeof(decoder->pendingDamage)); LG_UNLOCK(decoder->lock); } @@ -1068,8 +1147,11 @@ void networkVideoDecoderReset(NetworkVideoDecoder * decoder) networkVideoRestoreCurrent(decoder, &previous); } } - decoder->referenceFrameID = 0; - decoder->referenceValid = false; + decoder->referenceFrameID = 0; + decoder->pendingDecodeTimeNs = 0; + decoder->referenceValid = false; + decoder->pendingChromaLeft = false; + memset(&decoder->pendingDamage, 0, sizeof(decoder->pendingDamage)); LG_UNLOCK(decoder->lock); } @@ -1765,6 +1847,11 @@ static bool networkVideoDamageFull(const LGNetVideoDamage * damage) return (damage->flags & LG_NET_VIDEO_DAMAGE_FULL) != 0; } +static bool networkVideoDamagePresent(const LGNetVideoDamage * damage) +{ + return networkVideoDamageFull(damage) || damage->count; +} + static void networkVideoDispatchRect(GLint offsetUniform, GLint extentUniform, uint32_t x, uint32_t y, uint32_t width, uint32_t height) @@ -1994,6 +2081,95 @@ static bool networkVideoAppendDamageRect(LGNetVideoDamage * damage, return true; } +static bool networkVideoUnionDamageRect(LGNetVideoDamage * damage, + const LGNetVideoDamageRect * source) +{ + uint32_t x = source->x; + uint32_t y = source->y; + uint32_t endX = source->x + source->width; + uint32_t endY = source->y + source->height; + for (uint16_t index = 0; index < damage->count;) + { + const LGNetVideoDamageRect * current = &damage->rects[index]; + const uint32_t currentEndX = + current->x + current->width; + const uint32_t currentEndY = + current->y + current->height; + if (endX < current->x || currentEndX < x || + endY < current->y || currentEndY < y) + { + ++index; + continue; + } + + x = current->x < x ? current->x : x; + y = current->y < y ? current->y : y; + endX = currentEndX > endX ? currentEndX : endX; + endY = currentEndY > endY ? currentEndY : endY; + damage->rects[index] = damage->rects[--damage->count]; + } + return networkVideoAppendDamageRect( + damage, x, y, endX - x, endY - y); +} + +static void networkVideoMergeDamage(LGNetVideoDamage * destination, + const LGNetVideoDamage * source) +{ + if (!networkVideoDamagePresent(source) || + networkVideoDamageFull(destination)) + return; + + if (networkVideoDamageFull(source)) + { + memset(destination, 0, sizeof(*destination)); + destination->flags = + (LGNetVideoDamageFlags)LG_NET_VIDEO_DAMAGE_FULL; + return; + } + + for (uint16_t index = 0; index < source->count; ++index) + { + const LGNetVideoDamageRect * rect = &source->rects[index]; + if (networkVideoUnionDamageRect(destination, rect)) + continue; + + uint32_t left = UINT32_MAX; + uint32_t top = UINT32_MAX; + uint32_t right = 0; + uint32_t bottom = 0; + for (unsigned set = 0; set < 2; ++set) + { + const LGNetVideoDamageRect * rects = set ? + source->rects : destination->rects; + const uint16_t count = set ? + source->count : destination->count; + for (uint16_t rectIndex = 0; rectIndex < count; ++rectIndex) + { + const LGNetVideoDamageRect * current = &rects[rectIndex]; + const uint32_t currentRight = + current->x + current->width; + const uint32_t currentBottom = + current->y + current->height; + left = current->x < left ? current->x : left; + top = current->y < top ? current->y : top; + right = currentRight > right ? currentRight : right; + bottom = currentBottom > bottom ? currentBottom : bottom; + } + } + + memset(destination, 0, sizeof(*destination)); + destination->count = 1; + destination->rects[0] = (LGNetVideoDamageRect) + { + .x = (uint16_t)left, + .y = (uint16_t)top, + .width = (uint16_t)(right - left), + .height = (uint16_t)(bottom - top), + }; + return; + } +} + static void networkVideoIntersectDamage( const LGNetVideoDamage * source, const LGNetVideoDamage * reconstruction, @@ -2074,10 +2250,20 @@ static void networkVideoBuildDamage(const LGNetVideoDamage * damage, result->damageRectsCount = damage->count; } -static struct NetworkVideoExternalFrame * networkVideoConvertNL( - NetworkVideoDecoder * decoder, bool chromaLeft, uint64_t generation, - const LGNetVideoDamage * damage) +static bool networkVideoPresentNL(NetworkVideoDecoder * decoder, + struct NetworkVideoTexture pooled, bool chromaLeft, uint64_t generation, + const LGNetVideoDamage * damage, + struct NetworkVideoExternalFrame ** result) { + *result = NULL; + struct NetworkVideoExternalFrame * frame = calloc(1, sizeof(*frame)); + if (!frame) + { + networkVideoTexturePoolRecycleCurrent(decoder->texturePool, + pooled.texture, pooled.readySync, pooled.generation); + return false; + } + glUseProgram(decoder->yuvProgram); for (unsigned component = 0; component < NETWORK_VIDEO_COMPONENTS; ++component) @@ -2108,28 +2294,16 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL( glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT | GL_TEXTURE_FETCH_BARRIER_BIT); - struct NetworkVideoTexture pooled = - networkVideoTexturePoolAcquire(decoder->texturePool); - if (!pooled.texture) - return NULL; networkVideoCopyCanonicalNL(decoder, pooled.texture, pooled.generation, generation, damage); GLsync readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); - glFlush(); if (!readySync || glGetError() != GL_NO_ERROR) { + free(frame); networkVideoTexturePoolRecycleCurrent( decoder->texturePool, pooled.texture, readySync, 0); - return NULL; - } - - struct NetworkVideoExternalFrame * frame = calloc(1, sizeof(*frame)); - if (!frame) - { - networkVideoTexturePoolRecycleCurrent( - decoder->texturePool, pooled.texture, readySync, 0); - return NULL; + return false; } frame->base.ops = &NETWORK_VIDEO_EXTERNAL_OPS; frame->pool = decoder->texturePool; @@ -2138,7 +2312,43 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL( frame->generation = generation; networkVideoTexturePoolRetain(frame->pool); atomic_init(&frame->references, 1U); - return frame; + *result = frame; + return true; +} + +static void networkVideoBuildFrameNL(NetworkVideoDecoder * decoder, + struct NetworkVideoExternalFrame * external, + const LGNetVideoDamage * damage, uint64_t decodeTime, + NetworkVideoDecodedFrame * result) +{ + uint32_t formatVersion = (uint32_t)decoder->config.configEpoch; + if (!formatVersion) + formatVersion = 1; + + result->externalFrame = &external->base; + result->decodeTimeNs = decodeTime; + networkVideoBuildDamage(damage, result); + result->format = (LG_TransportFrameFormat) + { + .version = formatVersion, + .type = FRAME_TYPE_RGBA16F, + .screenWidth = decoder->config.width, + .screenHeight = decoder->config.height, + .dataWidth = decoder->config.width, + .dataHeight = decoder->config.height, + .frameWidth = decoder->config.width, + .frameHeight = decoder->config.height, + .rotation = FRAME_ROT_0, + .stride = decoder->config.width, + .pitch = decoder->config.width * 8U, + .hdr = (decoder->config.flags & LG_NET_VIDEO_STREAM_HDR) != 0, + .hdrPQ = decoder->config.colorTransfer == + LG_NET_COLOR_TRANSFER_PQ, + .hdrMetadata = false, + .sdrWhiteLevel = LG_SDR_WHITE_LEVEL_DEFAULT, + }; + result->timing.providerValid = true; + result->timing.prepareTime = decodeTime; } NetworkVideoDecodeResult networkVideoDecoderDecode( @@ -2186,7 +2396,9 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( LG_UNLOCK(decoder->lock); return NETWORK_VIDEO_DECODE_ERROR; } - if (update && !changedCount) + const bool codecChanges = !update || changedCount; + if (!codecChanges && + !networkVideoDamagePresent(&decoder->pendingDamage)) { networkVideoCommitPendingPacketsNL(decoder); decoder->referenceFrameID = frameID; @@ -2194,10 +2406,13 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( return NETWORK_VIDEO_DECODE_NO_FRAME; } - if (update) - networkVideoPrepareDirtyNL(decoder); - else - networkVideoClearDirtyNL(decoder); + if (codecChanges) + { + if (update) + networkVideoPrepareDirtyNL(decoder); + else + networkVideoClearDirtyNL(decoder); + } struct NetworkVideoCurrent previous; if (!networkVideoMakeCurrent(decoder, &previous)) @@ -2210,52 +2425,71 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( while (glGetError() != GL_NO_ERROR) { } - struct NetworkVideoUpload * upload = networkVideoUploadNL( - decoder, data, size); - if (!upload) + struct NetworkVideoUpload * upload = NULL; + if (codecChanges) { - networkVideoDiscardPendingPacketsNL(decoder); - networkVideoRestoreCurrent(decoder, &previous); - LG_UNLOCK(decoder->lock); - return NETWORK_VIDEO_DECODE_ERROR; + upload = networkVideoUploadNL(decoder, data, size); + if (!upload) + { + networkVideoDiscardPendingPacketsNL(decoder); + networkVideoRestoreCurrent(decoder, &previous); + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_ERROR; + } } - const uint64_t decodeStart = nanotime(); - uint64_t generation = decoder->outputGeneration + 1U; + const uint64_t decodeStart = nanotime(); + uint64_t generation = decoder->outputGeneration + 1U; + LGNetVideoDamage damage = { 0 }; if (!generation) generation = 1U; - networkVideoDequantNL(decoder, upload, update); - upload->readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); - networkVideoIDWTNL(decoder, update); - LGNetVideoDamage reconstructionDamage; - LGNetVideoDamage damage; - networkVideoBuildReconstructionDamageNL( - decoder, update, &reconstructionDamage); - /* Wire damage accumulates against the sender's acknowledged reference. - * Limit it to output reconstructed against the decoder's current state. */ - if (update) - networkVideoIntersectDamage( - &layout.damage, &reconstructionDamage, &damage); - else - damage = reconstructionDamage; - const bool hasPresentationDamage = - networkVideoDamageFull(&damage) || damage.count; + if (codecChanges) + { + networkVideoDequantNL(decoder, upload, update); + networkVideoIDWTNL(decoder, update); + LGNetVideoDamage reconstructionDamage; + networkVideoBuildReconstructionDamageNL( + decoder, update, &reconstructionDamage); + /* Wire damage accumulates against the sender's acknowledged reference. + * Limit it to output reconstructed against the decoder's current state. */ + if (update) + networkVideoIntersectDamage( + &layout.damage, &reconstructionDamage, &damage); + else + damage = reconstructionDamage; + } - struct NetworkVideoExternalFrame * external = NULL; + LGNetVideoDamage presentationDamage = decoder->pendingDamage; + networkVideoMergeDamage(&presentationDamage, &damage); + const bool hasPresentationDamage = + networkVideoDamagePresent(&presentationDamage); + + struct NetworkVideoTexture pooled = { 0 }; + struct NetworkVideoExternalFrame * external = NULL; + bool presented = true; if (hasPresentationDamage) - external = networkVideoConvertNL(decoder, - layout.chromaLeft, generation, &damage); - else - glFlush(); + { + pooled = networkVideoTexturePoolAcquire(decoder->texturePool); + if (pooled.texture) + presented = networkVideoPresentNL(decoder, pooled, + layout.chromaLeft, generation, &presentationDamage, &external); + } + if (upload) + upload->readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); + glFlush(); const uint64_t decodeTime = nanotime() - decodeStart; - const bool success = - (!hasPresentationDamage || external) && glGetError() == GL_NO_ERROR; + const bool success = presented && + (!upload || upload->readySync) && glGetError() == GL_NO_ERROR; networkVideoRestoreCurrent(decoder, &previous); if (!success) { networkVideoDiscardPendingPacketsNL(decoder); - decoder->referenceFrameID = 0; - decoder->referenceValid = false; + decoder->referenceFrameID = 0; + decoder->referenceValid = false; + decoder->pendingDecodeTimeNs = 0; + decoder->pendingChromaLeft = false; + memset(&decoder->pendingDamage, 0, + sizeof(decoder->pendingDamage)); if (external) networkVideoExternalRelease(&external->base); LG_UNLOCK(decoder->lock); @@ -2270,41 +2504,111 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( LG_UNLOCK(decoder->lock); return NETWORK_VIDEO_DECODE_NO_FRAME; } + if (!pooled.texture) + { + decoder->pendingDamage = presentationDamage; + decoder->pendingDecodeTimeNs = networkVideoSaturatingAdd( + decoder->pendingDecodeTimeNs, decodeTime); + decoder->pendingChromaLeft = layout.chromaLeft; + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_DEFERRED; + } + const uint64_t totalDecodeTime = networkVideoSaturatingAdd( + decoder->pendingDecodeTimeNs, decodeTime); + memset(&decoder->pendingDamage, 0, sizeof(decoder->pendingDamage)); + decoder->pendingDecodeTimeNs = 0; + decoder->pendingChromaLeft = false; struct NetworkVideoDamageHistory * history = &decoder->damageHistory[generation % NETWORK_VIDEO_DAMAGE_HISTORY]; history->generation = generation; - history->damage = damage; + history->damage = presentationDamage; decoder->outputGeneration = generation; - uint32_t formatVersion = (uint32_t)config->configEpoch; - if (!formatVersion) - formatVersion = 1; + networkVideoBuildFrameNL( + decoder, external, &presentationDamage, totalDecodeTime, result); + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_OK; +} - result->externalFrame = &external->base; - result->decodeTimeNs = decodeTime; - if (update) - networkVideoBuildDamage(&damage, result); - result->format = (LG_TransportFrameFormat) +NetworkVideoDecodeResult networkVideoDecoderPresentPending( + NetworkVideoDecoder * decoder, NetworkVideoDecodedFrame * result) +{ + if (!decoder || !result) + return NETWORK_VIDEO_DECODE_ERROR; + memset(result, 0, sizeof(*result)); + + LG_LOCK(decoder->lock); + if (!decoder->configured) { - .version = formatVersion, - .type = FRAME_TYPE_RGBA16F, - .screenWidth = config->width, - .screenHeight = config->height, - .dataWidth = config->width, - .dataHeight = config->height, - .frameWidth = config->width, - .frameHeight = config->height, - .rotation = FRAME_ROT_0, - .stride = config->width, - .pitch = config->width * 8U, - .hdr = (config->flags & LG_NET_VIDEO_STREAM_HDR) != 0, - .hdrPQ = config->colorTransfer == LG_NET_COLOR_TRANSFER_PQ, - .hdrMetadata = false, - .sdrWhiteLevel = LG_SDR_WHITE_LEVEL_DEFAULT, - }; - result->timing.providerValid = true; - result->timing.prepareTime = decodeTime; + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_UNSUPPORTED; + } + if (!networkVideoDamagePresent(&decoder->pendingDamage)) + { + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_NO_FRAME; + } + + struct NetworkVideoCurrent previous; + if (!networkVideoMakeCurrent(decoder, &previous)) + { + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_ERROR; + } + + while (glGetError() != GL_NO_ERROR) + { + } + struct NetworkVideoTexture pooled = + networkVideoTexturePoolAcquire(decoder->texturePool); + if (!pooled.texture) + { + networkVideoRestoreCurrent(decoder, &previous); + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_DEFERRED; + } + + uint64_t generation = decoder->outputGeneration + 1U; + if (!generation) + generation = 1U; + const uint64_t presentStart = nanotime(); + struct NetworkVideoExternalFrame * external = NULL; + const bool presented = networkVideoPresentNL( + decoder, pooled, + decoder->pendingChromaLeft, generation, &decoder->pendingDamage, + &external); + glFlush(); + const uint64_t presentTime = nanotime() - presentStart; + const bool success = presented && glGetError() == GL_NO_ERROR; + networkVideoRestoreCurrent(decoder, &previous); + if (!success) + { + decoder->referenceFrameID = 0; + decoder->referenceValid = false; + decoder->pendingDecodeTimeNs = 0; + decoder->pendingChromaLeft = false; + memset(&decoder->pendingDamage, 0, + sizeof(decoder->pendingDamage)); + if (external) + networkVideoExternalRelease(&external->base); + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_ERROR; + } + + const LGNetVideoDamage damage = decoder->pendingDamage; + const uint64_t decodeTime = networkVideoSaturatingAdd( + decoder->pendingDecodeTimeNs, presentTime); + struct NetworkVideoDamageHistory * history = + &decoder->damageHistory[generation % NETWORK_VIDEO_DAMAGE_HISTORY]; + history->generation = generation; + history->damage = damage; + decoder->outputGeneration = generation; + memset(&decoder->pendingDamage, 0, sizeof(decoder->pendingDamage)); + decoder->pendingDecodeTimeNs = 0; + decoder->pendingChromaLeft = false; + networkVideoBuildFrameNL( + decoder, external, &damage, decodeTime, result); LG_UNLOCK(decoder->lock); return NETWORK_VIDEO_DECODE_OK; }