From 5eca9a687f898c55ab6ab501602fba4e6aec6b14 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Thu, 8 Oct 2026 12:46:42 +1100 Subject: [PATCH] [client/idd] network: process only damaged PyroWave blocks Limit GPU encoding and client decoding to damage-derived regions. Preserve per-level sparse transform state and cover mirrored inverse edges. Keep cumulative canonical packets so dropped frames remain recoverable. --- client/transports/Network/video.c | 119 +- client/transports/Network/video/decoder.h | 11 +- client/transports/Network/video/decoder_egl.c | 777 ++++++++- client/transports/Network/video/shaders.h | 44 +- idd/LGIdd/transport/network/CNetworkVideo.cpp | 1409 ++++++++++++++--- 5 files changed, 2028 insertions(+), 332 deletions(-) diff --git a/client/transports/Network/video.c b/client/transports/Network/video.c index f5b87664..79a4fb5e 100644 --- a/client/transports/Network/video.c +++ b/client/transports/Network/video.c @@ -286,6 +286,66 @@ static void networkVideoFreeOutput(struct NetworkVideoOutput ** output) *output = NULL; } +static void networkVideoMergeDamage(NetworkVideoDecodedFrame * destination, + const NetworkVideoDecodedFrame * source) +{ + if (!destination || !source) + return; + + if (!destination->damageRectsCount || !source->damageRectsCount || + destination->damageRectsCount > LG_TRANSPORT_MAX_DAMAGE_RECTS || + source->damageRectsCount > LG_TRANSPORT_MAX_DAMAGE_RECTS) + { + destination->damageRectsCount = 0; + return; + } + + if (destination->damageRectsCount >= + LG_TRANSPORT_MAX_DAMAGE_RECTS - source->damageRectsCount) + { + uint32_t left = UINT32_MAX; + uint32_t top = UINT32_MAX; + uint64_t right = 0; + uint64_t bottom = 0; + for (unsigned set = 0; set < 2; ++set) + { + const KVMFRFrameDamageRect * rects = set ? + source->damageRects : destination->damageRects; + const uint32_t count = set ? + source->damageRectsCount : destination->damageRectsCount; + for (uint32_t index = 0; index < count; ++index) + { + left = rects[index].x < left ? rects[index].x : left; + top = rects[index].y < top ? rects[index].y : top; + right = (uint64_t)rects[index].x + rects[index].width > right ? + (uint64_t)rects[index].x + rects[index].width : right; + bottom = (uint64_t)rects[index].y + rects[index].height > bottom ? + (uint64_t)rects[index].y + rects[index].height : bottom; + } + } + if (right <= left || bottom <= top || right - left > UINT32_MAX || + bottom - top > UINT32_MAX) + { + destination->damageRectsCount = 0; + return; + } + destination->damageRects[0] = (KVMFRFrameDamageRect) + { + .x = left, + .y = top, + .width = (uint32_t)(right - left), + .height = (uint32_t)(bottom - top), + }; + destination->damageRectsCount = 1; + return; + } + + memcpy(destination->damageRects + destination->damageRectsCount, + source->damageRects, + source->damageRectsCount * sizeof(*source->damageRects)); + destination->damageRectsCount += source->damageRectsCount; +} + static void networkVideoClearEncoded(struct NetworkVideoEncoded * encoded) { if (!encoded) @@ -902,7 +962,8 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video, if (assembly->active) { if (assembly->update && update && - assembly->baseFrameID != baseFrameID) + assembly->baseFrameID != baseFrameID && + (!baseFrameID || baseFrameID > video->lastDecodedFrameID)) return false; video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); if (!assembly->update && !update && *encodedCount < 2U && @@ -1080,26 +1141,47 @@ static void networkVideoDecode(NetworkVideo * video, return; } - NetworkVideoDecodedFrame decoded; - const NetworkVideoDecodeResult result = networkVideoDecoderDecode( + struct NetworkVideoOutput * output = calloc(1, sizeof(*output)); + if (!output) + { + free(encoded->ownedData); + networkVideoDecodeFailure( + video, encoded, NETWORK_VIDEO_DECODE_ERROR); + return; + } + + NetworkVideoDecodedFrame decoded = {0}; + const NetworkVideoDecodeResult result = networkVideoDecoderDecode( video->decoder, &encoded->config, encoded->data, encoded->length, encoded->blockCount, encoded->update, encoded->complete, encoded->baseFrameID, encoded->frameID, &decoded); free(encoded->ownedData); + if (result == NETWORK_VIDEO_DECODE_NO_FRAME) + { + free(output); + LG_LOCK(video->stateLock); + if (video->decodeGeneration == encoded->generation && + video->wireConfigured && + video->config.configEpoch == encoded->config.configEpoch && + video->rendererAttached && video->decoderReady && + encoded->frameID > video->lastDecodedFrameID) + { + if (encoded->complete) + video->lastCompleteFrameID = encoded->frameID; + video->lastDecodedFrameID = encoded->frameID; + networkVideoQueueFeedbackNL(video, 0); + } + LG_UNLOCK(video->stateLock); + return; + } if (result != NETWORK_VIDEO_DECODE_OK) { + free(output); networkVideoDecodeFailure(video, encoded, result); return; } - struct NetworkVideoOutput * output = calloc(1, sizeof(*output)); struct NetworkVideoOutput * replaced = NULL; - if (!output) - { - networkVideoDecoderRelease(&decoded); - networkVideoDecodeFailure(video, encoded, NETWORK_VIDEO_DECODE_ERROR); - return; - } output->decoded = decoded; output->frameID = encoded->frameID; output->timestamp = encoded->presentationTimestampNs; @@ -1122,6 +1204,7 @@ static void networkVideoDecode(NetworkVideo * video, { video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); replaced = video->output; + networkVideoMergeDamage(&output->decoded, &replaced->decoded); } if (encoded->complete) video->lastCompleteFrameID = output->frameID; @@ -1150,13 +1233,12 @@ static void networkVideoQueueDecode(NetworkVideo * video, bool queued = false; bool replaceable; LG_LOCK(video->stateLock); - baseMismatch = encoded->update && video->decodePending && - video->pendingDecode.update && - video->pendingDecode.baseFrameID != encoded->baseFrameID; + baseMismatch = encoded->update && + (!encoded->baseFrameID || + encoded->baseFrameID > video->lastDecodedFrameID); replaceable = !video->decodePending || !encoded->update || - (video->pendingDecode.update && - video->pendingDecode.baseFrameID == encoded->baseFrameID); + (video->pendingDecode.update && !baseMismatch); if (baseMismatch) { video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); @@ -1494,8 +1576,8 @@ static bool networkVideoHandleFrameUpdate(NetworkVideo * video, video->assembly.frameID <= update.frameID) { video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); - baseMismatch = video->assembly.update && - video->assembly.baseFrameID != update.baseFrameID; + baseMismatch = !update.baseFrameID || + update.baseFrameID > video->lastDecodedFrameID; retiredAssembly = networkVideoTakeAssemblyNL(video); if (baseMismatch) networkVideoRequestKeyframeNL(video, @@ -2234,6 +2316,9 @@ LG_TransportStatus networkVideoNextFrame(NetworkVideo * video, bool useDMA, frame->format = &output->decoded.format; frame->externalFrame = lgExternalFrameRetain(output->decoded.externalFrame); + frame->damageRects = output->decoded.damageRectsCount ? + output->decoded.damageRects : NULL; + frame->damageRectsCount = output->decoded.damageRectsCount; frame->releaseFn = networkVideoReleaseLease; frame->releaseOpaque = video; frame->releaseHandle = output->handle; diff --git a/client/transports/Network/video/decoder.h b/client/transports/Network/video/decoder.h index 9e82a708..e81b4715 100644 --- a/client/transports/Network/video/decoder.h +++ b/client/transports/Network/video/decoder.h @@ -35,6 +35,7 @@ typedef struct NetworkVideoDecoder NetworkVideoDecoder; typedef enum NetworkVideoDecodeResult { NETWORK_VIDEO_DECODE_OK, + NETWORK_VIDEO_DECODE_NO_FRAME, NETWORK_VIDEO_DECODE_UNSUPPORTED, NETWORK_VIDEO_DECODE_CORRUPT, NETWORK_VIDEO_DECODE_ERROR, @@ -43,10 +44,12 @@ NetworkVideoDecodeResult; typedef struct NetworkVideoDecodedFrame { - LG_ExternalFrame * externalFrame; - LG_TransportFrameFormat format; - LG_TransportFrameTiming timing; - uint64_t decodeTimeNs; + LG_ExternalFrame * externalFrame; + LG_TransportFrameFormat format; + LG_TransportFrameTiming timing; + KVMFRFrameDamageRect damageRects[LG_TRANSPORT_MAX_DAMAGE_RECTS]; + uint32_t damageRectsCount; + uint64_t decodeTimeNs; } NetworkVideoDecodedFrame; diff --git a/client/transports/Network/video/decoder_egl.c b/client/transports/Network/video/decoder_egl.c index 3dfe1678..11f73e15 100644 --- a/client/transports/Network/video/decoder_egl.c +++ b/client/transports/Network/video/decoder_egl.c @@ -71,6 +71,8 @@ #define NETWORK_VIDEO_POOL_BYTE_LIMIT \ (UINT64_C(192) * 1024U * 1024U) #define NETWORK_VIDEO_PIXEL_BYTES 8U +#define NETWORK_VIDEO_TILE_SIZE 32U +#define NETWORK_VIDEO_SEQUENCE_BITS (UINT32_C(7) << 28U) struct NetworkVideoBlockMeta { @@ -79,6 +81,20 @@ struct NetworkVideoBlockMeta uint32_t count; }; +struct NetworkVideoPacketState +{ + uint8_t * data; + uint32_t length; +}; + +struct NetworkVideoDirtyGrid +{ + uint8_t * tiles; + uint32_t width; + uint32_t height; + uint32_t count; +}; + struct NetworkVideoCurrent { EGLDisplay display; @@ -98,8 +114,9 @@ struct NetworkVideoUpload struct NetworkVideoTexture { - GLuint texture; - GLsync readySync; + GLuint texture; + GLsync readySync; + uint64_t generation; }; struct NetworkVideoTexturePool @@ -125,6 +142,7 @@ struct NetworkVideoUniforms GLint outputLayer; GLint blockOffset; GLint blockStride; + GLint workGroupOffset; GLint preserveMissing; } dequant; @@ -135,6 +153,7 @@ struct NetworkVideoUniforms GLint resolution; GLint invResolution; GLint baseLayer; + GLint workGroupOffset; GLint dcShift; } idwt; @@ -145,6 +164,7 @@ struct NetworkVideoUniforms GLint cbPlane; GLint crPlane; GLint outputSize; + GLint outputOffset; GLint bt2020; GLint chromaLeft; GLint lumaOffset; @@ -173,6 +193,13 @@ struct NetworkVideoDecoder struct NetworkVideoTexturePool * texturePool; struct NetworkVideoUniforms uniforms; uint32_t * offsets; + struct NetworkVideoPacketState * packetStates; + struct NetworkVideoPacketState * pendingPackets; + uint8_t * dirtyBlocks; + struct NetworkVideoDirtyGrid + dirty[NETWORK_VIDEO_COMPONENTS][NETWORK_VIDEO_LEVELS]; + uint8_t * outputDirty; + uint64_t * outputTileGenerations; LGNetVideoStreamConfig config; struct NetworkVideoBlockMeta @@ -182,7 +209,10 @@ struct NetworkVideoDecoder uint32_t alignedWidth; uint32_t alignedHeight; uint32_t blockCapacity; + uint32_t outputTileWidth; + uint32_t outputTileHeight; uint64_t referenceFrameID; + uint64_t outputGeneration; unsigned uploadIndex; GLint maxTextureSize; GLint maxArrayLayers; @@ -200,6 +230,7 @@ struct NetworkVideoExternalFrame struct NetworkVideoTexturePool * pool; GLuint texture; GLsync readySync; + uint64_t generation; }; static uint32_t networkVideoAlign(uint32_t value, uint32_t alignment) @@ -207,6 +238,32 @@ static uint32_t networkVideoAlign(uint32_t value, uint32_t alignment) return (value + alignment - 1U) & ~(alignment - 1U); } +static uint32_t networkVideoReadU32(const uint8_t * data) +{ + return (uint32_t)data[0] | + (uint32_t)data[1] << 8U | + (uint32_t)data[2] << 16U | + (uint32_t)data[3] << 24U; +} + +static void networkVideoWriteU32(uint8_t * data, uint32_t value) +{ + data[0] = (uint8_t)value; + data[1] = (uint8_t)(value >> 8U); + data[2] = (uint8_t)(value >> 16U); + data[3] = (uint8_t)(value >> 24U); +} + +static void networkVideoFreePacketStates( + struct NetworkVideoPacketState * states, uint32_t count) +{ + if (!states) + return; + for (uint32_t index = 0; index < count; ++index) + free(states[index].data); + free(states); +} + static void networkVideoSaveCurrent(struct NetworkVideoCurrent * current) { current->display = eglGetCurrentDisplay(); @@ -368,7 +425,8 @@ static void networkVideoTexturePoolRelease( } static bool networkVideoTexturePoolReturn( - struct NetworkVideoTexturePool * pool, GLuint texture, GLsync readySync) + struct NetworkVideoTexturePool * pool, GLuint texture, GLsync readySync, + uint64_t generation) { if (!pool || !texture) return false; @@ -379,18 +437,21 @@ static bool networkVideoTexturePoolReturn( { struct NetworkVideoTexture * pooled = &pool->textures[pool->textureCount++]; - pooled->texture = texture; - pooled->readySync = readySync; - retained = true; + pooled->texture = texture; + pooled->readySync = readySync; + pooled->generation = generation; + retained = true; } LG_UNLOCK(pool->lock); return retained; } static void networkVideoTexturePoolRecycleCurrent( - struct NetworkVideoTexturePool * pool, GLuint texture, GLsync readySync) + struct NetworkVideoTexturePool * pool, GLuint texture, GLsync readySync, + uint64_t generation) { - if (networkVideoTexturePoolReturn(pool, texture, readySync)) + if (networkVideoTexturePoolReturn( + pool, texture, readySync, generation)) return; if (readySync) @@ -468,21 +529,40 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder) glDeleteBuffers(1, &upload->payloadBuffer); } networkVideoTexturePoolRelease(decoder->texturePool, true); + networkVideoFreePacketStates( + decoder->packetStates, decoder->blockCapacity); + free(decoder->pendingPackets); + free(decoder->dirtyBlocks); + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level) + free(decoder->dirty[component][level].tiles); + free(decoder->outputDirty); + free(decoder->outputTileGenerations); free(decoder->offsets); memset(decoder->uploads, 0, sizeof(decoder->uploads)); - decoder->texturePool = NULL; - decoder->offsets = NULL; - decoder->alignedWidth = 0; - decoder->alignedHeight = 0; - decoder->blockCapacity = 0; - decoder->referenceFrameID = 0; - decoder->uploadIndex = 0; - decoder->configured = false; - decoder->referenceValid = false; + decoder->texturePool = NULL; + decoder->offsets = NULL; + decoder->packetStates = NULL; + decoder->pendingPackets = NULL; + decoder->dirtyBlocks = NULL; + decoder->outputDirty = NULL; + decoder->outputTileGenerations = NULL; + decoder->alignedWidth = 0; + decoder->alignedHeight = 0; + decoder->blockCapacity = 0; + decoder->outputTileWidth = 0; + decoder->outputTileHeight = 0; + decoder->referenceFrameID = 0; + decoder->outputGeneration = 0; + decoder->uploadIndex = 0; + decoder->configured = false; + decoder->referenceValid = false; memset(decoder->coefficientTextures, 0, sizeof(decoder->coefficientTextures)); memset(decoder->planeTextures, 0, sizeof(decoder->planeTextures)); + memset(decoder->dirty, 0, sizeof(decoder->dirty)); memset(&decoder->config, 0, sizeof(decoder->config)); memset(decoder->blocks, 0, sizeof(decoder->blocks)); } @@ -511,18 +591,22 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) decoder->dequantProgram, "blockOffset"); decoder->uniforms.dequant.blockStride = glGetUniformLocation( decoder->dequantProgram, "blockStride"); + decoder->uniforms.dequant.workGroupOffset = glGetUniformLocation( + decoder->dequantProgram, "workGroupOffset"); decoder->uniforms.dequant.preserveMissing = glGetUniformLocation( decoder->dequantProgram, "preserveMissing"); - decoder->uniforms.idwt.coefficients = glGetUniformLocation( + decoder->uniforms.idwt.coefficients = glGetUniformLocation( decoder->idwtProgram, "coefficients"); - decoder->uniforms.idwt.resolution = glGetUniformLocation( + decoder->uniforms.idwt.resolution = glGetUniformLocation( decoder->idwtProgram, "resolution"); - decoder->uniforms.idwt.invResolution = glGetUniformLocation( + decoder->uniforms.idwt.invResolution = glGetUniformLocation( decoder->idwtProgram, "invResolution"); - decoder->uniforms.idwt.baseLayer = glGetUniformLocation( + decoder->uniforms.idwt.baseLayer = glGetUniformLocation( decoder->idwtProgram, "baseLayer"); - decoder->uniforms.idwt.dcShift = glGetUniformLocation( + decoder->uniforms.idwt.workGroupOffset = glGetUniformLocation( + decoder->idwtProgram, "workGroupOffset"); + decoder->uniforms.idwt.dcShift = glGetUniformLocation( decoder->idwtProgram, "dcShift"); decoder->uniforms.yuv.lumaPlane = glGetUniformLocation( @@ -533,6 +617,8 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) decoder->yuvProgram, "crPlane"); decoder->uniforms.yuv.outputSize = glGetUniformLocation( decoder->yuvProgram, "outputSize"); + decoder->uniforms.yuv.outputOffset = glGetUniformLocation( + decoder->yuvProgram, "outputOffset"); decoder->uniforms.yuv.bt2020 = glGetUniformLocation( decoder->yuvProgram, "bt2020"); decoder->uniforms.yuv.chromaLeft = glGetUniformLocation( @@ -551,15 +637,18 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) uniforms->dequant.outputLayer < 0 || uniforms->dequant.blockOffset < 0 || uniforms->dequant.blockStride < 0 || + uniforms->dequant.workGroupOffset < 0 || uniforms->dequant.preserveMissing < 0 || uniforms->idwt.coefficients < 0 || uniforms->idwt.resolution < 0 || uniforms->idwt.invResolution < 0 || uniforms->idwt.baseLayer < 0 || + uniforms->idwt.workGroupOffset < 0 || uniforms->idwt.dcShift < 0 || uniforms->yuv.lumaPlane < 0 || uniforms->yuv.cbPlane < 0 || uniforms->yuv.crPlane < 0 || - uniforms->yuv.outputSize < 0 || uniforms->yuv.bt2020 < 0 || + uniforms->yuv.outputSize < 0 || uniforms->yuv.outputOffset < 0 || + uniforms->yuv.bt2020 < 0 || uniforms->yuv.chromaLeft < 0 || uniforms->yuv.lumaOffset < 0 || uniforms->yuv.lumaScale < 0 || uniforms->yuv.chromaMidpoint < 0 || @@ -672,7 +761,7 @@ static bool networkVideoCreateTexture2D(GLuint * texture, GLenum format, return false; } -static GLuint networkVideoTexturePoolAcquire( +static struct NetworkVideoTexture networkVideoTexturePoolAcquire( struct NetworkVideoTexturePool * pool) { struct NetworkVideoTexture pooled = { 0 }; @@ -687,13 +776,12 @@ static GLuint networkVideoTexturePoolAcquire( if (pooled.readySync) glDeleteSync(pooled.readySync); if (pooled.texture) - return pooled.texture; + return pooled; - GLuint texture = 0; - if (!networkVideoCreateTexture2D(&texture, GL_RGBA16F, + if (!networkVideoCreateTexture2D(&pooled.texture, GL_RGBA16F, pool->width, pool->height, GL_LINEAR)) - return 0; - return texture; + pooled.texture = 0; + return pooled; } static bool networkVideoAllocateNL(NetworkVideoDecoder * decoder) @@ -736,12 +824,51 @@ static bool networkVideoAllocateNL(NetworkVideoDecoder * decoder) return false; } - decoder->offsets = malloc( + decoder->offsets = malloc( (size_t)decoder->blockCapacity * sizeof(*decoder->offsets)); - decoder->texturePool = networkVideoTexturePoolCreate(decoder); - if (!decoder->offsets || !decoder->texturePool) + decoder->packetStates = calloc( + decoder->blockCapacity, sizeof(*decoder->packetStates)); + decoder->pendingPackets = calloc( + decoder->blockCapacity, sizeof(*decoder->pendingPackets)); + decoder->dirtyBlocks = calloc( + decoder->blockCapacity, sizeof(*decoder->dirtyBlocks)); + decoder->texturePool = networkVideoTexturePoolCreate(decoder); + decoder->outputTileWidth = + (decoder->config.width + NETWORK_VIDEO_TILE_SIZE - 1U) / + NETWORK_VIDEO_TILE_SIZE; + decoder->outputTileHeight = + (decoder->config.height + NETWORK_VIDEO_TILE_SIZE - 1U) / + NETWORK_VIDEO_TILE_SIZE; + const size_t outputTileCount = + (size_t)decoder->outputTileWidth * decoder->outputTileHeight; + decoder->outputDirty = calloc( + outputTileCount, sizeof(*decoder->outputDirty)); + decoder->outputTileGenerations = calloc( + outputTileCount, sizeof(*decoder->outputTileGenerations)); + if (!decoder->offsets || !decoder->packetStates || + !decoder->pendingPackets || !decoder->dirtyBlocks || + !decoder->texturePool || !decoder->outputDirty || + !decoder->outputTileGenerations) return false; + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + { + for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level) + { + const uint32_t inputWidth = decoder->alignedWidth >> (level + 1U); + const uint32_t inputHeight = decoder->alignedHeight >> (level + 1U); + struct NetworkVideoDirtyGrid * grid = + &decoder->dirty[component][level]; + grid->width = (inputWidth + 15U) / 16U; + grid->height = (inputHeight + 15U) / 16U; + grid->tiles = calloc( + (size_t)grid->width * grid->height, sizeof(*grid->tiles)); + if (!grid->tiles) + return false; + } + } + for (unsigned index = 0; index < NETWORK_VIDEO_UPLOADS; ++index) { struct NetworkVideoUpload * upload = &decoder->uploads[index]; @@ -771,7 +898,7 @@ static void networkVideoExternalDelete( { struct NetworkVideoTexturePool * pool = frame->pool; const bool retained = networkVideoTexturePoolReturn( - pool, frame->texture, frame->readySync); + pool, frame->texture, frame->readySync, frame->generation); if (!retained) { struct NetworkVideoCurrent previous; @@ -1066,6 +1193,269 @@ static bool networkVideoConfigMatches( left->bitDepth == right->bitDepth && left->flags == right->flags; } +static bool networkVideoPacketMatches( + const struct NetworkVideoPacketState * state, + const uint8_t * packet, uint32_t length) +{ + if (!state->data || state->length != length || length < 8U) + return !state->data && !packet; + if (!packet || + networkVideoReadU32(state->data) != + (networkVideoReadU32(packet) & ~NETWORK_VIDEO_SEQUENCE_BITS)) + return false; + return memcmp(state->data + sizeof(uint32_t), + packet + sizeof(uint32_t), length - sizeof(uint32_t)) == 0; +} + +static void networkVideoDiscardPendingPacketsNL( + NetworkVideoDecoder * decoder) +{ + for (uint32_t index = 0; index < decoder->blockCapacity; ++index) + if (decoder->dirtyBlocks[index] && + decoder->pendingPackets[index].data != + decoder->packetStates[index].data) + free(decoder->pendingPackets[index].data); +} + +static void networkVideoCommitPendingPacketsNL( + NetworkVideoDecoder * decoder) +{ + for (uint32_t index = 0; index < decoder->blockCapacity; ++index) + { + if (!decoder->dirtyBlocks[index]) + continue; + free(decoder->packetStates[index].data); + decoder->packetStates[index] = decoder->pendingPackets[index]; + } +} + +static bool networkVideoStagePacketsNL(NetworkVideoDecoder * decoder, + const void * data, size_t size, bool update, uint32_t * changedCount) +{ + const uint8_t * bytes = data; + memcpy(decoder->pendingPackets, decoder->packetStates, + (size_t)decoder->blockCapacity * sizeof(*decoder->pendingPackets)); + memset(decoder->dirtyBlocks, 0, decoder->blockCapacity); + *changedCount = 0; + + for (uint32_t index = 0; index < decoder->blockCapacity; ++index) + { + const bool present = decoder->offsets[index] != UINT32_MAX; + if (update && !present) + continue; + + const uint8_t * packet = NULL; + uint32_t length = 0; + if (present) + { + const size_t offset = + (size_t)decoder->offsets[index] * sizeof(uint32_t); + if (offset > size - 8U) + goto fail; + const uint32_t first = networkVideoReadU32(bytes + offset); + length = ((first >> 16U) & UINT32_C(0x00000fff)) * + sizeof(uint32_t); + if (length < 8U || length > size - offset) + goto fail; + if (first & UINT32_C(0x0000ffff)) + packet = bytes + offset; + } + + if (networkVideoPacketMatches( + &decoder->packetStates[index], packet, length)) + { + if (update) + decoder->offsets[index] = UINT32_MAX; + continue; + } + + struct NetworkVideoPacketState pending = { 0 }; + if (packet) + { + pending.data = malloc(length); + if (!pending.data) + goto fail; + memcpy(pending.data, packet, length); + networkVideoWriteU32(pending.data, + networkVideoReadU32(pending.data) & ~NETWORK_VIDEO_SEQUENCE_BITS); + pending.length = length; + } + decoder->pendingPackets[index] = pending; + decoder->dirtyBlocks[index] = 1U; + ++*changedCount; + } + return true; + +fail: + networkVideoDiscardPendingPacketsNL(decoder); + return false; +} + +static void networkVideoClearDirtyNL(NetworkVideoDecoder * decoder) +{ + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + { + for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level) + { + struct NetworkVideoDirtyGrid * grid = + &decoder->dirty[component][level]; + memset(grid->tiles, 0, (size_t)grid->width * grid->height); + grid->count = 0; + } + } + memset(decoder->outputDirty, 0, + (size_t)decoder->outputTileWidth * decoder->outputTileHeight); +} + +static uint32_t networkVideoMirrorTexel( + int32_t coordinate, uint32_t length) +{ + const int32_t period = (int32_t)(2U * length); + int32_t wrapped = coordinate % period; + if (wrapped < 0) + wrapped += period; + return (uint32_t)wrapped < length ? (uint32_t)wrapped : + (uint32_t)(period - wrapped - 1); +} + +static bool networkVideoIDWTGroupReads(uint32_t group, uint32_t length, + uint32_t begin, uint32_t end) +{ + const int32_t base = (int32_t)(group * 16U) - 2; + const int32_t limit = (int32_t)length; + for (int32_t local = 0; local <= 18; local += 2) + { + const int32_t coordinate = base + local; + for (unsigned parity = 0; parity < 2U; ++parity) + { + int32_t first = coordinate; + if (parity && coordinate < 0) + --first; + else if (!parity && coordinate + 1 >= limit) + ++first; + for (int32_t sample = 0; sample < 2; ++sample) + { + const uint32_t texel = networkVideoMirrorTexel( + first + sample, length); + if (texel >= begin && texel < end) + return true; + } + } + } + return false; +} + +static void networkVideoMarkGridRect(struct NetworkVideoDirtyGrid * grid, + uint32_t inputWidth, uint32_t inputHeight, + uint32_t x, uint32_t y, uint32_t width, uint32_t height) +{ + if (!width || !height) + return; + + const uint32_t endX = x + width; + const uint32_t endY = y + height; + const uint32_t startX = x < 18U ? 0U : (x - 18U) / 16U + 1U; + const uint32_t startY = y < 18U ? 0U : (y - 18U) / 16U + 1U; + uint32_t groupEndX = (endX + 17U) / 16U < grid->width ? + (endX + 17U) / 16U : grid->width; + uint32_t groupEndY = (endY + 17U) / 16U < grid->height ? + (endY + 17U) / 16U : grid->height; + if (groupEndX < grid->width && networkVideoIDWTGroupReads( + grid->width - 1U, inputWidth, x, endX)) + groupEndX = grid->width; + if (groupEndY < grid->height && networkVideoIDWTGroupReads( + grid->height - 1U, inputHeight, y, endY)) + groupEndY = grid->height; + + for (uint32_t groupY = startY; groupY < groupEndY; ++groupY) + { + for (uint32_t groupX = startX; groupX < groupEndX; ++groupX) + { + uint8_t * dirty = &grid->tiles[ + (size_t)groupY * grid->width + groupX]; + if (!*dirty) + { + *dirty = 1U; + ++grid->count; + } + } + } +} + +static void networkVideoMarkOutputRect(NetworkVideoDecoder * decoder, + unsigned component, uint32_t x, uint32_t y, + uint32_t width, uint32_t height) +{ + uint32_t expansion = component ? 1U : 0U; + if (component && + decoder->config.chromaSubsampling == LG_NET_VIDEO_CHROMA_420) + { + x *= 2U; + y *= 2U; + width *= 2U; + height *= 2U; + expansion = 2U; + } + + const uint32_t startX = x > expansion ? x - expansion : 0U; + const uint32_t startY = y > expansion ? y - expansion : 0U; + const uint64_t rawEndX = (uint64_t)x + width + expansion; + const uint64_t rawEndY = (uint64_t)y + height + expansion; + const uint32_t endX = rawEndX < decoder->config.width ? + (uint32_t)rawEndX : decoder->config.width; + const uint32_t endY = rawEndY < decoder->config.height ? + (uint32_t)rawEndY : decoder->config.height; + if (startX >= endX || startY >= endY) + return; + + const uint32_t tileStartX = startX / NETWORK_VIDEO_TILE_SIZE; + const uint32_t tileStartY = startY / NETWORK_VIDEO_TILE_SIZE; + const uint32_t tileEndX = + (endX + NETWORK_VIDEO_TILE_SIZE - 1U) / NETWORK_VIDEO_TILE_SIZE; + const uint32_t tileEndY = + (endY + NETWORK_VIDEO_TILE_SIZE - 1U) / NETWORK_VIDEO_TILE_SIZE; + for (uint32_t tileY = tileStartY; tileY < tileEndY; ++tileY) + for (uint32_t tileX = tileStartX; tileX < tileEndX; ++tileX) + decoder->outputDirty[ + (size_t)tileY * decoder->outputTileWidth + tileX] = 1U; +} + +static void networkVideoPrepareDirtyNL(NetworkVideoDecoder * decoder) +{ + networkVideoClearDirtyNL(decoder); + for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level) + { + const uint32_t width = decoder->alignedWidth >> (level + 1U); + const uint32_t height = decoder->alignedHeight >> (level + 1U); + for (unsigned component = 0; + component < NETWORK_VIDEO_COMPONENTS; ++component) + { + if (!level && component && + decoder->config.chromaSubsampling == LG_NET_VIDEO_CHROMA_420) + continue; + const unsigned firstBand = + level == NETWORK_VIDEO_LEVELS - 1U ? 0U : 1U; + for (unsigned band = firstBand; band < NETWORK_VIDEO_BANDS; ++band) + { + const struct NetworkVideoBlockMeta * meta = + &decoder->blocks[component][level][band]; + for (uint32_t local = 0; local < meta->count; ++local) + { + if (!decoder->dirtyBlocks[meta->offset + local]) + continue; + const uint32_t x = local % meta->stride * 32U; + const uint32_t y = local / meta->stride * 32U; + networkVideoMarkGridRect(&decoder->dirty[component][level], + width, height, x, y, + x < width ? (width - x < 32U ? width - x : 32U) : 0U, + y < height ? (height - y < 32U ? height - y : 32U) : 0U); + } + } + } + } +} + static size_t networkVideoPayloadCapacity(size_t size) { size_t capacity = 64U * 1024U; @@ -1114,12 +1504,12 @@ static struct NetworkVideoUpload * networkVideoUploadNL( } static void networkVideoDequantNL(NetworkVideoDecoder * decoder, - const struct NetworkVideoUpload * upload, bool preserveMissing) + const struct NetworkVideoUpload * upload, bool update) { glUseProgram(decoder->dequantProgram); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, upload->offsetBuffer); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, upload->payloadBuffer); - glUniform1i(decoder->uniforms.dequant.preserveMissing, preserveMissing); + glUniform1i(decoder->uniforms.dequant.preserveMissing, update); for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level) { @@ -1145,7 +1535,34 @@ static void networkVideoDequantNL(NetworkVideoDecoder * decoder, component * NETWORK_VIDEO_BANDS + band); glUniform1i(decoder->uniforms.dequant.blockOffset, meta->offset); glUniform1i(decoder->uniforms.dequant.blockStride, meta->stride); - glDispatchCompute((width + 31U) / 32U, (height + 31U) / 32U, 1); + if (!update) + { + glUniform2i(decoder->uniforms.dequant.workGroupOffset, 0, 0); + glDispatchCompute( + (width + 31U) / 32U, (height + 31U) / 32U, 1); + continue; + } + + const uint32_t rows = meta->count / meta->stride; + for (uint32_t y = 0; y < rows; ++y) + { + uint32_t x = 0; + while (x < meta->stride) + { + const size_t row = (size_t)meta->offset + + (size_t)y * meta->stride; + while (x < meta->stride && !decoder->dirtyBlocks[row + x]) + ++x; + const uint32_t start = x; + while (x < meta->stride && decoder->dirtyBlocks[row + x]) + ++x; + if (start == x) + continue; + glUniform2i(decoder->uniforms.dequant.workGroupOffset, + (GLint)start, (GLint)y); + glDispatchCompute(x - start, 1, 1); + } + } } } } @@ -1153,7 +1570,37 @@ static void networkVideoDequantNL(NetworkVideoDecoder * decoder, GL_TEXTURE_FETCH_BARRIER_BIT); } -static void networkVideoIDWTNL(NetworkVideoDecoder * decoder) +static void networkVideoPropagateDirtyNL(NetworkVideoDecoder * decoder, + unsigned component, unsigned level, bool final) +{ + const struct NetworkVideoDirtyGrid * grid = + &decoder->dirty[component][level]; + const uint32_t outputWidth = decoder->alignedWidth >> level; + const uint32_t outputHeight = decoder->alignedHeight >> level; + for (uint32_t y = 0; y < grid->height; ++y) + { + for (uint32_t x = 0; x < grid->width; ++x) + { + if (!grid->tiles[(size_t)y * grid->width + x]) + continue; + const uint32_t outputX = x * 32U; + const uint32_t outputY = y * 32U; + const uint32_t width = outputX < outputWidth ? + (outputWidth - outputX < 32U ? outputWidth - outputX : 32U) : 0U; + const uint32_t height = outputY < outputHeight ? + (outputHeight - outputY < 32U ? outputHeight - outputY : 32U) : 0U; + if (final) + networkVideoMarkOutputRect( + decoder, component, outputX, outputY, width, height); + else + networkVideoMarkGridRect(&decoder->dirty[component][level - 1U], + outputWidth, outputHeight, + outputX, outputY, width, height); + } + } +} + +static void networkVideoIDWTNL(NetworkVideoDecoder * decoder, bool update) { glUseProgram(decoder->idwtProgram); glActiveTexture(GL_TEXTURE0); @@ -1180,6 +1627,11 @@ static void networkVideoIDWTNL(NetworkVideoDecoder * decoder) glUniform1i(decoder->uniforms.idwt.dcShift, final ? GL_TRUE : GL_FALSE); + struct NetworkVideoDirtyGrid * grid = + &decoder->dirty[component][level]; + if (update && !grid->count) + continue; + if (final) { glBindImageTexture(1, decoder->planeTextures[component], 0, @@ -1191,12 +1643,43 @@ static void networkVideoIDWTNL(NetworkVideoDecoder * decoder) GL_FALSE, component * NETWORK_VIDEO_BANDS, GL_WRITE_ONLY, GL_R32F); } - glDispatchCompute( - (inputHeight + 15U) / 16U, (inputWidth + 15U) / 16U, 1); + if (!update) + { + glUniform2i(decoder->uniforms.idwt.workGroupOffset, 0, 0); + glDispatchCompute( + (inputHeight + 15U) / 16U, (inputWidth + 15U) / 16U, 1); + } + else + { + for (uint32_t y = 0; y < grid->height; ++y) + { + uint32_t x = 0; + while (x < grid->width) + { + const size_t row = (size_t)y * grid->width; + while (x < grid->width && !grid->tiles[row + x]) + ++x; + const uint32_t start = x; + while (x < grid->width && grid->tiles[row + x]) + ++x; + if (start == x) + continue; + glUniform2i(decoder->uniforms.idwt.workGroupOffset, + (GLint)y, (GLint)start); + glDispatchCompute(1, x - start, 1); + } + } + networkVideoPropagateDirtyNL( + decoder, component, (unsigned)level, final); + } glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT | GL_TEXTURE_FETCH_BARRIER_BIT); } } + + if (!update) + memset(decoder->outputDirty, 1, + (size_t)decoder->outputTileWidth * decoder->outputTileHeight); } static void networkVideoColorRange( @@ -1233,11 +1716,118 @@ static void networkVideoColorRange( } } -static struct NetworkVideoExternalFrame * networkVideoConvertNL( - NetworkVideoDecoder * decoder, bool chromaLeft) +static bool networkVideoTextureTileDirty( + const NetworkVideoDecoder * decoder, size_t index, + uint64_t textureGeneration) { - GLuint texture = networkVideoTexturePoolAcquire(decoder->texturePool); - if (!texture) + return decoder->outputDirty[index] || + decoder->outputTileGenerations[index] > textureGeneration; +} + +static void networkVideoBuildDamageNL(NetworkVideoDecoder * decoder, + NetworkVideoDecodedFrame * result) +{ + uint32_t count = 0; + bool overflow = false; + for (uint32_t tileY = 0; + tileY < decoder->outputTileHeight && !overflow; ++tileY) + { + uint32_t tileX = 0; + while (tileX < decoder->outputTileWidth) + { + const size_t row = (size_t)tileY * decoder->outputTileWidth; + while (tileX < decoder->outputTileWidth && + !decoder->outputDirty[row + tileX]) + ++tileX; + const uint32_t start = tileX; + while (tileX < decoder->outputTileWidth && + decoder->outputDirty[row + tileX]) + ++tileX; + if (start == tileX) + continue; + + const uint32_t x = start * NETWORK_VIDEO_TILE_SIZE; + const uint32_t y = tileY * NETWORK_VIDEO_TILE_SIZE; + const uint32_t endX = tileX * NETWORK_VIDEO_TILE_SIZE < + decoder->config.width ? + tileX * NETWORK_VIDEO_TILE_SIZE : decoder->config.width; + const uint32_t endY = y + NETWORK_VIDEO_TILE_SIZE < + decoder->config.height ? + y + NETWORK_VIDEO_TILE_SIZE : decoder->config.height; + bool merged = false; + for (uint32_t index = 0; index < count; ++index) + { + KVMFRFrameDamageRect * rect = &result->damageRects[index]; + if (rect->x == x && rect->width == endX - x && + rect->y + rect->height == y) + { + rect->height += endY - y; + merged = true; + break; + } + } + if (merged) + continue; + if (count + 1U >= LG_TRANSPORT_MAX_DAMAGE_RECTS) + { + overflow = true; + break; + } + result->damageRects[count++] = (KVMFRFrameDamageRect) + { + .x = x, + .y = y, + .width = endX - x, + .height = endY - y, + }; + } + } + + if (!overflow) + { + result->damageRectsCount = count; + return; + } + + uint32_t minX = decoder->config.width; + uint32_t minY = decoder->config.height; + uint32_t maxX = 0; + uint32_t maxY = 0; + for (uint32_t tileY = 0; tileY < decoder->outputTileHeight; ++tileY) + { + for (uint32_t tileX = 0; tileX < decoder->outputTileWidth; ++tileX) + { + if (!decoder->outputDirty[ + (size_t)tileY * decoder->outputTileWidth + tileX]) + continue; + const uint32_t x = tileX * NETWORK_VIDEO_TILE_SIZE; + const uint32_t y = tileY * NETWORK_VIDEO_TILE_SIZE; + minX = x < minX ? x : minX; + minY = y < minY ? y : minY; + maxX = x + NETWORK_VIDEO_TILE_SIZE > maxX ? + x + NETWORK_VIDEO_TILE_SIZE : maxX; + maxY = y + NETWORK_VIDEO_TILE_SIZE > maxY ? + y + NETWORK_VIDEO_TILE_SIZE : maxY; + } + } + maxX = maxX < decoder->config.width ? maxX : decoder->config.width; + maxY = maxY < decoder->config.height ? maxY : decoder->config.height; + result->damageRects[0] = (KVMFRFrameDamageRect) + { + .x = minX, + .y = minY, + .width = maxX - minX, + .height = maxY - minY, + }; + result->damageRectsCount = 1; +} + +static struct NetworkVideoExternalFrame * networkVideoConvertNL( + NetworkVideoDecoder * decoder, bool chromaLeft, uint64_t generation) +{ + struct NetworkVideoTexture pooled = + networkVideoTexturePoolAcquire(decoder->texturePool); + if (!pooled.texture) return NULL; glUseProgram(decoder->yuvProgram); @@ -1262,10 +1852,43 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL( glUniform1f(decoder->uniforms.yuv.lumaScale, lumaScale); glUniform1f(decoder->uniforms.yuv.chromaMidpoint, chromaMidpoint); glUniform1f(decoder->uniforms.yuv.chromaScale, chromaScale); - glBindImageTexture(0, texture, 0, GL_FALSE, 0, + glBindImageTexture(0, pooled.texture, 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGBA16F); - glDispatchCompute((decoder->config.width + 7U) / 8U, - (decoder->config.height + 7U) / 8U, 1); + if (!pooled.generation) + { + glUniform2i(decoder->uniforms.yuv.outputOffset, 0, 0); + glDispatchCompute((decoder->config.width + 7U) / 8U, + (decoder->config.height + 7U) / 8U, 1); + } + else + { + for (uint32_t tileY = 0; tileY < decoder->outputTileHeight; ++tileY) + { + uint32_t tileX = 0; + while (tileX < decoder->outputTileWidth) + { + const size_t row = (size_t)tileY * decoder->outputTileWidth; + while (tileX < decoder->outputTileWidth && + !networkVideoTextureTileDirty( + decoder, row + tileX, pooled.generation)) + ++tileX; + const uint32_t start = tileX; + while (tileX < decoder->outputTileWidth && + networkVideoTextureTileDirty( + decoder, row + tileX, pooled.generation)) + ++tileX; + if (start == tileX) + continue; + const uint32_t pixelWidth = + (tileX - start) * NETWORK_VIDEO_TILE_SIZE; + glUniform2i(decoder->uniforms.yuv.outputOffset, + (GLint)(start * NETWORK_VIDEO_TILE_SIZE), + (GLint)(tileY * NETWORK_VIDEO_TILE_SIZE)); + glDispatchCompute((pixelWidth + 7U) / 8U, + NETWORK_VIDEO_TILE_SIZE / 8U, 1); + } + } + } glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT | GL_TEXTURE_FETCH_BARRIER_BIT); @@ -1274,7 +1897,7 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL( if (!readySync || glGetError() != GL_NO_ERROR) { networkVideoTexturePoolRecycleCurrent( - decoder->texturePool, texture, readySync); + decoder->texturePool, pooled.texture, readySync, 0); return NULL; } @@ -1282,13 +1905,14 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL( if (!frame) { networkVideoTexturePoolRecycleCurrent( - decoder->texturePool, texture, readySync); + decoder->texturePool, pooled.texture, readySync, 0); return NULL; } - frame->base.ops = &NETWORK_VIDEO_EXTERNAL_OPS; - frame->pool = decoder->texturePool; - frame->texture = texture; - frame->readySync = readySync; + frame->base.ops = &NETWORK_VIDEO_EXTERNAL_OPS; + frame->pool = decoder->texturePool; + frame->texture = pooled.texture; + frame->readySync = readySync; + frame->generation = generation; networkVideoTexturePoolRetain(frame->pool); atomic_init(&frame->references, 1U); return frame; @@ -1314,7 +1938,8 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( if ((!update && baseFrameID) || (update && !complete) || (update && (!decoder->referenceValid || - decoder->referenceFrameID != baseFrameID || frameID <= baseFrameID))) + !baseFrameID || baseFrameID > decoder->referenceFrameID || + frameID <= decoder->referenceFrameID))) { LG_UNLOCK(decoder->lock); return NETWORK_VIDEO_DECODE_CORRUPT; @@ -1331,9 +1956,30 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( NETWORK_VIDEO_DECODE_UNSUPPORTED : NETWORK_VIDEO_DECODE_CORRUPT; } + uint32_t changedCount = 0; + if (!networkVideoStagePacketsNL( + decoder, data, size, update, &changedCount)) + { + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_ERROR; + } + if (update && !changedCount) + { + networkVideoCommitPendingPacketsNL(decoder); + decoder->referenceFrameID = frameID; + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_NO_FRAME; + } + + if (update) + networkVideoPrepareDirtyNL(decoder); + else + networkVideoClearDirtyNL(decoder); + struct NetworkVideoCurrent previous; if (!networkVideoMakeCurrent(decoder, &previous)) { + networkVideoDiscardPendingPacketsNL(decoder); LG_UNLOCK(decoder->lock); return NETWORK_VIDEO_DECODE_ERROR; } @@ -1345,22 +1991,27 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( decoder, data, size); if (!upload) { + networkVideoDiscardPendingPacketsNL(decoder); networkVideoRestoreCurrent(decoder, &previous); LG_UNLOCK(decoder->lock); return NETWORK_VIDEO_DECODE_ERROR; } const uint64_t decodeStart = nanotime(); + uint64_t generation = decoder->outputGeneration + 1U; + if (!generation) + generation = 1U; networkVideoDequantNL(decoder, upload, update); upload->readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); - networkVideoIDWTNL(decoder); + networkVideoIDWTNL(decoder, update); struct NetworkVideoExternalFrame * external = - networkVideoConvertNL(decoder, layout.chromaLeft); + networkVideoConvertNL(decoder, layout.chromaLeft, generation); const uint64_t decodeTime = nanotime() - decodeStart; const bool success = external && glGetError() == GL_NO_ERROR; networkVideoRestoreCurrent(decoder, &previous); if (!success) { + networkVideoDiscardPendingPacketsNL(decoder); decoder->referenceFrameID = 0; decoder->referenceValid = false; if (external) @@ -1369,11 +2020,15 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( return NETWORK_VIDEO_DECODE_ERROR; } - if (!update) - { - decoder->referenceFrameID = complete ? frameID : 0; - decoder->referenceValid = complete; - } + networkVideoCommitPendingPacketsNL(decoder); + const size_t outputTileCount = + (size_t)decoder->outputTileWidth * decoder->outputTileHeight; + for (size_t index = 0; index < outputTileCount; ++index) + if (decoder->outputDirty[index]) + decoder->outputTileGenerations[index] = generation; + decoder->outputGeneration = generation; + decoder->referenceFrameID = complete ? frameID : 0; + decoder->referenceValid = complete; uint32_t formatVersion = (uint32_t)config->configEpoch; if (!formatVersion) @@ -1381,6 +2036,8 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( result->externalFrame = &external->base; result->decodeTimeNs = decodeTime; + if (update) + networkVideoBuildDamageNL(decoder, result); result->format = (LG_TransportFrameFormat) { .version = formatVersion, diff --git a/client/transports/Network/video/shaders.h b/client/transports/Network/video/shaders.h index 80ba9f2f..fb0d70f8 100644 --- a/client/transports/Network/video/shaders.h +++ b/client/transports/Network/video/shaders.h @@ -55,6 +55,7 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_0[] = "layout(std430,binding=0) readonly buffer OffsetBuffer { uint offsets[]; };\n" "layout(std430,binding=1) readonly buffer PayloadBuffer { uint payload[]; };\n" "uniform ivec2 resolution;\n" + "uniform ivec2 workGroupOffset;\n" "uniform int outputLayer;\n" "uniform int blockOffset;\n" "uniform int blockStride;\n" @@ -88,21 +89,22 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_0[] = " }\n" "}\n" "void main() {\n" - " uint localIndex = gl_LocalInvocationIndex;\n" - " int blockIndex = blockOffset + int(gl_WorkGroupID.y) * blockStride +\n" - " int(gl_WorkGroupID.x);\n" - " uint sourceWord = offsets[blockIndex];\n" - " uint blockPresent = uint(sourceWord != 0xffffffffu);\n" + " uint localIndex = gl_LocalInvocationIndex;\n" + " ivec2 logicalGroup = ivec2(gl_WorkGroupID.xy) + workGroupOffset;\n" + " int blockIndex = blockOffset + logicalGroup.y * blockStride +\n" + " logicalGroup.x;\n" + " uint sourceWord = offsets[blockIndex];\n" + " uint blockPresent = uint(sourceWord != 0xffffffffu);\n" " // Missing blocks must still reach every workgroup barrier.\n" " sourceWord *= blockPresent;\n" - " uint subblock = bitfieldExtract(localIndex,0,3);\n" - " uint blockX = bitfieldExtract(localIndex,3,2);\n" - " uint blockY = bitfieldExtract(localIndex,5,2);\n" - " uint group = blockY * 4u + blockX;\n" - " ivec2 coord = ivec2(gl_WorkGroupID.xy) * 32 +\n" + " uint subblock = bitfieldExtract(localIndex,0,3);\n" + " uint blockX = bitfieldExtract(localIndex,3,2);\n" + " uint blockY = bitfieldExtract(localIndex,5,2);\n" + " uint group = blockY * 4u + blockX;\n" + " ivec2 coord = logicalGroup * 32 +\n" " 8 * ivec2(blockX,blockY) + unswizzle8x8(subblock << 3u);\n" - " uint sourceByte = sourceWord * 4u;\n" - " uint ballot = (payload[sourceWord] & 65535u) * blockPresent;\n" + " uint sourceByte = sourceWord * 4u;\n" + " uint ballot = (payload[sourceWord] & 65535u) * blockPresent;\n" " if (localIndex == 0u) {\n" " uint cursor = sourceByte + 8u + 3u * uint(bitCount(ballot));\n" " for (uint index=0u; index<16u; ++index) {\n" @@ -192,6 +194,7 @@ static const char NETWORK_VIDEO_IDWT_SHADER_0[] = "layout(binding=0) uniform highp sampler2DArray coefficients;\n" "layout(r32f,binding=1) writeonly uniform highp image2D outputImage;\n" "uniform ivec2 resolution;\n" + "uniform ivec2 workGroupOffset;\n" "uniform vec2 invResolution;\n" "uniform int baseLayer;\n" "uniform bool dcShift;\n" @@ -230,8 +233,10 @@ static const char NETWORK_VIDEO_IDWT_SHADER_0[] = " sharedBlock[coord.y+1][2*coord.x+3]=vec2(b.w,d.w);\n" "}\n" "void loadApron(uint index) {\n" - " ivec2 base=ivec2(gl_WorkGroupID.xy)*HALF_BLOCK-HALF_APRON;\n" - " ivec2 local=2*unswizzle8x8(index); ivec2 coord=base+local;\n" + " ivec2 workGroup = ivec2(gl_WorkGroupID.xy)+workGroupOffset;\n" + " ivec2 base = workGroup*HALF_BLOCK-HALF_APRON;\n" + " ivec2 local = 2*unswizzle8x8(index);\n" + " ivec2 coord = base+local;\n" " store4(local,gatherBand(coord,true,true,0).wxzy,\n" " gatherBand(coord,false,true,2).wxzy,\n" " gatherBand(coord,true,false,1).wxzy,\n" @@ -280,11 +285,13 @@ static const char NETWORK_VIDEO_IDWT_SHADER_1[] = "}\n" "void main(){uint index=gl_LocalInvocationIndex;loadApron(index);\n" " inverse8x2(index);inverse4x2(index,index<32u,HALF_BLOCK);barrier();\n" - " inverse8x2(index);barrier();ivec2 local=unswizzle8x8(index);\n" - " ivec2 outputSize=imageSize(outputImage);\n" + " inverse8x2(index);barrier();\n" + " ivec2 local = unswizzle8x8(index);\n" + " ivec2 outputSize = imageSize(outputImage);\n" + " ivec2 workGroup = ivec2(gl_WorkGroupID.xy)+workGroupOffset;\n" " for(int y=local.y;y> 1u; const uint chromaHeight = Dim0.w >> 1u; - if (tid.x >= chromaWidth || tid.y >= chromaHeight) + if (pixel.x >= chromaWidth || pixel.y >= chromaHeight) return; const uint2 chromaLimit = Dim0.xy >> 1u; - const uint2 chromaCoord = min(tid.xy, chromaLimit - 1u); + const uint2 chromaCoord = min(pixel, chromaLimit - 1u); const int2 chromaSrc = int2(chromaCoord * 2u); const float3 a = loadSourceYCbCr(chromaSrc, bt2020); const float3 b = loadSourceYCbCr( @@ -469,21 +521,21 @@ void ColorMain(uint3 tid : SV_DispatchThreadID) float3 yb = b; float3 yc = c; float3 yd = d; - const int2 lumaSrc = int2(tid.xy * 2u); - if (any(tid.xy != chromaCoord)) + const int2 lumaSrc = int2(pixel * 2u); + if (any(pixel != chromaCoord)) { ya = loadSourceYCbCr(lumaSrc, bt2020); yb = loadSourceYCbCr(lumaSrc + int2(1, 0), bt2020); yc = loadSourceYCbCr(lumaSrc + int2(0, 1), bt2020); yd = loadSourceYCbCr(lumaSrc + int2(1, 1), bt2020); } - const uint y0 = (tid.y * 2u) * Dim0.z + tid.x * 2u; + const uint y0 = (pixel.y * 2u) * Dim0.z + pixel.x * 2u; const uint y1 = y0 + Dim0.z; Planes[Op0.x + y0] = lumaValue(ya.x); Planes[Op0.x + y0 + 1u] = lumaValue(yb.x); Planes[Op0.x + y1] = lumaValue(yc.x); Planes[Op0.x + y1 + 1u] = lumaValue(yd.x); - const uint chroma = tid.y * chromaWidth + tid.x; + const uint chroma = pixel.y * chromaWidth + pixel.x; Planes[Op0.y + chroma] = chromaValue( (a.y + b.y + c.y + d.y) * 0.25); Planes[Op0.z + chroma] = chromaValue( @@ -491,10 +543,10 @@ void ColorMain(uint3 tid : SV_DispatchThreadID) return; } - if (tid.x >= Dim0.z || tid.y >= Dim0.w) + if (pixel.x >= Dim0.z || pixel.y >= Dim0.w) return; - const float3 yuv = loadSourceYCbCr(int2(tid.xy), bt2020); - const uint index = tid.y * Dim0.z + tid.x; + const float3 yuv = loadSourceYCbCr(int2(pixel), bt2020); + const uint index = pixel.y * Dim0.z + pixel.x; Planes[Op0.x + index] = lumaValue(yuv.x); Planes[Op0.y + index] = chromaValue(yuv.y); Planes[Op0.z + index] = chromaValue(yuv.z); @@ -513,12 +565,12 @@ uint2 unswizzle8x8(uint index) float loadPlaneValue(int2 coord) { - const uint x = mirrorCoord(coord.x, Op0.x); - const uint y = mirrorCoord(coord.y, Op0.y); - const uint index = Op1.w + y * Dim0.z + x; + const uint x = mirrorCoord(coord.x, Op0.x); + const uint y = mirrorCoord(coord.y, Op0.y); if (Op1.x == 0u) - return Planes[index]; - return Op1.y != 0u ? Scratch[index] : Wave[index]; + return Planes[Op1.w + y * Dim0.z + x]; + const uint index = Op1.y + y * Dim0.z + x; + return (Op1.x & 1u) != 0u ? Wave[index] : Scratch[index]; } void storeDWTValue(uint x, uint y, float value, bool low) @@ -526,13 +578,17 @@ void storeDWTValue(uint x, uint y, float value, bool low) if (x >= Op0.x || y >= Op0.y) return; const uint index = Op1.w + y * Dim0.z + x; - Atlas[index] = value; + // Intermediate low-pass values feed the next level while their atlas + // region retains the deeper-level coefficients from the prior frame. + if (!low || Work.z != 0u) + Atlas[index] = value; if (!low) return; - if (Op1.z != 0u) - Scratch[index] = value; + const uint lowIndex = Op1.z + y * Dim0.z + x; + if ((Op1.x & 1u) != 0u) + Scratch[lowIndex] = value; else - Wave[index] = value; + Wave[lowIndex] = value; } float2 loadDWTShared(uint y, uint x) @@ -642,25 +698,26 @@ void forwardTransform4x2(uint lane, bool active, uint yOffset) [numthreads(64, 1, 1)] void DWTMain(uint3 groupID : SV_GroupID, uint lane : SV_GroupIndex) { - const uint2 local = 2u * unswizzle8x8(lane); - loadDWTQuad(groupID.xy, local); - loadDWTQuad(groupID.xy, local + uint2(16u, 0u)); - loadDWTQuad(groupID.xy, local + uint2(0u, 16u)); - loadDWTQuad(groupID.xy, local + uint2(16u, 16u)); + const uint2 workGroup = groupID.xy + Work.xy; + const uint2 local = 2u * unswizzle8x8(lane); + loadDWTQuad(workGroup, local); + loadDWTQuad(workGroup, local + uint2(16u, 0u)); + loadDWTQuad(workGroup, local + uint2(0u, 16u)); + loadDWTQuad(workGroup, local + uint2(16u, 16u)); const uint2 right = uint2(32u + 2u * (lane & 3u), 2u * (lane >> 2u)); - loadDWTQuad(groupID.xy, right); + loadDWTQuad(workGroup, right); const uint2 bottom = uint2(2u * (lane & 15u), 32u + 2u * (lane >> 4u)); - loadDWTQuad(groupID.xy, bottom); + loadDWTQuad(workGroup, bottom); if (lane < 16u) { const uint2 corner = uint2(32u + 2u * (lane & 3u), 32u + 2u * (lane >> 2u)); - loadDWTQuad(groupID.xy, corner); + loadDWTQuad(workGroup, corner); } GroupMemoryBarrierWithGroupSync(); @@ -680,8 +737,8 @@ void DWTMain(uint3 groupID : SV_GroupID, uint lane : SV_GroupIndex) { const float2 v0 = loadDWTShared(y, x); const float2 v1 = loadDWTShared(y, x + 1u); - const uint outX = groupID.x * 16u + (x >> 1u); - const uint outY = groupID.y * 16u + y; + const uint outX = workGroup.x * 16u + (x >> 1u); + const uint outY = workGroup.y * 16u + y; if (outX < bandWidth && outY < bandHeight) { storeDWTValue(outX, outY, v0.x, true); @@ -735,9 +792,9 @@ void ClearMain(uint3 tid : SV_DispatchThreadID) if (tid.x == 0u) QuantPayload[0] = 1u; if (tid.x < Dim0.x) - Aux[AuxBase.w + RDOStateWords + tid.x] = 0u; - if (tid.x < Dim0.y) Aux[AuxBase.w + tid.x] = 0u; + if (Work.x != 0u && tid.x < Dim0.y) + QuantPayload[State.x + tid.x] = 0u; } )hlsl" R"hlsl( @@ -755,15 +812,16 @@ groupshared uint QuantPayloadValid[16]; [numthreads(128, 1, 1)] void QuantMain(uint3 groupID : SV_GroupID, uint lane : SV_GroupIndex) { - const uint block = lane >> 3u; - const uint subblock = lane & 7u; - const uint blockX = groupID.x * 4u + (block & 3u); - const uint blockY = groupID.y * 4u + (block >> 2u); - const bool valid = blockX < Op0.z && blockY < Op0.w; - const uint baseX = blockX * 8u + (subblock >> 2u) * 4u; - const uint baseY = blockY * 8u + (subblock & 3u) * 2u; - float values [8]; - float localMax = 0.0; + const uint2 workGroup = groupID.xy + Work.xy; + const uint block = lane >> 3u; + const uint subblock = lane & 7u; + const uint blockX = workGroup.x * 4u + (block & 3u); + const uint blockY = workGroup.y * 4u + (block >> 2u); + const bool valid = blockX < Op0.z && blockY < Op0.w; + const uint baseX = blockX * 8u + (subblock >> 2u) * 4u; + const uint baseY = blockY * 8u + (subblock & 3u) * 2u; + float values [8]; + float localMax = 0.0; for (uint x = 0u; x < 4u; ++x) { const uint xCoord = baseX + x; @@ -972,12 +1030,13 @@ groupshared float AnalyzeDistortion [QuantLevels]; [numthreads(64, 1, 1)] void AnalyzeMain(uint3 groupID : SV_GroupID, uint lane : SV_GroupIndex) { - const uint blockSlot = lane & 15u; - const uint quantGroup = lane >> 4u; - const uint blockX = groupID.x * 4u + (blockSlot & 3u); - const uint blockY = groupID.y * 4u + (blockSlot >> 2u); - const bool valid = blockX < Op0.z && blockY < Op0.w; - const uint blockIndex8 = Op1.z + blockY * Op0.z + blockX; + const uint2 workGroup = groupID.xy + Work.xy; + const uint blockSlot = lane & 15u; + const uint quantGroup = lane >> 4u; + const uint blockX = workGroup.x * 4u + (blockSlot & 3u); + const uint blockY = workGroup.y * 4u + (blockSlot >> 2u); + const bool valid = blockX < Op0.z && blockY < Op0.w; + const uint blockIndex8 = Op1.z + blockY * Op0.z + blockX; for (uint round = 0u; round < (QuantLevels + 3u) / 4u; ++round) { @@ -1017,12 +1076,14 @@ void AnalyzeMain(uint3 groupID : SV_GroupID, uint lane : SV_GroupIndex) if (lane == 0u) { - const uint blockIndex = Op1.y + groupID.y * Op1.x + groupID.x; + const uint blockIndex = Op1.y + workGroup.y * Op1.x + workGroup.x; const uint perDivision = max(uint(Values.w), 1u); const uint division = min( blockIndex / perDivision, RDODivisions - 1u); uint ignored; InterlockedAdd(Aux[AuxBase.w], AnalyzeCost[0], ignored); + InterlockedAdd(Aux[AuxBase.w + 2u], + QuantPayload[State.x + blockIndex + 1u], ignored); uint previous = 0u; for (uint quant = 1u; quant < QuantLevels; ++quant) @@ -1077,18 +1138,23 @@ void FinalizeRDOMain(uint lane : SV_GroupIndex) [numthreads(64, 1, 1)] void ResolveMain(uint3 tid : SV_DispatchThreadID) { - const uint blockIndex = tid.x; - if (blockIndex >= Dim0.y) + if (tid.x >= Work.y) return; + const uint blockIndex = Work.x + tid.x; - const uint consumed = Aux[AuxBase.w]; - if (consumed <= Dim0.x) + const uint canonicalTotal = QuantPayload[State.x]; + const uint oldDirty = Aux[AuxBase.w + 2u]; + const uint fixed = canonicalTotal >= oldDirty ? + canonicalTotal - oldDirty : 0u; + const uint consumed = fixed + Aux[AuxBase.w]; + const uint target = min(Work.z, fixed + Work.w); + if (consumed <= target) { Aux[AuxBase.w + RDOStateWords + blockIndex] = 0u; return; } - const uint required = consumed - Dim0.x; + const uint required = consumed - target; const uint subdivision = min( blockIndex / max(Dim0.z, 1u), RDODivisions - 1u); uint selected = 0u; @@ -1206,12 +1272,14 @@ uint quantPayloadSubblockOffset( [numthreads(64, 1, 1)] void PackMain(uint3 groupID : SV_GroupID, uint lane : SV_GroupIndex) { + const uint2 workGroup = groupID.xy + Work.xy; const uint record = lane >> 4u; const uint block = lane & 15u; - const uint packetX = groupID.x * 2u + (record & 1u); - const uint packetY = groupID.y * 2u + (record >> 1u); + const uint packetX = workGroup.x * 2u + (record & 1u); + const uint packetY = workGroup.y * 2u + (record >> 1u); const uint blocksY = (Dim0.w + 31u) / 32u; - const bool packetLive = packetX < Op1.x && packetY < blocksY; + const bool packetLive = (Work.z & (1u << record)) != 0u && + packetX < Op1.x && packetY < blocksY; const uint blockIndex = Op1.y + packetY * Op1.x + packetX; const uint quant = packetLive ? min( Aux[AuxBase.w + RDOStateWords + blockIndex], QuantLevels - 1u) : 0u; @@ -1313,6 +1381,19 @@ void PackMain(uint3 groupID : SV_GroupID, uint lane : SV_GroupIndex) } GroupMemoryBarrierWithGroupSync(); + if (block == 0u && packetLive && + (PackBallot[record] == 0u || PackValid[record] != 0u)) + { + const uint packetWords = PackBallot[record] != 0u ? + PackSize[record] / 4u : 0u; + uint oldWords; + uint ignored; + InterlockedExchange( + QuantPayload[State.x + blockIndex + 1u], packetWords, oldWords); + InterlockedAdd(QuantPayload[State.x], packetWords - oldWords, ignored); + } + GroupMemoryBarrierWithGroupSync(); + const uint ballot = PackBallot[record]; const bool active = PackValid[record] != 0u && (ballot & (1u << block)) != 0u; @@ -1386,17 +1467,22 @@ void PackMain(uint3 groupID : SV_GroupID, uint lane : SV_GroupIndex) void FinalizeMain() { uint packetCount = PackState.Load(4u); - if (packetCount == 0u) + if (packetCount == 0u && QuantPayload[State.x] == 0u) { - const uint packetWord0 = 1u | (3u << 16u) | (Op0.x << 28u); - Packed.Store(8u, packetWord0); - Packed.Store(12u, Op0.w); - Packed.Store(16u, 6u << 20u); - Aux[AuxBase.z] = 8u; - Aux[AuxBase.z + 1u] = 12u; - PackState.Store(0u, 12u); - PackState.Store(4u, 1u); - packetCount = 1u; + QuantPayload[State.x] = 3u; + QuantPayload[State.x + 1u] = 3u; + if (Work.x != 0u) + { + const uint packetWord0 = 1u | (3u << 16u) | (Op0.x << 28u); + Packed.Store(8u, packetWord0); + Packed.Store(12u, Op0.w); + Packed.Store(16u, 6u << 20u); + Aux[AuxBase.z] = 8u; + Aux[AuxBase.z + 1u] = 12u; + PackState.Store(0u, 12u); + PackState.Store(4u, 1u); + packetCount = 1u; + } } const uint hdr = Op0.z != 0u ? 1u : 0u; const uint word0 = (Dim0.x - 1u) | @@ -1424,6 +1510,8 @@ void FinalizeMain() uint64_t captureTimeNs = 0; uint32_t deadlineMs = 0; uint32_t blockCapacity = 0; + uint32_t hardLimit = 0; + std::vector damagedBlocks; std::vector data; std::vector blocks; std::shared_ptr> wholePayload; @@ -1469,10 +1557,12 @@ void FinalizeMain() uint32_t op0 [4] = {}; uint32_t op1 [4] = {}; uint32_t auxBase[4] = {}; + uint32_t work [4] = {}; + uint32_t state [4] = {}; float values [4] = {}; }; - static_assert(sizeof(ShaderParameters) == 20U * sizeof(uint32_t), + static_assert(sizeof(ShaderParameters) == 28U * sizeof(uint32_t), "Unexpected shader parameter layout"); constexpr UINT SHADER_PARAMETER_DWORDS = static_cast( sizeof(ShaderParameters) / sizeof(uint32_t)); @@ -1496,34 +1586,82 @@ void FinalizeMain() struct Plane { + uint32_t width = 0; + uint32_t height = 0; + uint32_t levels = 0; + uint32_t offset = 0; + uint32_t lowpassOffset[PYROWAVE_LEVELS] = {}; + }; + + struct PlaneRect + { + uint32_t left = 0; + uint32_t top = 0; + uint32_t right = 0; + uint32_t bottom = 0; + }; + + struct GroupRange + { + uint32_t begin = 0; + uint32_t end = 0; + }; + + struct RowRun + { + uint32_t x = 0; + uint32_t y = 0; uint32_t width = 0; uint32_t height = 0; - uint32_t levels = 0; - uint32_t offset = 0; + }; + + struct PackRun + { + uint32_t x = 0; + uint32_t y = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t activeMask = 0; + }; + + struct BandWork + { + uint32_t blockBase = 0; + uint32_t block8Base = 0; + uint32_t blocksX = 0; + uint32_t blocksY = 0; + std::vector blockRuns; + std::vector packRuns; + }; + + struct FrameWork + { + std::vector damagedBlocks; + std::vector colorRuns; + std::vector dwtRuns[3][PYROWAVE_LEVELS]; + std::vector bands; + bool full = false; }; struct Slot { ComPtr descriptors; ComPtr clearDescriptors; - ComPtr wave; - ComPtr scratch; - ComPtr atlas; - ComPtr planes; ComPtr packed; ComPtr packState; ComPtr aux; - ComPtr quantPayload; ComPtr readback; D12Sync computeSync; D12Sync sync; TexLease lease; FrameIn frame; + FrameWork work; const uint8_t * mapped = nullptr; uint64_t captureTime = 0; uint64_t order = 0; uint32_t sequence = 0; uint32_t frameLimit = 0; + uint32_t hardLimit = 0; bool inFlight = false; bool used = false; }; @@ -1533,6 +1671,11 @@ void FinalizeMain() CD3D12CommandQueue m_copyQueue; ComPtr m_rootSignature; ComPtr m_pipelines[PIPELINE_COUNT]; + ComPtr m_wave; + ComPtr m_scratch; + ComPtr m_atlas; + ComPtr m_planeData; + ComPtr m_quantPayload; Slot m_slots [ENCODER_SLOT_COUNT]; FrameCfg m_cfg; Plane m_planes [3] = {}; @@ -1548,6 +1691,8 @@ void FinalizeMain() uint32_t m_auxWords = 0; uint32_t m_payloadWords = 0; uint32_t m_sequence = 0; + uint32_t m_hardLimit = 0; + bool m_persistentValid = false; bool m_ready = false; HANDLE m_wakeEvent = nullptr; @@ -1761,25 +1906,25 @@ void FinalizeMain() { ID3D12Resource * resources[] = { - slot.wave.Get(), - slot.scratch.Get(), - slot.atlas.Get(), - slot.planes.Get(), + m_wave.Get(), + m_scratch.Get(), + m_atlas.Get(), + m_planeData.Get(), slot.packed.Get(), slot.packState.Get(), slot.aux.Get(), - slot.quantPayload.Get(), + m_quantPayload.Get(), }; const uint64_t elements[] = { - slot.wave->GetDesc().Width / sizeof(float), - slot.scratch->GetDesc().Width / sizeof(float), - slot.atlas->GetDesc().Width / sizeof(float), - slot.planes->GetDesc().Width / sizeof(float), + m_wave->GetDesc().Width / sizeof(float), + m_scratch->GetDesc().Width / sizeof(float), + m_atlas->GetDesc().Width / sizeof(float), + m_planeData->GetDesc().Width / sizeof(float), slot.packed->GetDesc().Width / sizeof(uint32_t), slot.packState->GetDesc().Width / sizeof(uint32_t), slot.aux->GetDesc().Width / sizeof(uint32_t), - slot.quantPayload->GetDesc().Width / sizeof(uint32_t), + m_quantPayload->GetDesc().Width / sizeof(uint32_t), }; constexpr uint64_t MAX_ELEMENTS = 1ULL << D3D12_REQ_BUFFER_RESOURCE_TEXEL_COUNT_2_TO_EXP; @@ -1841,6 +1986,7 @@ void FinalizeMain() ShaderParameters values = parameters; std::copy(std::begin(m_auxBase), std::end(m_auxBase), std::begin(values.auxBase)); + values.state[0] = m_payloadWords; list->SetPipelineState(m_pipelines[pipeline].Get()); list->SetComputeRoot32BitConstants(1, SHADER_PARAMETER_DWORDS, &values, 0); @@ -1888,7 +2034,7 @@ void FinalizeMain() return true; } - bool CreateSlot(Slot& slot, uint64_t planePixels) + bool CreateSlot(Slot& slot) { D3D12_DESCRIPTOR_HEAP_DESC heap = {}; @@ -1918,23 +2064,7 @@ void FinalizeMain() return false; } - if (!CreateBuffer(m_device.Get(), planePixels * sizeof(float), - D3D12_HEAP_TYPE_DEFAULT, - D3D12_RESOURCE_STATE_UNORDERED_ACCESS, - D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, slot.wave) || - !CreateBuffer(m_device.Get(), planePixels * sizeof(float), - D3D12_HEAP_TYPE_DEFAULT, - D3D12_RESOURCE_STATE_UNORDERED_ACCESS, - D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, slot.scratch) || - !CreateBuffer(m_device.Get(), planePixels * sizeof(float), - D3D12_HEAP_TYPE_DEFAULT, - D3D12_RESOURCE_STATE_UNORDERED_ACCESS, - D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, slot.atlas) || - !CreateBuffer(m_device.Get(), planePixels * sizeof(float), - D3D12_HEAP_TYPE_DEFAULT, - D3D12_RESOURCE_STATE_UNORDERED_ACCESS, - D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, slot.planes) || - !CreateBuffer(m_device.Get(), m_packedCapacity, + if (!CreateBuffer(m_device.Get(), m_packedCapacity, D3D12_HEAP_TYPE_DEFAULT, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, slot.packed) || @@ -1947,12 +2077,6 @@ void FinalizeMain() D3D12_HEAP_TYPE_DEFAULT, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, slot.aux) || - !CreateBuffer(m_device.Get(), - static_cast(m_payloadWords) * sizeof(uint32_t), - D3D12_HEAP_TYPE_DEFAULT, - D3D12_RESOURCE_STATE_UNORDERED_ACCESS, - D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, - slot.quantPayload) || !CreateBuffer(m_device.Get(), m_readbackBytes, D3D12_HEAP_TYPE_READBACK, D3D12_RESOURCE_STATE_COPY_DEST, @@ -2004,7 +2128,26 @@ void FinalizeMain() if (planePixels > UINT32_MAX || pixels > UINT32_MAX - planePixels) return false; planes[i].offset = static_cast(planePixels); - planePixels += pixels; + // Sparse transforms retain untouched LL samples between frames, so + // every level needs a disjoint row range in its parity buffer. + uint32_t levelHeight = planes[i].height; + uint32_t lowpassRows[2] = {}; + for (uint32_t level = 0; level < planes[i].levels; ++level) + { + levelHeight >>= 1U; + const unsigned buffer = level & 1U; + if (lowpassRows[buffer] > planes[i].height || + levelHeight > planes[i].height - lowpassRows[buffer]) + return false; + const uint64_t lowpassOffset = planePixels + + static_cast(lowpassRows[buffer]) * planes[i].width; + if (lowpassOffset > UINT32_MAX) + return false; + planes[i].lowpassOffset[level] = + static_cast(lowpassOffset); + lowpassRows[buffer] += levelHeight; + } + planePixels += pixels; } constexpr uint64_t MAX_ELEMENTS = 1ULL << D3D12_REQ_BUFFER_RESOURCE_TEXEL_COUNT_2_TO_EXP; @@ -2030,7 +2173,10 @@ void FinalizeMain() const uint64_t payloadWords = 1U + static_cast(block8Capacity) * PYROWAVE_QUANT_WORDS_8X8; - if (auxWords > MAX_ELEMENTS || payloadWords > MAX_ELEMENTS || + const uint64_t payloadTotal = + payloadWords + static_cast(blockCapacity) + 1U; + if (auxWords > MAX_ELEMENTS || payloadWords > UINT32_MAX || + payloadTotal > MAX_ELEMENTS || rdoStateBase > UINT32_MAX) return false; @@ -2081,8 +2227,33 @@ void FinalizeMain() Reset(); return false; } + if (!CreateBuffer(m_device.Get(), planePixels * sizeof(float), + D3D12_HEAP_TYPE_DEFAULT, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, m_wave) || + !CreateBuffer(m_device.Get(), planePixels * sizeof(float), + D3D12_HEAP_TYPE_DEFAULT, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, m_scratch) || + !CreateBuffer(m_device.Get(), planePixels * sizeof(float), + D3D12_HEAP_TYPE_DEFAULT, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, m_atlas) || + !CreateBuffer(m_device.Get(), planePixels * sizeof(float), + D3D12_HEAP_TYPE_DEFAULT, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, m_planeData) || + !CreateBuffer(m_device.Get(), + payloadTotal * sizeof(uint32_t), + D3D12_HEAP_TYPE_DEFAULT, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS, + D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, m_quantPayload)) + { + Reset(); + return false; + } for (Slot& slot : m_slots) - if (!CreateSlot(slot, planePixels)) + if (!CreateSlot(slot)) { Reset(); return false; @@ -2135,10 +2306,11 @@ void FinalizeMain() } bool RecordBands(ID3D12GraphicsCommandList * list, Pipeline pipeline, - uint32_t sequence, uint32_t packedLimit) + const FrameWork& work, uint32_t sequence, uint32_t packedLimit) { uint32_t blockBase = 0; uint32_t blockBase8 = 0; + size_t bandIndex = 0; for (int level = static_cast(PYROWAVE_LEVELS) - 1; level >= 0; --level) for (unsigned component = 0; component < 3; ++component) @@ -2161,8 +2333,15 @@ void FinalizeMain() level == static_cast(PYROWAVE_LEVELS) - 1 ? 0 : 1; for (unsigned band = firstBand; band < 4; ++band) { - const uint32_t quantBits = + if (bandIndex >= work.bands.size()) + return false; + const BandWork& bandWork = work.bands[bandIndex++]; + const uint32_t quantBits = QuantBits(level, component, band); + if (bandWork.blockBase != blockBase || + bandWork.block8Base != blockBase8 || + bandWork.blocksX != blocksX || bandWork.blocksY != blocksY) + return false; ShaderParameters parameters = {}; parameters.dim0[0] = m_planes[component].width; parameters.dim0[1] = m_planes[component].offset; @@ -2187,10 +2366,36 @@ void FinalizeMain() m_blocksPerRDO); if (pipeline == PIPELINE_QUANT || pipeline == PIPELINE_ANALYZE) - Dispatch(list, pipeline, parameters, blocksX, blocksY); + { + if (work.full) + Dispatch(list, pipeline, parameters, blocksX, blocksY); + else + for (const RowRun& run : bandWork.blockRuns) + { + parameters.work[0] = run.x; + parameters.work[1] = run.y; + Dispatch(list, pipeline, parameters, + run.width, run.height); + } + } else if (pipeline == PIPELINE_PACK) - Dispatch(list, pipeline, parameters, - (blocksX + 1U) / 2U, (blocksY + 1U) / 2U); + { + if (work.full) + { + parameters.work[2] = 0xfU; + Dispatch(list, pipeline, parameters, + (blocksX + 1U) / 2U, (blocksY + 1U) / 2U); + } + else + for (const PackRun& run : bandWork.packRuns) + { + parameters.work[0] = run.x; + parameters.work[1] = run.y; + parameters.work[2] = run.activeMask; + Dispatch(list, pipeline, parameters, + run.width, run.height); + } + } else return false; @@ -2198,11 +2403,13 @@ void FinalizeMain() blockBase8 += blocks8X * blocks8Y; } } - return blockBase == m_blockCapacity && + return bandIndex == work.bands.size() && + blockBase == m_blockCapacity && blockBase8 == m_block8Capacity; } - void RecordTransforms(ID3D12GraphicsCommandList * list) + void RecordTransforms(ID3D12GraphicsCommandList * list, + const FrameWork& work) { uint32_t activeWidth [3] = {}; uint32_t activeHeight[3] = {}; @@ -2226,12 +2433,23 @@ void FinalizeMain() transform.op0[0] = activeWidth [component]; transform.op0[1] = activeHeight[component]; transform.op1[0] = level; - transform.op1[1] = level ? ((level - 1U) & 1U) : 0U; - transform.op1[2] = level & 1U; + transform.op1[1] = level ? + plane.lowpassOffset[level - 1U] : 0U; + transform.op1[2] = plane.lowpassOffset[level]; transform.op1[3] = plane.offset; - Dispatch(list, PIPELINE_DWT, transform, - (activeWidth[component] + 31U) / 32U, - (activeHeight[component] + 31U) / 32U); + transform.work[2] = level + 1U == plane.levels ? 1U : 0U; + if (work.full) + Dispatch(list, PIPELINE_DWT, transform, + (activeWidth[component] + 31U) / 32U, + (activeHeight[component] + 31U) / 32U); + else + for (const RowRun& run : work.dwtRuns[component][level]) + { + transform.work[0] = run.x; + transform.work[1] = run.y; + Dispatch(list, PIPELINE_DWT, transform, + run.width, run.height); + } activeWidth [component] >>= 1; activeHeight[component] >>= 1; } @@ -2269,13 +2487,402 @@ void FinalizeMain() slot.packed.Get(), 0, slot.frameLimit); } - bool Compact(const Slot& slot, const uint8_t * readback, + static GroupRange AffectedGroups( + uint32_t begin, uint32_t end, uint32_t length, + uint32_t groupCount) + { + GroupRange result; + begin = (std::min)(begin, length); + end = (std::min)(end, length); + if (begin >= end || !length || !groupCount) + return result; + result.begin = begin < 36U ? 0U : (begin - 36U) / 32U + 1U; + result.end = (std::min)(groupCount, (end + 35U) / 32U); + const uint32_t mirroredEdge = length > 36U ? length - 36U : 0U; + if (end > mirroredEdge) + { + result.begin = (std::min)(result.begin, groupCount - 1U); + result.end = groupCount; + } + result.begin = (std::min)(result.begin, result.end); + return result; + } + + static bool BuildRowRuns(const std::vector& mask, + size_t base, uint32_t width, uint32_t height, + std::vector& runs) + { + const size_t count = static_cast(width) * height; + if (base > mask.size() || count > mask.size() - base) + return false; + + runs.clear(); + std::vector previous; + std::vector current; + for (uint32_t y = 0; y < height; ++y) + { + current.clear(); + size_t previousPosition = 0; + uint32_t x = 0; + while (x < width) + { + while (x < width && !mask[base + y * width + x]) + ++x; + const uint32_t begin = x; + while (x < width && mask[base + y * width + x]) + ++x; + if (begin != x) + { + const uint32_t runWidth = x - begin; + while (previousPosition < previous.size() && + runs[previous[previousPosition]].x < begin) + ++previousPosition; + if (previousPosition < previous.size()) + { + RowRun& run = runs[previous[previousPosition]]; + if (run.x == begin && run.width == runWidth && + run.y + run.height == y) + { + ++run.height; + current.push_back(previous[previousPosition]); + ++previousPosition; + continue; + } + } + runs.push_back({ begin, y, runWidth, 1U }); + current.push_back(runs.size() - 1U); + } + } + previous.swap(current); + } + return true; + } + + static void MarkGroups(std::vector& mask, + uint32_t width, const GroupRange& x, const GroupRange& y) + { + for (uint32_t row = y.begin; row < y.end; ++row) + std::fill(mask.begin() + static_cast(row) * width + x.begin, + mask.begin() + static_cast(row) * width + x.end, + static_cast(1U)); + } + + bool BuildFrameWork(const FrameDesc& frame, bool forceFull, + FrameWork& work) const + { + work = {}; + work.full = forceFull || frame.damage == FrameDamage::FULL; + work.damagedBlocks.assign( + m_blockCapacity, static_cast(work.full)); + if (!work.full && frame.damage != FrameDamage::NONE && + (frame.damage != FrameDamage::RECTS || !frame.count || + frame.count > FRAME_DAMAGE_MAX)) + return false; + + constexpr size_t INVALID_BAND = + (std::numeric_limits::max)(); + size_t bandIndices[PYROWAVE_LEVELS][3][4]; + uint32_t blockBase = 0; + uint32_t blockBase8 = 0; + for (auto& levelBands : bandIndices) + for (auto& componentBands : levelBands) + std::fill(std::begin(componentBands), std::end(componentBands), + INVALID_BAND); + for (int level = static_cast(PYROWAVE_LEVELS) - 1; + level >= 0; --level) + for (unsigned component = 0; component < 3; ++component) + { + if (m_chroma == CHROMA_420 && component && !level) + continue; + const uint32_t localLevel = + m_chroma == CHROMA_420 && component ? + static_cast(level - 1) : + static_cast(level); + const uint32_t bandWidth = + m_planes[component].width >> (localLevel + 1U); + const uint32_t bandHeight = + m_planes[component].height >> (localLevel + 1U); + const uint32_t blocksX = (bandWidth + 31U) / 32U; + const uint32_t blocksY = (bandHeight + 31U) / 32U; + const uint32_t blocks8X = (bandWidth + 7U) / 8U; + const uint32_t blocks8Y = (bandHeight + 7U) / 8U; + const unsigned firstBand = + level == static_cast(PYROWAVE_LEVELS) - 1 ? 0U : 1U; + for (unsigned band = firstBand; band < 4; ++band) + { + bandIndices[level][component][band] = work.bands.size(); + work.bands.push_back( + { + blockBase, + blockBase8, + blocksX, + blocksY, + {}, + {}, + }); + blockBase += blocksX * blocksY; + blockBase8 += blocks8X * blocks8Y; + } + } + if (blockBase != m_blockCapacity || + blockBase8 != m_block8Capacity) + return false; + + if (!work.full && frame.damage == FrameDamage::RECTS) + { + for (unsigned index = 0; index < frame.count; ++index) + { + const RECT& rect = frame.rects[index]; + if (rect.left < 0 || rect.top < 0 || + rect.right <= rect.left || rect.bottom <= rect.top || + rect.right > static_cast(m_cfg.width) || + rect.bottom > static_cast(m_cfg.height)) + { + work.full = true; + std::fill(work.damagedBlocks.begin(), + work.damagedBlocks.end(), static_cast(1U)); + return true; + } + } + + const uint32_t colorDivisor = m_chroma == CHROMA_420 ? 2U : 1U; + const uint32_t colorWidth = m_planes[0].width / colorDivisor; + const uint32_t colorHeight = m_planes[0].height / colorDivisor; + const uint32_t colorGroupsX = (colorWidth + 7U) / 8U; + const uint32_t colorGroupsY = (colorHeight + 7U) / 8U; + std::vector colorMask( + static_cast(colorGroupsX) * colorGroupsY); + for (unsigned index = 0; index < frame.count; ++index) + { + const RECT& source = frame.rects[index]; + uint32_t left = + static_cast(source.left) / colorDivisor; + uint32_t top = + static_cast(source.top) / colorDivisor; + uint32_t right = + (static_cast(source.right) + colorDivisor - 1U) / + colorDivisor; + uint32_t bottom = + (static_cast(source.bottom) + colorDivisor - 1U) / + colorDivisor; + if (source.right == static_cast(m_cfg.width)) + right = colorWidth; + if (source.bottom == static_cast(m_cfg.height)) + bottom = colorHeight; + const GroupRange x = + { + left / 8U, + (std::min)(colorGroupsX, (right + 7U) / 8U), + }; + const GroupRange y = + { + top / 8U, + (std::min)(colorGroupsY, (bottom + 7U) / 8U), + }; + MarkGroups(colorMask, colorGroupsX, x, y); + } + if (!BuildRowRuns(colorMask, 0, colorGroupsX, colorGroupsY, + work.colorRuns)) + return false; + bool completeWork = std::all_of(colorMask.begin(), colorMask.end(), + [](uint8_t active) { return active != 0; }); + + std::vector dwtMasks[3][PYROWAVE_LEVELS]; + for (unsigned component = 0; component < 3; ++component) + { + const Plane& plane = m_planes[component]; + const bool subsampled = + m_chroma == CHROMA_420 && component; + const uint32_t divisor = subsampled ? 2U : 1U; + const uint32_t sourceWidth = + (m_cfg.width + divisor - 1U) / divisor; + const uint32_t sourceHeight = + (m_cfg.height + divisor - 1U) / divisor; + + std::vector currentRects; + currentRects.reserve(frame.count); + for (unsigned index = 0; index < frame.count; ++index) + { + const RECT& source = frame.rects[index]; + PlaneRect rect; + rect.left = static_cast(source.left) / divisor; + rect.top = static_cast(source.top) / divisor; + rect.right = + (static_cast(source.right) + divisor - 1U) / + divisor; + rect.bottom = + (static_cast(source.bottom) + divisor - 1U) / + divisor; + if (rect.right == sourceWidth) + rect.right = plane.width; + if (rect.bottom == sourceHeight) + rect.bottom = plane.height; + currentRects.push_back(rect); + } + + uint32_t activeWidth = plane.width; + uint32_t activeHeight = plane.height; + for (uint32_t localLevel = 0; localLevel < plane.levels; + ++localLevel) + { + const uint32_t globalLevel = localLevel + + (subsampled ? 1U : 0U); + const uint32_t bandWidth = activeWidth / 2U; + const uint32_t bandHeight = activeHeight / 2U; + const uint32_t groupCountX = (activeWidth + 31U) / 32U; + const uint32_t groupCountY = (activeHeight + 31U) / 32U; + const uint32_t blocksX = (bandWidth + 31U) / 32U; + const uint32_t blocksY = (bandHeight + 31U) / 32U; + const unsigned firstBand = + localLevel + 1U == plane.levels ? 0U : 1U; + std::vector& dwtMask = + dwtMasks[component][localLevel]; + dwtMask.assign( + static_cast(groupCountX) * groupCountY, + static_cast(0U)); + + std::vector nextRects; + nextRects.reserve(currentRects.size()); + for (const PlaneRect& rect : currentRects) + { + const GroupRange x = AffectedGroups( + rect.left, rect.right, activeWidth, groupCountX); + const GroupRange y = AffectedGroups( + rect.top, rect.bottom, activeHeight, groupCountY); + if (x.begin == x.end || y.begin == y.end) + continue; + MarkGroups(dwtMask, groupCountX, x, y); + + const uint32_t blockBeginX = x.begin / 2U; + const uint32_t blockBeginY = y.begin / 2U; + const uint32_t blockEndX = (x.end + 1U) / 2U; + const uint32_t blockEndY = (y.end + 1U) / 2U; + for (unsigned band = firstBand; band < 4; ++band) + { + const size_t bandIndex = + bandIndices[globalLevel][component][band]; + if (bandIndex == INVALID_BAND || + bandIndex >= work.bands.size()) + return false; + const BandWork& bandWork = work.bands[bandIndex]; + for (uint32_t blockY = blockBeginY; + blockY < blockEndY; ++blockY) + for (uint32_t blockX = blockBeginX; + blockX < blockEndX; ++blockX) + { + if (blockX >= blocksX || blockY >= blocksY) + return false; + work.damagedBlocks[bandWork.blockBase + + blockY * blocksX + blockX] = + static_cast(1U); + } + } + + nextRects.push_back( + { + x.begin * 16U, + y.begin * 16U, + (std::min)(bandWidth, x.end * 16U), + (std::min)(bandHeight, y.end * 16U), + }); + } + if (!BuildRowRuns(dwtMask, 0, groupCountX, groupCountY, + work.dwtRuns[component][localLevel])) + return false; + completeWork = completeWork && + std::all_of(dwtMask.begin(), dwtMask.end(), + [](uint8_t active) { return active != 0; }); + currentRects = std::move(nextRects); + activeWidth = bandWidth; + activeHeight = bandHeight; + } + } + if (completeWork && + std::all_of(work.damagedBlocks.begin(), + work.damagedBlocks.end(), + [](uint8_t damaged) { return damaged != 0; })) + { + work.full = true; + return true; + } + } + + for (BandWork& band : work.bands) + { + if (work.full) + continue; + if (!BuildRowRuns(work.damagedBlocks, band.blockBase, + band.blocksX, band.blocksY, band.blockRuns)) + return false; + + const uint32_t packWidth = (band.blocksX + 1U) / 2U; + const uint32_t packHeight = (band.blocksY + 1U) / 2U; + std::vector previous; + std::vector current; + for (uint32_t y = 0; y < packHeight; ++y) + { + current.clear(); + size_t previousPosition = 0; + uint32_t x = 0; + while (x < packWidth) + { + const auto activeMask = [&](uint32_t groupX) + { + uint32_t result = 0; + for (uint32_t record = 0; record < 4U; ++record) + { + const uint32_t blockX = groupX * 2U + (record & 1U); + const uint32_t blockY = y * 2U + (record >> 1U); + if (blockX < band.blocksX && blockY < band.blocksY && + work.damagedBlocks[band.blockBase + + blockY * band.blocksX + blockX]) + result |= 1U << record; + } + return result; + }; + + const uint32_t mask = activeMask(x); + if (!mask) + { + ++x; + continue; + } + const uint32_t begin = x++; + while (x < packWidth && activeMask(x) == mask) + ++x; + const uint32_t runWidth = x - begin; + while (previousPosition < previous.size() && + band.packRuns[previous[previousPosition]].x < begin) + ++previousPosition; + if (previousPosition < previous.size()) + { + PackRun& run = band.packRuns[previous[previousPosition]]; + if (run.x == begin && run.width == runWidth && + run.activeMask == mask && run.y + run.height == y) + { + ++run.height; + current.push_back(previous[previousPosition]); + ++previousPosition; + continue; + } + } + band.packRuns.push_back( + { begin, y, runWidth, 1U, mask }); + current.push_back(band.packRuns.size() - 1U); + } + previous.swap(current); + } + } + return true; + } + + bool Compact(Slot& slot, const uint8_t * readback, EncodedFrame& output) const { const uint32_t packedBytes = LoadLE32(readback); const uint32_t packetCount = LoadLE32(readback + 4U); const uint32_t overflow = LoadLE32(readback + 8U); - if (overflow || !packetCount || packetCount > m_blockCapacity || + if (overflow || packetCount > m_blockCapacity || packedBytes < 8U || packedBytes > m_packedCapacity || packedBytes > slot.frameLimit || packedBytes > MAX_FRAME_LENGTH) @@ -2308,11 +2915,23 @@ void FinalizeMain() for (uint32_t blockIndex = 0; blockIndex < m_blockCapacity; ++blockIndex) { + if (blockIndex >= slot.work.damagedBlocks.size()) + return false; const uint8_t * meta = packetMeta + static_cast(blockIndex) * PYROWAVE_PACKET_META_WORDS * sizeof(uint32_t); const uint32_t sourceOffset = LoadLE32(meta); const uint32_t length = LoadLE32(meta + sizeof(uint32_t)); + const bool damaged = slot.work.damagedBlocks[blockIndex]; + const bool dummy = !damaged && blockIndex == 0U && + packetCount == 1U && packedBytes >= 20U && + sourceOffset == 8U && length == 12U && + LoadLE32(packed + 8U) == + (1U | (3U << 16U) | (slot.sequence << 28U)) && + LoadLE32(packed + 12U) == 8U * (4U + 12U) && + LoadLE32(packed + 16U) == 6U << 20U; + if (!damaged && !dummy) + continue; if (!length) { if (sourceOffset) @@ -2348,11 +2967,14 @@ void FinalizeMain() return false; } - output.blocks[0].offset = 0; - output.blocks[0].length += 8U; + if (!output.blocks.empty()) + { + output.blocks[0].offset = 0; + output.blocks[0].length += 8U; + } output.blockCapacity = m_blockCapacity; - - return true; + output.damagedBlocks = std::move(slot.work.damagedBlocks); + return output.damagedBlocks.size() == output.blockCapacity; } Slot * OldestSlot() @@ -2381,7 +3003,9 @@ void FinalizeMain() slot.captureTime = 0; slot.order = 0; slot.frameLimit = 0; + slot.hardLimit = 0; slot.frame = {}; + slot.work = {}; slot.lease.Reset(); } @@ -2431,11 +3055,13 @@ void FinalizeMain() } bool Submit(FrameIn frame, TexLease lease, const FrameCfg& cfg, - LGNetVideoChromaSubsampling chroma, uint32_t frameLimit) + LGNetVideoChromaSubsampling chroma, uint32_t hardFrameLimit, + uint32_t updateFrameLimit) { if (!lease || !lease.Get() || lease->State() != D3D12_RESOURCE_STATE_COMMON || - frameLimit < PYROWAVE_MIN_FRAME_BYTES) + hardFrameLimit < PYROWAVE_MIN_FRAME_BYTES || + updateFrameLimit < PYROWAVE_MIN_FRAME_BYTES) return false; ID3D12Resource * source = lease->Get(); if (!m_ready || !SameCfg(m_cfg, cfg) || m_chroma != chroma || @@ -2443,11 +3069,26 @@ void FinalizeMain() if (!Configure(cfg, chroma, source)) return false; - const uint32_t packedLimit = (std::min)( - m_packedCapacity, frameLimit & ~UINT32_C(3)); - if (packedLimit < PYROWAVE_MIN_FRAME_BYTES) + const uint32_t hardLimit = (std::min)( + m_packedCapacity, hardFrameLimit & ~UINT32_C(3)); + const uint32_t packedLimit = (std::min)(hardLimit, + updateFrameLimit & ~UINT32_C(3)); + if (hardLimit < PYROWAVE_MIN_FRAME_BYTES || + packedLimit < PYROWAVE_MIN_FRAME_BYTES) return false; + FrameWork work; + try + { + if (!BuildFrameWork(frame.desc.frame, + !m_persistentValid || hardLimit != m_hardLimit, work)) + return false; + } + catch (...) + { + return false; + } + unsigned slotIndex = 0; while (slotIndex < ENCODER_SLOT_COUNT && m_slots[slotIndex].inFlight) ++slotIndex; @@ -2472,9 +3113,11 @@ void FinalizeMain() return false; } - slot.captureTime = lease->time.postStart; - slot.sequence = m_sequence; - slot.frameLimit = packedLimit; + slot.captureTime = lease->time.postStart; + slot.sequence = m_sequence; + slot.frameLimit = packedLimit; + slot.hardLimit = hardLimit; + slot.work = std::move(work); CreateSourceDescriptor(slot, source); ID3D12DescriptorHeap * heaps[] = { slot.descriptors.Get() }; @@ -2486,6 +3129,7 @@ void FinalizeMain() computeList->SetComputeRootSignature(m_rootSignature.Get()); computeList->SetComputeRootDescriptorTable( 0, slot.descriptors->GetGPUDescriptorHandleForHeapStart()); + UAVBarrier(computeList.Get(), nullptr); if (slot.used) { Transition(computeList.Get(), slot.packed.Get(), @@ -2504,13 +3148,16 @@ void FinalizeMain() UAVBarrier(computeList.Get(), slot.packState.Get()); ShaderParameters clearState = {}; - clearState.dim0[0] = m_blockCapacity; - clearState.dim0[1] = PYROWAVE_RDO_WORDS; + clearState.dim0[0] = PYROWAVE_RDO_WORDS; + clearState.dim0[1] = m_blockCapacity + 1U; + clearState.work[0] = slot.work.full; + const uint32_t clearWords = slot.work.full ? + (std::max)(PYROWAVE_RDO_WORDS, m_blockCapacity + 1U) : + PYROWAVE_RDO_WORDS; Dispatch(computeList.Get(), PIPELINE_CLEAR, clearState, - ((std::max)(m_blockCapacity, PYROWAVE_RDO_WORDS) + 255U) / 256U, - 1); + (clearWords + 255U) / 256U, 1); UAVBarrier(computeList.Get(), slot.aux.Get()); - UAVBarrier(computeList.Get(), slot.quantPayload.Get()); + UAVBarrier(computeList.Get(), m_quantPayload.Get()); ShaderParameters color = {}; color.dim0[0] = m_cfg.width; @@ -2526,23 +3173,32 @@ void FinalizeMain() m_planes[0].width / 2U : m_planes[0].width; const uint32_t colorHeight = m_chroma == CHROMA_420 ? m_planes[0].height / 2U : m_planes[0].height; - Dispatch(computeList.Get(), PIPELINE_COLOR, color, - (colorWidth + 7U) / 8U, (colorHeight + 7U) / 8U); - UAVBarrier(computeList.Get(), slot.planes.Get()); + if (slot.work.full) + Dispatch(computeList.Get(), PIPELINE_COLOR, color, + (colorWidth + 7U) / 8U, (colorHeight + 7U) / 8U); + else + for (const RowRun& run : slot.work.colorRuns) + { + color.work[0] = run.x; + color.work[1] = run.y; + Dispatch(computeList.Get(), PIPELINE_COLOR, color, + run.width, run.height); + } + UAVBarrier(computeList.Get(), m_planeData.Get()); - RecordTransforms(computeList.Get()); + RecordTransforms(computeList.Get(), slot.work); if (!RecordBands(computeList.Get(), PIPELINE_QUANT, - slot.sequence, m_packedCapacity)) + slot.work, slot.sequence, m_packedCapacity)) { copyCommand->Cancel(); computeCommand->Cancel(); return false; } UAVBarrier(computeList.Get(), slot.aux.Get()); - UAVBarrier(computeList.Get(), slot.quantPayload.Get()); + UAVBarrier(computeList.Get(), m_quantPayload.Get()); UAVBarrier(computeList.Get(), slot.packState.Get()); if (!RecordBands(computeList.Get(), PIPELINE_ANALYZE, - slot.sequence, m_packedCapacity)) + slot.work, slot.sequence, m_packedCapacity)) { copyCommand->Cancel(); computeCommand->Cancel(); @@ -2558,12 +3214,29 @@ void FinalizeMain() resolve.dim0[0] = (packedLimit - 8U) / UINT32_C(4); resolve.dim0[1] = m_blockCapacity; resolve.dim0[2] = m_blocksPerRDO; - Dispatch(computeList.Get(), PIPELINE_RESOLVE, resolve, - (m_blockCapacity + 63U) / 64U, 1); + resolve.work[2] = (hardLimit - 8U) / UINT32_C(4); + resolve.work[3] = (packedLimit - 8U) / UINT32_C(4); + if (slot.work.full) + { + resolve.work[1] = m_blockCapacity; + Dispatch(computeList.Get(), PIPELINE_RESOLVE, resolve, + (m_blockCapacity + 63U) / 64U, 1); + } + else + for (const BandWork& band : slot.work.bands) + for (const RowRun& run : band.blockRuns) + for (uint32_t row = 0; row < run.height; ++row) + { + resolve.work[0] = band.blockBase + + (run.y + row) * band.blocksX + run.x; + resolve.work[1] = run.width; + Dispatch(computeList.Get(), PIPELINE_RESOLVE, resolve, + (run.width + 63U) / 64U, 1); + } UAVBarrier(computeList.Get(), slot.aux.Get()); if (!RecordBands(computeList.Get(), PIPELINE_PACK, - slot.sequence, packedLimit)) + slot.work, slot.sequence, packedLimit)) { copyCommand->Cancel(); computeCommand->Cancel(); @@ -2572,6 +3245,7 @@ void FinalizeMain() UAVBarrier(computeList.Get(), slot.packed.Get()); UAVBarrier(computeList.Get(), slot.packState.Get()); UAVBarrier(computeList.Get(), slot.aux.Get()); + UAVBarrier(computeList.Get(), m_quantPayload.Get()); ShaderParameters finalize = {}; finalize.dim0[0] = m_cfg.width; finalize.dim0[1] = m_cfg.height; @@ -2581,6 +3255,7 @@ void FinalizeMain() m_cfg.profile.signal == FrameSignal::PQ_BT2020; finalize.op0[3] = 8U * (4U + QuantBits( static_cast(PYROWAVE_LEVELS) - 1, 0, 0)); + finalize.work[0] = slot.work.full; Dispatch(computeList.Get(), PIPELINE_FINALIZE, finalize, 1, 1); PrepareReadback(computeList.Get(), slot); RecordReadback(copyList.Get(), slot); @@ -2593,7 +3268,9 @@ void FinalizeMain() Reset(); return false; } - slot.used = true; + m_persistentValid = true; + m_hardLimit = hardLimit; + slot.used = true; if (!copyCommand->WaitFor( slot.computeSync.fence.Get(), slot.computeSync.value)) { @@ -2650,6 +3327,7 @@ void FinalizeMain() output.frameID = frameID; output.captureTimeNs = slot->captureTime; output.deadlineMs = MAX_FRAME_LATENCY_MS; + output.hardLimit = slot->hardLimit; } ReleaseSlot(*slot); return result; @@ -2670,18 +3348,18 @@ void FinalizeMain() slot.mapped = nullptr; } slot.readback.Reset(); - slot.quantPayload.Reset(); slot.aux.Reset(); slot.packState.Reset(); slot.packed.Reset(); - slot.planes.Reset(); - slot.atlas.Reset(); - slot.scratch.Reset(); - slot.wave.Reset(); slot.clearDescriptors.Reset(); slot.descriptors.Reset(); slot.used = false; } + m_planeData.Reset(); + m_atlas.Reset(); + m_scratch.Reset(); + m_wave.Reset(); + m_quantPayload.Reset(); for (ComPtr& pipeline : m_pipelines) pipeline.Reset(); m_rootSignature.Reset(); @@ -2699,6 +3377,8 @@ void FinalizeMain() m_packedCapacity = 0; m_auxWords = 0; m_payloadWords = 0; + m_hardLimit = 0; + m_persistentValid = false; std::fill(std::begin(m_auxBase), std::end(m_auxBase), 0U); std::fill(std::begin(m_planes), std::end(m_planes), Plane {}); } @@ -2707,6 +3387,13 @@ void FinalizeMain() struct CNetworkVideo::Impl { + struct ReferenceFrame + { + uint64_t frameID = 0; + std::shared_ptr frame; + std::vector subsequentBlocks; + }; + struct PeerState { NetworkVideoPeer peer = {}; @@ -2728,6 +3415,7 @@ struct CNetworkVideo::Impl uint32_t droppedFrames = 0; uint8_t blockedDeliveries = 0; std::shared_ptr referenceFrame; + std::deque referenceHistory; std::vector updateBlocks; bool statusSent = false; bool subscribed = false; @@ -2801,6 +3489,7 @@ struct CNetworkVideo::Impl std::vector m_peers; std::deque m_frames; std::deque m_sends; + std::shared_ptr m_canonicalFrame; std::shared_ptr m_latestFrame; size_t m_sendBytes = 0; @@ -2913,6 +3602,7 @@ struct CNetworkVideo::Impl static void ResetPeerReferenceNL(PeerState& peer) { peer.referenceFrame.reset(); + peer.referenceHistory.clear(); peer.updateBlocks.clear(); peer.referenceFrameID = 0; peer.referencePending = false; @@ -3045,28 +3735,152 @@ struct CNetworkVideo::Impl const EncodedFrame& encoded, size_t& bytes) { size_t blockBytes; + size_t frameBytes; size_t encodedBytes; return Multiply(encoded.blocks.size(), sizeof(EncodedBlock), blockBytes) && - Add(encoded.data.size(), blockBytes, encodedBytes) && + Add(encoded.data.size(), encoded.damagedBlocks.size(), frameBytes) && + Add(frameBytes, blockBytes, encodedBytes) && Add(encodedBytes, encoded.wholePayload ? encoded.wholePayload->size() : 0, bytes); } - static std::shared_ptr BuildUpdateFrame( - const EncodedFrame& reference, const EncodedFrame& current, - const std::vector& previousBlocks, - std::vector& updateBlocks) + static std::shared_ptr BuildCanonicalFrame( + const std::shared_ptr& previous, + EncodedFrame&& current) + { + if (!current.blockCapacity || current.data.size() < 8U || + current.damagedBlocks.size() != current.blockCapacity || + current.hardLimit < 20U || current.data.size() > current.hardLimit) + return {}; + if (!previous || previous->configEpoch != current.configEpoch || + previous->blockCapacity != current.blockCapacity || + previous->data.size() < 8U || + std::all_of(current.damagedBlocks.begin(), + current.damagedBlocks.end(), + [](uint8_t damaged) { return damaged != 0; })) + { + try + { + return std::make_shared(std::move(current)); + } + catch (...) + { + return {}; + } + } + + try + { + std::shared_ptr result = + std::make_shared(); + result->configEpoch = current.configEpoch; + result->sourceSerial = current.sourceSerial; + result->frameID = current.frameID; + result->captureTimeNs = current.captureTimeNs; + result->deadlineMs = current.deadlineMs; + result->blockCapacity = current.blockCapacity; + result->hardLimit = current.hardLimit; + result->damagedBlocks = std::move(current.damagedBlocks); + result->blocks.reserve((std::max)( + previous->blocks.size(), current.blocks.size())); + result->data.reserve((std::max)( + previous->data.size(), current.data.size())); + result->data.resize(8U); + + constexpr uint32_t SEQUENCE_BITS = PYROWAVE_SEQUENCE_MASK << 28U; + const uint32_t firstWord = LoadLE32(current.data.data()); + const uint32_t sequence = + (firstWord >> 28U) & PYROWAVE_SEQUENCE_MASK; + size_t previousPosition = 0; + size_t currentPosition = 0; + for (uint32_t index = 0; index < current.blockCapacity; ++index) + { + while (previousPosition < previous->blocks.size() && + previous->blocks[previousPosition].index < index) + ++previousPosition; + while (currentPosition < current.blocks.size() && + current.blocks[currentPosition].index < index) + ++currentPosition; + const EncodedBlock * previousBlock = + previousPosition < previous->blocks.size() && + previous->blocks[previousPosition].index == index ? + &previous->blocks[previousPosition] : nullptr; + const EncodedBlock * currentBlock = + currentPosition < current.blocks.size() && + current.blocks[currentPosition].index == index ? + ¤t.blocks[currentPosition] : nullptr; + const bool damaged = + result->damagedBlocks[index] != 0; + const EncodedFrame& sourceFrame = damaged ? current : *previous; + const EncodedBlock * sourceBlock = damaged ? + currentBlock : previousBlock; + if (!sourceBlock) + continue; + + const EncodedPacket packet = Packet(sourceFrame, *sourceBlock); + if (!packet.data || packet.length < 8U || + packet.length > PYROWAVE_PACKET_BYTES || + result->data.size() > current.hardLimit || + packet.length > current.hardLimit - result->data.size() || + (LoadLE32(packet.data + sizeof(uint32_t)) >> 8U) != index) + return {}; + const uint32_t offset = static_cast(result->data.size()); + result->data.insert(result->data.end(), + packet.data, packet.data + packet.length); + const uint32_t packetFirst = + (LoadLE32(result->data.data() + offset) & ~SEQUENCE_BITS) | + (sequence << 28U); + StoreLE32(result->data.data() + offset, packetFirst); + result->blocks.push_back({ offset, packet.length, index }); + } + if (result->blocks.empty()) + { + constexpr uint32_t DUMMY_PACKET_BYTES = 12U; + constexpr uint32_t DUMMY_QUANT_CODE = 8U * (4U + 12U); + if (result->data.size() > current.hardLimit || + DUMMY_PACKET_BYTES > current.hardLimit - result->data.size()) + return {}; + const uint32_t offset = static_cast(result->data.size()); + result->data.resize(result->data.size() + DUMMY_PACKET_BYTES); + StoreLE32(result->data.data() + offset, + 1U | (3U << 16U) | (sequence << 28U)); + StoreLE32(result->data.data() + offset + sizeof(uint32_t), + DUMMY_QUANT_CODE); + StoreLE32(result->data.data() + offset + 2U * sizeof(uint32_t), + 6U << 20U); + result->blocks.push_back({ offset, DUMMY_PACKET_BYTES, 0U }); + } + if (result->blocks.size() > UINT32_C(0x00ffffff) || + result->data.size() > current.hardLimit) + return {}; + + const uint32_t secondWord = + (LoadLE32(current.data.data() + sizeof(uint32_t)) & + ~(PYROWAVE_HEADER_CODE_MASK | UINT32_C(0x00ffffff))) | + static_cast(result->blocks.size()); + StoreLE32(result->data.data(), firstWord); + StoreLE32(result->data.data() + sizeof(uint32_t), secondWord); + result->blocks[0].offset = 0; + result->blocks[0].length += 8U; + return result; + } + catch (...) + { + return {}; + } + } + + static bool AccumulateUpdateBlocks(const EncodedFrame& reference, + const EncodedFrame& current, std::vector& updateBlocks) { - updateBlocks.clear(); if (!reference.blockCapacity || reference.blockCapacity != current.blockCapacity || reference.configEpoch != current.configEpoch || reference.data.size() < 8U || current.data.size() < 8U) - return {}; + return false; try { - updateBlocks = previousBlocks; updateBlocks.resize(current.blockCapacity, 0); size_t referencePosition = 0; size_t currentPosition = 0; @@ -3092,7 +3906,33 @@ struct CNetworkVideo::Impl EncodedPacket {})) updateBlocks[index] = 1; } + } + catch (...) + { + return false; + } + return true; + } + static std::shared_ptr BuildUpdateFrame( + const EncodedFrame& reference, const EncodedFrame& current, + const std::vector& previousBlocks, + std::vector& updateBlocks) + { + try + { + updateBlocks = previousBlocks; + } + catch (...) + { + updateBlocks.clear(); + return {}; + } + if (!AccumulateUpdateBlocks(reference, current, updateBlocks)) + return {}; + + try + { uint32_t operationCount = 0; for (uint8_t changed : updateBlocks) operationCount += changed != 0; @@ -3119,9 +3959,9 @@ struct CNetworkVideo::Impl StoreLE32(update->data.data(), firstWord); StoreLE32(update->data.data() + sizeof(uint32_t), secondWord); - const uint32_t sequence = + const uint32_t sequence = (firstWord >> 28U) & PYROWAVE_SEQUENCE_MASK; - currentPosition = 0; + size_t currentPosition = 0; for (uint32_t index = 0; index < current.blockCapacity; ++index) { while (currentPosition < current.blocks.size() && @@ -3177,6 +4017,99 @@ struct CNetworkVideo::Impl } } + static void RecordReferenceFrameNL(PeerState& peer, uint64_t frameID, + const std::shared_ptr& frame) + { + if (!frameID || !frame) + return; + + if (!peer.referenceHistory.empty()) + { + ReferenceFrame& candidate = peer.referenceHistory.front(); + if (frameID <= candidate.frameID) + return; + if (candidate.frame && AccumulateUpdateBlocks( + *candidate.frame, *frame, candidate.subsequentBlocks)) + return; + if (candidate.subsequentBlocks.size() == frame->blockCapacity) + std::fill(candidate.subsequentBlocks.begin(), + candidate.subsequentBlocks.end(), static_cast(1U)); + return; + } + + try + { + ReferenceFrame candidate; + candidate.frameID = frameID; + candidate.frame = frame; + candidate.subsequentBlocks.assign( + frame->blockCapacity, static_cast(0U)); + peer.referenceHistory.push_back(std::move(candidate)); + } + catch (...) + { + } + } + + static void RemoveReferenceFrameNL(PeerState& peer, uint64_t frameID) + { + peer.referenceHistory.erase(std::remove_if( + peer.referenceHistory.begin(), peer.referenceHistory.end(), + [frameID](const ReferenceFrame& retained) + { + return retained.frameID == frameID; + }), peer.referenceHistory.end()); + } + + static bool AdvancePeerReferenceNL(PeerState& peer, + uint64_t lastCompleteFrameID, uint64_t lastDecodedFrameID) + { + const uint64_t acknowledgedID = (std::min)( + lastCompleteFrameID, lastDecodedFrameID); + ReferenceFrame * candidate = nullptr; + for (auto retained = peer.referenceHistory.rbegin(); + retained != peer.referenceHistory.rend(); ++retained) + if (retained->frameID >= peer.referenceFrameID && + retained->frameID <= acknowledgedID) + { + candidate = &*retained; + break; + } + + if (!candidate) + { + if (!peer.referencePending || !peer.referenceFrameID || + peer.referenceFrameID > acknowledgedID) + return false; + peer.referencePending = false; + peer.keyframePending = false; + return true; + } + + const uint64_t promotedID = + candidate->frameID; + const std::shared_ptr promotedFrame = + candidate->frame; + if (!promotedFrame) + return false; + + if (candidate->subsequentBlocks.size() != promotedFrame->blockCapacity) + return false; + + std::vector subsequentBlocks = + std::move(candidate->subsequentBlocks); + + peer.referenceFrame = promotedFrame; + peer.referenceFrameID = promotedID; + peer.referencePending = false; + peer.keyframePending = false; + peer.updateBlocks = std::move(subsequentBlocks); + while (!peer.referenceHistory.empty() && + peer.referenceHistory.front().frameID <= promotedID) + peer.referenceHistory.pop_front(); + return true; + } + bool EncodedQueuedNL(const EncodedFrame * encoded, uint64_t excludingSerial = 0) const { @@ -3315,6 +4248,7 @@ struct CNetworkVideo::Impl m_failed = true; m_latestSourceSerial = 0; m_frames.clear(); + m_canonicalFrame.reset(); m_latestFrame.reset(); for (PeerState& peer : m_peers) { @@ -3343,6 +4277,7 @@ struct CNetworkVideo::Impl if (item->frameID && item->frameID != activeFrame && SamePeer(item->peer, peer.peer)) { + RemoveReferenceFrameNL(peer, item->frameID); ReleaseSendAccountingNL(*item); item = m_sends.erase(item); } @@ -3355,6 +4290,7 @@ struct CNetworkVideo::Impl for (auto item = m_sends.begin(); item != m_sends.end();) if (SamePeer(item->peer, peer.peer)) { + RemoveReferenceFrameNL(peer, item->frameID); ReleaseSendAccountingNL(*item); item = m_sends.erase(item); } @@ -3868,10 +4804,12 @@ struct CNetworkVideo::Impl peer.lastCaptureTimeNs = frame->captureTimeNs; if (establishReference) { + peer.referenceHistory.clear(); peer.referenceFrame = frame; peer.referenceFrameID = frameID; peer.referencePending = true; peer.updateBlocks = std::move(cleanBlocks); + RecordReferenceFrameNL(peer, frameID, frame); } peer.keyframePending = establishReference; return true; @@ -3931,6 +4869,7 @@ struct CNetworkVideo::Impl if (PeerNeedsReferenceNL(peer, *current)) return QueueFrameNL( peer, current, frameID, presentationBaseNs, true); + DropPeerFramesNL(peer); std::vector updateBlocks; const std::shared_ptr update = BuildUpdateFrame( *peer.referenceFrame, *current, peer.updateBlocks, @@ -3961,6 +4900,7 @@ struct CNetworkVideo::Impl frameID, presentationBaseNs)) return false; peer.updateBlocks = std::move(updateBlocks); + RecordReferenceFrameNL(peer, frameID, current); return true; } @@ -4026,51 +4966,49 @@ struct CNetworkVideo::Impl void PublishFrame(EncodedFrame&& frame) { - bool buildWholePayload = false; - { - CSRWSharedLock lock(m_lock); - if (!m_started || m_stopped || m_failed || !m_active || - frame.configEpoch != m_configEpoch) - return; - for (const PeerState& peer : m_peers) - if (peer.subscribed && peer.configured && - !peer.referencePending && - PeerNeedsReferenceNL(peer, frame) && - PeerFrameDueNL(peer, frame) && - PeerUsesWholeFrameNL(peer, frame)) - { - buildWholePayload = true; - break; - } - } - if (buildWholePayload) - { - frame.wholePayload = BuildWholeFramePayload(frame, frame.deadlineMs, - frame.frameID, frame.captureTimeNs); - if (!frame.wholePayload) - return; - } - - std::shared_ptr encoded; - try - { - encoded = std::make_shared(std::move(frame)); - } - catch (...) - { - return; - } - CSRWExclusiveLock lock(m_lock); if (!m_started || m_stopped || m_failed || !m_active || - encoded->configEpoch != m_configEpoch) + frame.configEpoch != m_configEpoch) return; - uint64_t frameID = encoded->frameID; + std::shared_ptr canonical = BuildCanonicalFrame( + m_canonicalFrame, std::move(frame)); + if (!canonical) + { + FailNL(4); + return; + } + + uint64_t frameID = canonical->frameID; if (frameID != m_nextFrameID) { - frameID = Next64(m_nextFrameID); - m_nextFrameID = frameID; + frameID = Next64(m_nextFrameID); + m_nextFrameID = frameID; + canonical->frameID = frameID; } + bool buildWholePayload = false; + for (const PeerState& peer : m_peers) + if (peer.subscribed && peer.configured && + !peer.referencePending && + PeerNeedsReferenceNL(peer, *canonical) && + PeerFrameDueNL(peer, *canonical) && + PeerUsesWholeFrameNL(peer, *canonical)) + { + buildWholePayload = true; + break; + } + if (buildWholePayload) + { + canonical->wholePayload = BuildWholeFramePayload(*canonical, + canonical->deadlineMs, frameID, canonical->captureTimeNs); + if (!canonical->wholePayload) + { + FailNL(4); + return; + } + } + + const std::shared_ptr encoded = canonical; + m_canonicalFrame = encoded; // An older in-flight encode must not replace a newer retained source. if (encoded->sourceSerial && encoded->sourceSerial == m_latestSourceSerial) @@ -4090,8 +5028,9 @@ struct CNetworkVideo::Impl bool SubmitPending() { PendingFrame pending; - LGNetVideoChromaSubsampling chroma = CHROMA_444; - uint32_t frameLimit = MAX_FRAME_LENGTH; + LGNetVideoChromaSubsampling chroma = CHROMA_444; + uint32_t hardLimit = MAX_FRAME_LENGTH; + uint32_t updateLimit = MAX_FRAME_LENGTH; { CSRWExclusiveLock lock(m_lock); if (m_encoder.Busy() || m_frames.empty() || !m_started || @@ -4101,8 +5040,8 @@ struct CNetworkVideo::Impl for (const PeerState& peer : m_peers) if (peer.subscribed && peer.configured) { - frameLimit = (std::min)( - frameLimit, peer.subscription.maxFrameLength); + hardLimit = (std::min)( + hardLimit, peer.subscription.maxFrameLength); configured = true; } if (!configured) @@ -4112,15 +5051,15 @@ struct CNetworkVideo::Impl if (pending.configEpoch != m_configEpoch || !SameCfg(pending.cfg, m_activeCfg)) return true; - frameLimit = (std::min)( - frameLimit, FrameLimitNL(pending.lease->time.postStart)); - chroma = m_videoChroma; + updateLimit = (std::min)( + hardLimit, FrameLimitNL(pending.lease->time.postStart)); + chroma = m_videoChroma; } if (m_encoder.Submit(std::move(pending.frame), std::move(pending.lease), pending.cfg, chroma, - frameLimit)) + hardLimit, updateLimit)) return true; CSRWExclusiveLock lock(m_lock); @@ -4337,15 +5276,19 @@ void CNetworkVideo::DetachSender() if (!m_impl) return; CSRWExclusiveLock processLock(m_impl->m_processLock); - CSRWExclusiveLock callbackLock(m_impl->m_senderCallbackLock); - CSRWExclusiveLock lock(m_impl->m_lock); - m_impl->m_sender = nullptr; - m_impl->m_latestSourceSerial = 0; - m_impl->m_sendBytes = 0; - m_impl->m_frames.clear(); - m_impl->m_sends.clear(); - m_impl->m_peers.clear(); - m_impl->m_latestFrame.reset(); + { + CSRWExclusiveLock callbackLock(m_impl->m_senderCallbackLock); + CSRWExclusiveLock lock(m_impl->m_lock); + m_impl->m_sender = nullptr; + m_impl->m_latestSourceSerial = 0; + m_impl->m_sendBytes = 0; + m_impl->m_frames.clear(); + m_impl->m_sends.clear(); + m_impl->m_peers.clear(); + m_impl->m_canonicalFrame.reset(); + m_impl->m_latestFrame.reset(); + } + m_impl->m_encoder.Reset(); } bool CNetworkVideo::Start(uint32_t videoBitrateMbps, @@ -4388,6 +5331,7 @@ void CNetworkVideo::Stop() m_impl->m_latestSourceSerial = 0; m_impl->m_frames.clear(); m_impl->m_sends.clear(); + m_impl->m_canonicalFrame.reset(); m_impl->m_latestFrame.reset(); for (Impl::PeerState& peer : m_impl->m_peers) { @@ -4609,19 +5553,13 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket( return NetworkVideoPacketResult::IGNORED; if (!peer->subscribed) return NetworkVideoPacketResult::INVALID; - bool referenceAccepted = false; peer->feedback = feedback; peer->lastCompleteID = feedback.lastCompleteFrameID; peer->congested = feedback.flags & LG_NET_VIDEO_FEEDBACK_CONGESTED; - if (peer->referencePending && peer->referenceFrameID && - feedback.lastCompleteFrameID >= peer->referenceFrameID && - feedback.lastDecodedFrameID >= peer->referenceFrameID) - { - peer->referencePending = false; - peer->keyframePending = false; - referenceAccepted = true; - } + const bool referenceAccepted = + m_impl->AdvancePeerReferenceNL(*peer, + feedback.lastCompleteFrameID, feedback.lastDecodedFrameID); if (feedback.flags & (LG_NET_VIDEO_FEEDBACK_REQUEST_KEYFRAME | LG_NET_VIDEO_FEEDBACK_DECODER_STALLED)) { @@ -4742,6 +5680,7 @@ void CNetworkVideo::Commit() m_impl->m_lastBudgetCaptureTimeNs = 0; m_impl->m_latestSourceSerial = 0; m_impl->m_frames.clear(); + m_impl->m_canonicalFrame.reset(); m_impl->m_latestFrame.reset(); for (Impl::PeerState& peer : m_impl->m_peers) if (peer.subscribed) @@ -4798,7 +5737,11 @@ PushResult CNetworkVideo::Push(FrameIn frame, TexLease lease) noexcept { // Keep the newest source available until a subscriber can encode it. if (m_impl->m_frames.size() >= MAX_PENDING_FRAMES) + { + MergeFrameDamage(frame.desc.frame, + m_impl->m_frames.front().frame.desc.frame); m_impl->m_frames.pop_front(); + } Impl::PendingFrame pending; pending.frame = std::move(frame); pending.lease = std::move(lease);