From ccff282cd084fcf243b2c183992880e18e38356d Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Thu, 8 Oct 2026 14:18:46 +1100 Subject: [PATCH] [client/idd] network: preserve exact frame damage --- client/include/interface/transport.h | 2 +- client/renderers/EGL/damage.c | 3 +- client/renderers/EGL/damage.h | 2 +- client/renderers/EGL/egl.c | 19 +- client/renderers/EGL/texture_framebuffer.c | 13 +- client/src/render_queue.c | 4 +- client/transports/Network/video.c | 2 +- client/transports/Network/video/decoder_egl.c | 462 +++++++++--------- client/transports/Network/video/pyrowave.c | 41 +- client/transports/Network/video/pyrowave.h | 7 +- client/transports/Network/video/shaders.h | 25 +- common/include/common/types.h | 2 + idd/LGIdd/transport/network/CNetworkVideo.cpp | 228 ++++++++- repos/LGProtocol | 2 +- 14 files changed, 519 insertions(+), 293 deletions(-) diff --git a/client/include/interface/transport.h b/client/include/interface/transport.h index a0eb42f9..e67c870a 100644 --- a/client/include/interface/transport.h +++ b/client/include/interface/transport.h @@ -33,7 +33,7 @@ #include "interface/external_frame.h" #include "interface/input.h" -#define LG_TRANSPORT_MAX_DAMAGE_RECTS LG_MAX_FRAME_DAMAGE_RECTS +#define LG_TRANSPORT_MAX_DAMAGE_RECTS LG_MAX_RUNTIME_DAMAGE_RECTS typedef struct LG_Transport LG_Transport; typedef struct LG_RendererInterop LG_RendererInterop; diff --git a/client/renderers/EGL/damage.c b/client/renderers/EGL/damage.c index ff5db98e..c40a525e 100644 --- a/client/renderers/EGL/damage.c +++ b/client/renderers/EGL/damage.c @@ -88,7 +88,8 @@ bool egl_damageInit(EGL_Damage ** damage) return false; } - if (!egl_desktopRectsInit(&(*damage)->mesh, LG_MAX_FRAME_DAMAGE_RECTS)) + if (!egl_desktopRectsInit(&(*damage)->mesh, + LG_MAX_RUNTIME_DAMAGE_RECTS)) { DEBUG_ERROR("Failed to initialize the mesh"); return false; diff --git a/client/renderers/EGL/damage.h b/client/renderers/EGL/damage.h index e6a0d52d..9018ef2c 100644 --- a/client/renderers/EGL/damage.h +++ b/client/renderers/EGL/damage.h @@ -29,7 +29,7 @@ struct DesktopDamage { LG_RendererFrameToken frameToken; int count; - KVMFRFrameDamageRect rects[LG_MAX_FRAME_DAMAGE_RECTS]; + KVMFRFrameDamageRect rects[LG_MAX_RUNTIME_DAMAGE_RECTS]; }; typedef struct EGL_Damage EGL_Damage; diff --git a/client/renderers/EGL/egl.c b/client/renderers/EGL/egl.c index 4e3bc891..86824604 100644 --- a/client/renderers/EGL/egl.c +++ b/client/renderers/EGL/egl.c @@ -58,10 +58,11 @@ #include "postprocess.h" #include "util.h" -#define MAX_BUFFER_AGE 3 -#define DESKTOP_DAMAGE_COUNT 4 -#define DESKTOP_DAMAGE_MARGIN 1 -#define MAX_ACCUMULATED_DAMAGE ((LG_MAX_FRAME_DAMAGE_RECTS + MAX_OVERLAY_RECTS + 2) * MAX_BUFFER_AGE) +#define MAX_BUFFER_AGE 3 +#define DESKTOP_DAMAGE_COUNT 4 +#define DESKTOP_DAMAGE_MARGIN 1 +#define MAX_ACCUMULATED_DAMAGE \ + ((LG_MAX_RUNTIME_DAMAGE_RECTS + MAX_OVERLAY_RECTS + 2) * MAX_BUFFER_AGE) #define IDX_AGO(counter, i, total) (((counter) + (total) - (i)) % (total)) struct Options @@ -474,7 +475,7 @@ static void egl_addSwSurfaceDamage( damage->frameToken = LG_RENDERER_FRAME_TOKEN_NONE; if (damage->count >= 0) { - if (damage->count + 1 >= LG_MAX_FRAME_DAMAGE_RECTS) + if (damage->count + 1 > LG_MAX_RUNTIME_DAMAGE_RECTS) damage->count = -1; else damage->rects[damage->count++] = rect; @@ -880,7 +881,7 @@ static bool egl_onFrame(LG_Renderer * renderer, if (unlikely( damage->count == -1 || damageRectsCount == 0 || - damage->count + damageRectsCount >= LG_MAX_FRAME_DAMAGE_RECTS)) + damage->count + damageRectsCount > LG_MAX_RUNTIME_DAMAGE_RECTS)) { damage->count = -1; } @@ -1443,7 +1444,7 @@ static void egl_requeueDesktopDamage( this->desktopDamage + this->desktopDamageIdx; if (damage->count < 0 || pending->count < 0 || - damage->count + pending->count >= LG_MAX_FRAME_DAMAGE_RECTS) + damage->count + pending->count > LG_MAX_RUNTIME_DAMAGE_RECTS) pending->count = -1; else { @@ -1698,8 +1699,8 @@ static bool egl_render(LG_Renderer * renderer, LG_RendererRotate rotate, /* The diagnostics being displayed must not contribute to Compose. */ timing->composeTime = nanotime() - composeStart; - struct Rect damage[LG_MAX_FRAME_DAMAGE_RECTS + MAX_OVERLAY_RECTS + 2]; - int damageIdx = app_renderOverlay(damage, MAX_OVERLAY_RECTS); + struct Rect damage[LG_MAX_RUNTIME_DAMAGE_RECTS + MAX_OVERLAY_RECTS + 2]; + int damageIdx = app_renderOverlay(damage, MAX_OVERLAY_RECTS); if (unlikely(damageIdx != 0)) { if (damageIdx == -1) diff --git a/client/renderers/EGL/texture_framebuffer.c b/client/renderers/EGL/texture_framebuffer.c index 6e812fdb..259895c5 100644 --- a/client/renderers/EGL/texture_framebuffer.c +++ b/client/renderers/EGL/texture_framebuffer.c @@ -27,7 +27,7 @@ struct TexDamage { int count; - KVMFRFrameDamageRect rects[LG_MAX_FRAME_DAMAGE_RECTS]; + KVMFRFrameDamageRect rects[LG_MAX_RUNTIME_DAMAGE_RECTS]; }; typedef struct TexFB @@ -87,10 +87,11 @@ static bool egl_texFBUpdate(EGL_Texture * texture, const EGL_TexUpdate * update) if (!egl_texBufferStreamLock(parent)) return false; - struct TexDamage * damage = this->damage + parent->bufIndex; - bool damageAll = !update->rects || update->rectCount == 0 || damage->count < 0 || - damage->count + update->rectCount > LG_MAX_FRAME_DAMAGE_RECTS; - bool complete; + struct TexDamage * damage = this->damage + parent->bufIndex; + bool damageAll = + !update->rects || update->rectCount == 0 || damage->count < 0 || + damage->count + update->rectCount > LG_MAX_RUNTIME_DAMAGE_RECTS; + bool complete; if (damageAll) { @@ -172,7 +173,7 @@ static bool egl_texFBUpdate(EGL_Texture * texture, const EGL_TexUpdate * update) if (i == parent->bufIndex) damage->count = 0; else if (update->rects && update->rectCount > 0 && damage->count >= 0 && - damage->count + update->rectCount <= LG_MAX_FRAME_DAMAGE_RECTS) + damage->count + update->rectCount <= LG_MAX_RUNTIME_DAMAGE_RECTS) { memcpy(damage->rects + damage->count, update->rects, update->rectCount * sizeof(KVMFRFrameDamageRect)); diff --git a/client/src/render_queue.c b/client/src/render_queue.c index 679365a3..a3cc7280 100644 --- a/client/src/render_queue.c +++ b/client/src/render_queue.c @@ -147,7 +147,7 @@ typedef struct RenderQueueSwSurface bool damageFull; int damageCount; - KVMFRFrameDamageRect damage[LG_MAX_FRAME_DAMAGE_RECTS]; + KVMFRFrameDamageRect damage[LG_MAX_RUNTIME_DAMAGE_RECTS]; bool updateQueued; RenderCommand updateCommand; @@ -361,7 +361,7 @@ static void swSurfaceDamageAdd(RenderQueueSwSurface * surface, i = 0; } - if (surface->damageCount < LG_MAX_FRAME_DAMAGE_RECTS) + if (surface->damageCount < LG_MAX_RUNTIME_DAMAGE_RECTS) break; int best = 0; diff --git a/client/transports/Network/video.c b/client/transports/Network/video.c index 79a4fb5e..a5181b1b 100644 --- a/client/transports/Network/video.c +++ b/client/transports/Network/video.c @@ -300,7 +300,7 @@ static void networkVideoMergeDamage(NetworkVideoDecodedFrame * destination, return; } - if (destination->damageRectsCount >= + if (destination->damageRectsCount > LG_TRANSPORT_MAX_DAMAGE_RECTS - source->damageRectsCount) { uint32_t left = UINT32_MAX; diff --git a/client/transports/Network/video/decoder_egl.c b/client/transports/Network/video/decoder_egl.c index 11f73e15..af5d301e 100644 --- a/client/transports/Network/video/decoder_egl.c +++ b/client/transports/Network/video/decoder_egl.c @@ -68,10 +68,10 @@ #define NETWORK_VIDEO_MIN_SIZE (4U << NETWORK_VIDEO_LEVELS) #define NETWORK_VIDEO_UPLOADS 3U #define NETWORK_VIDEO_POOL_SIZE 3U +#define NETWORK_VIDEO_DAMAGE_HISTORY 64U #define NETWORK_VIDEO_POOL_BYTE_LIMIT \ (UINT64_C(192) * 1024U * 1024U) #define NETWORK_VIDEO_PIXEL_BYTES 8U -#define NETWORK_VIDEO_TILE_SIZE 32U #define NETWORK_VIDEO_SEQUENCE_BITS (UINT32_C(7) << 28U) struct NetworkVideoBlockMeta @@ -119,6 +119,12 @@ struct NetworkVideoTexture uint64_t generation; }; +struct NetworkVideoDamageHistory +{ + uint64_t generation; + LGNetVideoDamage damage; +}; + struct NetworkVideoTexturePool { LG_Lock lock; @@ -165,6 +171,7 @@ struct NetworkVideoUniforms GLint crPlane; GLint outputSize; GLint outputOffset; + GLint outputExtent; GLint bt2020; GLint chromaLeft; GLint lumaOffset; @@ -173,6 +180,14 @@ struct NetworkVideoUniforms GLint chromaScale; } yuv; + + struct + { + GLint inputImage; + GLint outputOffset; + GLint outputExtent; + } + copy; }; struct NetworkVideoDecoder @@ -187,8 +202,10 @@ struct NetworkVideoDecoder GLuint dequantProgram; GLuint idwtProgram; GLuint yuvProgram; + GLuint copyProgram; GLuint coefficientTextures[NETWORK_VIDEO_LEVELS]; GLuint planeTextures [NETWORK_VIDEO_COMPONENTS]; + GLuint canonicalTexture; struct NetworkVideoUpload uploads [NETWORK_VIDEO_UPLOADS]; struct NetworkVideoTexturePool * texturePool; struct NetworkVideoUniforms uniforms; @@ -198,8 +215,8 @@ struct NetworkVideoDecoder uint8_t * dirtyBlocks; struct NetworkVideoDirtyGrid dirty[NETWORK_VIDEO_COMPONENTS][NETWORK_VIDEO_LEVELS]; - uint8_t * outputDirty; - uint64_t * outputTileGenerations; + struct NetworkVideoDamageHistory + damageHistory[NETWORK_VIDEO_DAMAGE_HISTORY]; LGNetVideoStreamConfig config; struct NetworkVideoBlockMeta @@ -209,8 +226,6 @@ struct NetworkVideoDecoder uint32_t alignedWidth; uint32_t alignedHeight; uint32_t blockCapacity; - uint32_t outputTileWidth; - uint32_t outputTileHeight; uint64_t referenceFrameID; uint64_t outputGeneration; unsigned uploadIndex; @@ -518,6 +533,8 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder) { glDeleteTextures(NETWORK_VIDEO_LEVELS, decoder->coefficientTextures); glDeleteTextures(NETWORK_VIDEO_COMPONENTS, decoder->planeTextures); + if (decoder->canonicalTexture) + glDeleteTextures(1, &decoder->canonicalTexture); for (unsigned index = 0; index < NETWORK_VIDEO_UPLOADS; ++index) { struct NetworkVideoUpload * upload = &decoder->uploads[index]; @@ -537,28 +554,24 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder) component < NETWORK_VIDEO_COMPONENTS; ++component) for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level) free(decoder->dirty[component][level].tiles); - free(decoder->outputDirty); - free(decoder->outputTileGenerations); free(decoder->offsets); memset(decoder->uploads, 0, sizeof(decoder->uploads)); - decoder->texturePool = NULL; - decoder->offsets = NULL; - decoder->packetStates = NULL; - decoder->pendingPackets = NULL; - decoder->dirtyBlocks = NULL; - decoder->outputDirty = NULL; - decoder->outputTileGenerations = NULL; - decoder->alignedWidth = 0; - decoder->alignedHeight = 0; - decoder->blockCapacity = 0; - decoder->outputTileWidth = 0; - decoder->outputTileHeight = 0; - decoder->referenceFrameID = 0; - decoder->outputGeneration = 0; - decoder->uploadIndex = 0; - decoder->configured = false; - decoder->referenceValid = false; + memset(decoder->damageHistory, 0, sizeof(decoder->damageHistory)); + decoder->texturePool = NULL; + decoder->offsets = NULL; + decoder->packetStates = NULL; + decoder->pendingPackets = NULL; + decoder->dirtyBlocks = NULL; + decoder->canonicalTexture = 0; + decoder->alignedWidth = 0; + decoder->alignedHeight = 0; + decoder->blockCapacity = 0; + decoder->referenceFrameID = 0; + decoder->outputGeneration = 0; + decoder->uploadIndex = 0; + decoder->configured = false; + decoder->referenceValid = false; memset(decoder->coefficientTextures, 0, sizeof(decoder->coefficientTextures)); memset(decoder->planeTextures, 0, sizeof(decoder->planeTextures)); @@ -575,9 +588,12 @@ static void networkVideoDeleteProgramsNL(NetworkVideoDecoder * decoder) glDeleteProgram(decoder->idwtProgram); if (decoder->yuvProgram) glDeleteProgram(decoder->yuvProgram); + if (decoder->copyProgram) + glDeleteProgram(decoder->copyProgram); decoder->dequantProgram = 0; decoder->idwtProgram = 0; decoder->yuvProgram = 0; + decoder->copyProgram = 0; memset(&decoder->uniforms, 0, sizeof(decoder->uniforms)); } @@ -619,6 +635,8 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) decoder->yuvProgram, "outputSize"); decoder->uniforms.yuv.outputOffset = glGetUniformLocation( decoder->yuvProgram, "outputOffset"); + decoder->uniforms.yuv.outputExtent = glGetUniformLocation( + decoder->yuvProgram, "outputExtent"); decoder->uniforms.yuv.bt2020 = glGetUniformLocation( decoder->yuvProgram, "bt2020"); decoder->uniforms.yuv.chromaLeft = glGetUniformLocation( @@ -632,6 +650,13 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) decoder->uniforms.yuv.chromaScale = glGetUniformLocation( decoder->yuvProgram, "chromaScale"); + decoder->uniforms.copy.inputImage = glGetUniformLocation( + decoder->copyProgram, "inputImage"); + decoder->uniforms.copy.outputOffset = glGetUniformLocation( + decoder->copyProgram, "outputOffset"); + decoder->uniforms.copy.outputExtent = glGetUniformLocation( + decoder->copyProgram, "outputExtent"); + const struct NetworkVideoUniforms * uniforms = &decoder->uniforms; if (uniforms->dequant.resolution < 0 || uniforms->dequant.outputLayer < 0 || @@ -648,11 +673,15 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) uniforms->yuv.lumaPlane < 0 || uniforms->yuv.cbPlane < 0 || uniforms->yuv.crPlane < 0 || uniforms->yuv.outputSize < 0 || uniforms->yuv.outputOffset < 0 || + uniforms->yuv.outputExtent < 0 || uniforms->yuv.bt2020 < 0 || uniforms->yuv.chromaLeft < 0 || uniforms->yuv.lumaOffset < 0 || uniforms->yuv.lumaScale < 0 || uniforms->yuv.chromaMidpoint < 0 || - uniforms->yuv.chromaScale < 0) + uniforms->yuv.chromaScale < 0 || + uniforms->copy.inputImage < 0 || + uniforms->copy.outputOffset < 0 || + uniforms->copy.outputExtent < 0) return false; glUseProgram(decoder->idwtProgram); @@ -661,6 +690,8 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) glUniform1i(uniforms->yuv.lumaPlane, 0); glUniform1i(uniforms->yuv.cbPlane, 1); glUniform1i(uniforms->yuv.crPlane, 2); + glUseProgram(decoder->copyProgram); + glUniform1i(uniforms->copy.inputImage, 0); return glGetError() == GL_NO_ERROR; } @@ -824,31 +855,22 @@ static bool networkVideoAllocateNL(NetworkVideoDecoder * decoder) return false; } - decoder->offsets = malloc( + if (!networkVideoCreateTexture2D(&decoder->canonicalTexture, GL_RGBA16F, + decoder->config.width, decoder->config.height, GL_LINEAR)) + return false; + + decoder->offsets = malloc( (size_t)decoder->blockCapacity * sizeof(*decoder->offsets)); - decoder->packetStates = calloc( + decoder->packetStates = calloc( decoder->blockCapacity, sizeof(*decoder->packetStates)); - decoder->pendingPackets = calloc( + decoder->pendingPackets = calloc( decoder->blockCapacity, sizeof(*decoder->pendingPackets)); - decoder->dirtyBlocks = calloc( + decoder->dirtyBlocks = calloc( decoder->blockCapacity, sizeof(*decoder->dirtyBlocks)); - decoder->texturePool = networkVideoTexturePoolCreate(decoder); - decoder->outputTileWidth = - (decoder->config.width + NETWORK_VIDEO_TILE_SIZE - 1U) / - NETWORK_VIDEO_TILE_SIZE; - decoder->outputTileHeight = - (decoder->config.height + NETWORK_VIDEO_TILE_SIZE - 1U) / - NETWORK_VIDEO_TILE_SIZE; - const size_t outputTileCount = - (size_t)decoder->outputTileWidth * decoder->outputTileHeight; - decoder->outputDirty = calloc( - outputTileCount, sizeof(*decoder->outputDirty)); - decoder->outputTileGenerations = calloc( - outputTileCount, sizeof(*decoder->outputTileGenerations)); + decoder->texturePool = networkVideoTexturePoolCreate(decoder); if (!decoder->offsets || !decoder->packetStates || !decoder->pendingPackets || !decoder->dirtyBlocks || - !decoder->texturePool || !decoder->outputDirty || - !decoder->outputTileGenerations) + !decoder->texturePool) return false; for (unsigned component = 0; @@ -1105,8 +1127,12 @@ bool networkVideoDecoderAttach(NetworkVideoDecoder * decoder, NETWORK_VIDEO_YUV_SOURCES, sizeof(NETWORK_VIDEO_YUV_SOURCES) / sizeof(NETWORK_VIDEO_YUV_SOURCES[0])); + decoder->copyProgram = networkVideoCreateProgram( + NETWORK_VIDEO_COPY_SOURCES, + sizeof(NETWORK_VIDEO_COPY_SOURCES) / + sizeof(NETWORK_VIDEO_COPY_SOURCES[0])); const bool programsReady = decoder->dequantProgram && - decoder->idwtProgram && decoder->yuvProgram && + decoder->idwtProgram && decoder->yuvProgram && decoder->copyProgram && networkVideoCacheUniformsNL(decoder); networkVideoRestoreCurrent(decoder, &previous); if (!programsReady) @@ -1304,8 +1330,6 @@ static void networkVideoClearDirtyNL(NetworkVideoDecoder * decoder) grid->count = 0; } } - memset(decoder->outputDirty, 0, - (size_t)decoder->outputTileWidth * decoder->outputTileHeight); } static uint32_t networkVideoMirrorTexel( @@ -1383,44 +1407,6 @@ static void networkVideoMarkGridRect(struct NetworkVideoDirtyGrid * grid, } } -static void networkVideoMarkOutputRect(NetworkVideoDecoder * decoder, - unsigned component, uint32_t x, uint32_t y, - uint32_t width, uint32_t height) -{ - uint32_t expansion = component ? 1U : 0U; - if (component && - decoder->config.chromaSubsampling == LG_NET_VIDEO_CHROMA_420) - { - x *= 2U; - y *= 2U; - width *= 2U; - height *= 2U; - expansion = 2U; - } - - const uint32_t startX = x > expansion ? x - expansion : 0U; - const uint32_t startY = y > expansion ? y - expansion : 0U; - const uint64_t rawEndX = (uint64_t)x + width + expansion; - const uint64_t rawEndY = (uint64_t)y + height + expansion; - const uint32_t endX = rawEndX < decoder->config.width ? - (uint32_t)rawEndX : decoder->config.width; - const uint32_t endY = rawEndY < decoder->config.height ? - (uint32_t)rawEndY : decoder->config.height; - if (startX >= endX || startY >= endY) - return; - - const uint32_t tileStartX = startX / NETWORK_VIDEO_TILE_SIZE; - const uint32_t tileStartY = startY / NETWORK_VIDEO_TILE_SIZE; - const uint32_t tileEndX = - (endX + NETWORK_VIDEO_TILE_SIZE - 1U) / NETWORK_VIDEO_TILE_SIZE; - const uint32_t tileEndY = - (endY + NETWORK_VIDEO_TILE_SIZE - 1U) / NETWORK_VIDEO_TILE_SIZE; - for (uint32_t tileY = tileStartY; tileY < tileEndY; ++tileY) - for (uint32_t tileX = tileStartX; tileX < tileEndX; ++tileX) - decoder->outputDirty[ - (size_t)tileY * decoder->outputTileWidth + tileX] = 1U; -} - static void networkVideoPrepareDirtyNL(NetworkVideoDecoder * decoder) { networkVideoClearDirtyNL(decoder); @@ -1573,6 +1559,9 @@ static void networkVideoDequantNL(NetworkVideoDecoder * decoder, static void networkVideoPropagateDirtyNL(NetworkVideoDecoder * decoder, unsigned component, unsigned level, bool final) { + if (final) + return; + const struct NetworkVideoDirtyGrid * grid = &decoder->dirty[component][level]; const uint32_t outputWidth = decoder->alignedWidth >> level; @@ -1589,13 +1578,9 @@ static void networkVideoPropagateDirtyNL(NetworkVideoDecoder * decoder, (outputWidth - outputX < 32U ? outputWidth - outputX : 32U) : 0U; const uint32_t height = outputY < outputHeight ? (outputHeight - outputY < 32U ? outputHeight - outputY : 32U) : 0U; - if (final) - networkVideoMarkOutputRect( - decoder, component, outputX, outputY, width, height); - else - networkVideoMarkGridRect(&decoder->dirty[component][level - 1U], - outputWidth, outputHeight, - outputX, outputY, width, height); + networkVideoMarkGridRect(&decoder->dirty[component][level - 1U], + outputWidth, outputHeight, + outputX, outputY, width, height); } } } @@ -1676,10 +1661,6 @@ static void networkVideoIDWTNL(NetworkVideoDecoder * decoder, bool update) GL_TEXTURE_FETCH_BARRIER_BIT); } } - - if (!update) - memset(decoder->outputDirty, 1, - (size_t)decoder->outputTileWidth * decoder->outputTileHeight); } static void networkVideoColorRange( @@ -1716,120 +1697,125 @@ static void networkVideoColorRange( } } -static bool networkVideoTextureTileDirty( - const NetworkVideoDecoder * decoder, size_t index, - uint64_t textureGeneration) +static bool networkVideoDamageFull(const LGNetVideoDamage * damage) { - return decoder->outputDirty[index] || - decoder->outputTileGenerations[index] > textureGeneration; + return (damage->flags & LG_NET_VIDEO_DAMAGE_FULL) != 0; } -static void networkVideoBuildDamageNL(NetworkVideoDecoder * decoder, - NetworkVideoDecodedFrame * result) +static void networkVideoDispatchRect(GLint offsetUniform, + GLint extentUniform, uint32_t x, uint32_t y, + uint32_t width, uint32_t height) { - uint32_t count = 0; - bool overflow = false; - for (uint32_t tileY = 0; - tileY < decoder->outputTileHeight && !overflow; ++tileY) - { - uint32_t tileX = 0; - while (tileX < decoder->outputTileWidth) - { - const size_t row = (size_t)tileY * decoder->outputTileWidth; - while (tileX < decoder->outputTileWidth && - !decoder->outputDirty[row + tileX]) - ++tileX; - const uint32_t start = tileX; - while (tileX < decoder->outputTileWidth && - decoder->outputDirty[row + tileX]) - ++tileX; - if (start == tileX) - continue; + glUniform2i(offsetUniform, (GLint)x, (GLint)y); + glUniform2i(extentUniform, (GLint)width, (GLint)height); + glDispatchCompute((width + 7U) / 8U, (height + 7U) / 8U, 1); +} - const uint32_t x = start * NETWORK_VIDEO_TILE_SIZE; - const uint32_t y = tileY * NETWORK_VIDEO_TILE_SIZE; - const uint32_t endX = tileX * NETWORK_VIDEO_TILE_SIZE < - decoder->config.width ? - tileX * NETWORK_VIDEO_TILE_SIZE : decoder->config.width; - const uint32_t endY = y + NETWORK_VIDEO_TILE_SIZE < - decoder->config.height ? - y + NETWORK_VIDEO_TILE_SIZE : decoder->config.height; - bool merged = false; - for (uint32_t index = 0; index < count; ++index) - { - KVMFRFrameDamageRect * rect = &result->damageRects[index]; - if (rect->x == x && rect->width == endX - x && - rect->y + rect->height == y) - { - rect->height += endY - y; - merged = true; - break; - } - } - if (merged) - continue; - if (count + 1U >= LG_TRANSPORT_MAX_DAMAGE_RECTS) - { - overflow = true; - break; - } - result->damageRects[count++] = (KVMFRFrameDamageRect) - { - .x = x, - .y = y, - .width = endX - x, - .height = endY - y, - }; - } - } - - if (!overflow) +static void networkVideoDispatchDamageNL(NetworkVideoDecoder * decoder, + GLint offsetUniform, GLint extentUniform, + const LGNetVideoDamage * damage) +{ + if (networkVideoDamageFull(damage)) { - result->damageRectsCount = count; + networkVideoDispatchRect(offsetUniform, extentUniform, 0, 0, + decoder->config.width, decoder->config.height); return; } - uint32_t minX = decoder->config.width; - uint32_t minY = decoder->config.height; - uint32_t maxX = 0; - uint32_t maxY = 0; - for (uint32_t tileY = 0; tileY < decoder->outputTileHeight; ++tileY) + for (uint16_t index = 0; index < damage->count; ++index) { - for (uint32_t tileX = 0; tileX < decoder->outputTileWidth; ++tileX) - { - if (!decoder->outputDirty[ - (size_t)tileY * decoder->outputTileWidth + tileX]) - continue; - const uint32_t x = tileX * NETWORK_VIDEO_TILE_SIZE; - const uint32_t y = tileY * NETWORK_VIDEO_TILE_SIZE; - minX = x < minX ? x : minX; - minY = y < minY ? y : minY; - maxX = x + NETWORK_VIDEO_TILE_SIZE > maxX ? - x + NETWORK_VIDEO_TILE_SIZE : maxX; - maxY = y + NETWORK_VIDEO_TILE_SIZE > maxY ? - y + NETWORK_VIDEO_TILE_SIZE : maxY; - } + const LGNetVideoDamageRect * rect = &damage->rects[index]; + networkVideoDispatchRect(offsetUniform, extentUniform, + rect->x, rect->y, rect->width, rect->height); } - maxX = maxX < decoder->config.width ? maxX : decoder->config.width; - maxY = maxY < decoder->config.height ? maxY : decoder->config.height; - result->damageRects[0] = (KVMFRFrameDamageRect) +} + +static const LGNetVideoDamage * networkVideoFindDamageNL( + const NetworkVideoDecoder * decoder, uint64_t generation) +{ + const struct NetworkVideoDamageHistory * history = + &decoder->damageHistory[generation % NETWORK_VIDEO_DAMAGE_HISTORY]; + return history->generation == generation ? &history->damage : NULL; +} + +static bool networkVideoCopyNeedsFullNL( + const NetworkVideoDecoder * decoder, uint64_t textureGeneration, + uint64_t generation, const LGNetVideoDamage * damage) +{ + if (!textureGeneration || textureGeneration >= generation || + generation - textureGeneration > NETWORK_VIDEO_DAMAGE_HISTORY || + networkVideoDamageFull(damage)) + return true; + + for (uint64_t previous = textureGeneration + 1U; + previous < generation; ++previous) { - .x = minX, - .y = minY, - .width = maxX - minX, - .height = maxY - minY, - }; - result->damageRectsCount = 1; + const LGNetVideoDamage * prior = + networkVideoFindDamageNL(decoder, previous); + if (!prior || networkVideoDamageFull(prior)) + return true; + } + return false; +} + +static void networkVideoCopyCanonicalNL(NetworkVideoDecoder * decoder, + GLuint texture, uint64_t textureGeneration, uint64_t generation, + const LGNetVideoDamage * damage) +{ + glUseProgram(decoder->copyProgram); + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, decoder->canonicalTexture); + glBindImageTexture(0, texture, 0, GL_FALSE, 0, + GL_WRITE_ONLY, GL_RGBA16F); + + if (networkVideoCopyNeedsFullNL( + decoder, textureGeneration, generation, damage)) + { + networkVideoDispatchRect(decoder->uniforms.copy.outputOffset, + decoder->uniforms.copy.outputExtent, 0, 0, + decoder->config.width, decoder->config.height); + } + else + { + for (uint64_t previous = textureGeneration + 1U; + previous < generation; ++previous) + networkVideoDispatchDamageNL(decoder, + decoder->uniforms.copy.outputOffset, + decoder->uniforms.copy.outputExtent, + networkVideoFindDamageNL(decoder, previous)); + networkVideoDispatchDamageNL(decoder, + decoder->uniforms.copy.outputOffset, + decoder->uniforms.copy.outputExtent, damage); + } + + glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT | + GL_TEXTURE_FETCH_BARRIER_BIT); +} + +static void networkVideoBuildDamage(const LGNetVideoDamage * damage, + NetworkVideoDecodedFrame * result) +{ + if (networkVideoDamageFull(damage)) + return; + + for (uint16_t index = 0; index < damage->count; ++index) + { + const LGNetVideoDamageRect * source = &damage->rects[index]; + result->damageRects[index] = (KVMFRFrameDamageRect) + { + .x = source->x, + .y = source->y, + .width = source->width, + .height = source->height, + }; + } + result->damageRectsCount = damage->count; } static struct NetworkVideoExternalFrame * networkVideoConvertNL( - NetworkVideoDecoder * decoder, bool chromaLeft, uint64_t generation) + NetworkVideoDecoder * decoder, bool chromaLeft, uint64_t generation, + const LGNetVideoDamage * damage) { - struct NetworkVideoTexture pooled = - networkVideoTexturePoolAcquire(decoder->texturePool); - if (!pooled.texture) - return NULL; - glUseProgram(decoder->yuvProgram); for (unsigned component = 0; component < NETWORK_VIDEO_COMPONENTS; ++component) @@ -1852,46 +1838,21 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL( glUniform1f(decoder->uniforms.yuv.lumaScale, lumaScale); glUniform1f(decoder->uniforms.yuv.chromaMidpoint, chromaMidpoint); glUniform1f(decoder->uniforms.yuv.chromaScale, chromaScale); - glBindImageTexture(0, pooled.texture, 0, GL_FALSE, 0, + glBindImageTexture(0, decoder->canonicalTexture, 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGBA16F); - if (!pooled.generation) - { - glUniform2i(decoder->uniforms.yuv.outputOffset, 0, 0); - glDispatchCompute((decoder->config.width + 7U) / 8U, - (decoder->config.height + 7U) / 8U, 1); - } - else - { - for (uint32_t tileY = 0; tileY < decoder->outputTileHeight; ++tileY) - { - uint32_t tileX = 0; - while (tileX < decoder->outputTileWidth) - { - const size_t row = (size_t)tileY * decoder->outputTileWidth; - while (tileX < decoder->outputTileWidth && - !networkVideoTextureTileDirty( - decoder, row + tileX, pooled.generation)) - ++tileX; - const uint32_t start = tileX; - while (tileX < decoder->outputTileWidth && - networkVideoTextureTileDirty( - decoder, row + tileX, pooled.generation)) - ++tileX; - if (start == tileX) - continue; - const uint32_t pixelWidth = - (tileX - start) * NETWORK_VIDEO_TILE_SIZE; - glUniform2i(decoder->uniforms.yuv.outputOffset, - (GLint)(start * NETWORK_VIDEO_TILE_SIZE), - (GLint)(tileY * NETWORK_VIDEO_TILE_SIZE)); - glDispatchCompute((pixelWidth + 7U) / 8U, - NETWORK_VIDEO_TILE_SIZE / 8U, 1); - } - } - } + networkVideoDispatchDamageNL(decoder, + decoder->uniforms.yuv.outputOffset, + decoder->uniforms.yuv.outputExtent, damage); glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT | GL_TEXTURE_FETCH_BARRIER_BIT); + struct NetworkVideoTexture pooled = + networkVideoTexturePoolAcquire(decoder->texturePool); + if (!pooled.texture) + return NULL; + networkVideoCopyCanonicalNL(decoder, pooled.texture, + pooled.generation, generation, damage); + GLsync readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); glFlush(); if (!readySync || glGetError() != GL_NO_ERROR) @@ -1956,6 +1917,17 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( NETWORK_VIDEO_DECODE_UNSUPPORTED : NETWORK_VIDEO_DECODE_CORRUPT; } + LGNetVideoDamage damage = layout.damage; + if (!update) + { + damage = (LGNetVideoDamage) + { + .flags = (LGNetVideoDamageFlags)LG_NET_VIDEO_DAMAGE_FULL, + }; + } + const bool hasPresentationDamage = + networkVideoDamageFull(&damage) || damage.count; + uint32_t changedCount = 0; if (!networkVideoStagePacketsNL( decoder, data, size, update, &changedCount)) @@ -2004,10 +1976,15 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( networkVideoDequantNL(decoder, upload, update); upload->readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); networkVideoIDWTNL(decoder, update); - struct NetworkVideoExternalFrame * external = - networkVideoConvertNL(decoder, layout.chromaLeft, generation); + struct NetworkVideoExternalFrame * external = NULL; + if (hasPresentationDamage) + external = networkVideoConvertNL(decoder, + layout.chromaLeft, generation, &damage); + else + glFlush(); const uint64_t decodeTime = nanotime() - decodeStart; - const bool success = external && glGetError() == GL_NO_ERROR; + const bool success = + (!hasPresentationDamage || external) && glGetError() == GL_NO_ERROR; networkVideoRestoreCurrent(decoder, &previous); if (!success) { @@ -2021,14 +1998,19 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( } networkVideoCommitPendingPacketsNL(decoder); - const size_t outputTileCount = - (size_t)decoder->outputTileWidth * decoder->outputTileHeight; - for (size_t index = 0; index < outputTileCount; ++index) - if (decoder->outputDirty[index]) - decoder->outputTileGenerations[index] = generation; - decoder->outputGeneration = generation; decoder->referenceFrameID = complete ? frameID : 0; decoder->referenceValid = complete; + if (!hasPresentationDamage) + { + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_NO_FRAME; + } + + struct NetworkVideoDamageHistory * history = + &decoder->damageHistory[generation % NETWORK_VIDEO_DAMAGE_HISTORY]; + history->generation = generation; + history->damage = damage; + decoder->outputGeneration = generation; uint32_t formatVersion = (uint32_t)config->configEpoch; if (!formatVersion) @@ -2037,7 +2019,7 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( result->externalFrame = &external->base; result->decodeTimeNs = decodeTime; if (update) - networkVideoBuildDamageNL(decoder, result); + networkVideoBuildDamage(&damage, result); result->format = (LG_TransportFrameFormat) { .version = formatVersion, diff --git a/client/transports/Network/video/pyrowave.c b/client/transports/Network/video/pyrowave.c index 47254cbc..95f70f80 100644 --- a/client/transports/Network/video/pyrowave.c +++ b/client/transports/Network/video/pyrowave.c @@ -247,11 +247,11 @@ NetworkPyroWaveResult networkPyroWaveParse( size_t cursor = 0; uint32_t uniqueBlocks = 0; uint32_t declaredBlocks = 0; - uint32_t headerFirst = 0; uint32_t headerSecond = 0; uint8_t frameSequence = 0; bool haveSequence = false; bool haveHeader = false; + bool haveDamage = false; bool haveClear = false; bool valid = true; @@ -285,21 +285,39 @@ NetworkPyroWaveResult networkPyroWaveParse( width != config->width || height != config->height || !blocks || blocks > capacity || !networkPyroWaveSequenceProfile(config, second) || - (haveHeader && - (first != headerFirst || second != headerSecond))) + haveHeader) { valid = false; break; } declaredBlocks = blocks; - headerFirst = first; headerSecond = second; haveHeader = true; cursor += NETWORK_PYROWAVE_HEADER_SIZE; + + if (code == LG_NET_VIDEO_PYROWAVE_MODE_DELTA) + { + size_t damageSize = 0; + if (lgNetVideoDamageDecode(&layout->damage, + bytes + cursor, size - cursor, + config->width, config->height, &damageSize) != LG_NET_PARSE_OK) + { + valid = false; + break; + } + cursor += damageSize; + haveDamage = true; + } continue; } + if (!haveHeader) + { + valid = false; + break; + } + const uint32_t ballot = first & UINT32_C(0x0000FFFF); const uint32_t payloadWords = (first >> 16) & UINT32_C(0x00000FFF); const uint32_t blockIndex = second >> 8; @@ -322,20 +340,19 @@ NetworkPyroWaveResult networkPyroWaveParse( cursor += packetSize; } + const bool delta = + ((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) & + LG_NET_VIDEO_PYROWAVE_MODE_MASK) == + LG_NET_VIDEO_PYROWAVE_MODE_DELTA; valid = valid && cursor == size && haveHeader && haveSequence && + haveDamage == delta && declaredBlocks == uniqueBlocks && expectedBlocks == uniqueBlocks && - (!haveClear || - ((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) & - LG_NET_VIDEO_PYROWAVE_MODE_MASK) == - LG_NET_VIDEO_PYROWAVE_MODE_DELTA); + (!haveClear || delta); if (!valid) return NETWORK_PYROWAVE_INVALID; layout->sequence = frameSequence; layout->chromaLeft = (headerSecond >> 31) != 0; - layout->delta = - ((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) & - LG_NET_VIDEO_PYROWAVE_MODE_MASK) == - LG_NET_VIDEO_PYROWAVE_MODE_DELTA; + layout->delta = delta; return NETWORK_PYROWAVE_OK; } diff --git a/client/transports/Network/video/pyrowave.h b/client/transports/Network/video/pyrowave.h index e024a1bc..d57bba06 100644 --- a/client/transports/Network/video/pyrowave.h +++ b/client/transports/Network/video/pyrowave.h @@ -36,9 +36,10 @@ NetworkPyroWaveResult; typedef struct NetworkPyroWaveLayout { - uint8_t sequence; - bool chromaLeft; - bool delta; + uint8_t sequence; + bool chromaLeft; + bool delta; + LGNetVideoDamage damage; } NetworkPyroWaveLayout; diff --git a/client/transports/Network/video/shaders.h b/client/transports/Network/video/shaders.h index fb0d70f8..8ebb08d5 100644 --- a/client/transports/Network/video/shaders.h +++ b/client/transports/Network/video/shaders.h @@ -308,6 +308,7 @@ static const char NETWORK_VIDEO_YUV_SHADER[] = "layout(rgba16f,binding=0) writeonly uniform highp image2D outputImage;\n" "uniform ivec2 outputSize;\n" "uniform ivec2 outputOffset;\n" + "uniform ivec2 outputExtent;\n" "uniform bool bt2020;\n" "uniform bool chromaLeft;\n" "uniform float lumaOffset;\n" @@ -315,7 +316,9 @@ static const char NETWORK_VIDEO_YUV_SHADER[] = "uniform float chromaMidpoint;\n" "uniform float chromaScale;\n" "void main(){\n" - " ivec2 pixel = ivec2(gl_GlobalInvocationID.xy)+outputOffset;\n" + " ivec2 local = ivec2(gl_GlobalInvocationID.xy);\n" + " if(any(greaterThanEqual(local,outputExtent)))return;\n" + " ivec2 pixel = local+outputOffset;\n" " if(any(greaterThanEqual(pixel,outputSize)))return;\n" " ivec2 planeSize = textureSize(lumaPlane,0);\n" " vec2 uv = (vec2(pixel)+0.5)/vec2(planeSize);\n" @@ -335,6 +338,21 @@ static const char NETWORK_VIDEO_YUV_SHADER[] = " vec3 rgb = clamp(m*vec3(y,cb,cr),vec3(0.0),vec3(1.0));\n" " imageStore(outputImage,pixel,vec4(rgb,1.0));}\n"; +static const char NETWORK_VIDEO_COPY_SHADER[] = + "#version 310 es\n" + "precision highp float;\n" + "precision highp int;\n" + "layout(local_size_x=8,local_size_y=8) in;\n" + "layout(binding=0) uniform highp sampler2D inputImage;\n" + "layout(rgba16f,binding=0) writeonly uniform highp image2D outputImage;\n" + "uniform ivec2 outputOffset;\n" + "uniform ivec2 outputExtent;\n" + "void main(){\n" + " ivec2 local = ivec2(gl_GlobalInvocationID.xy);\n" + " if(any(greaterThanEqual(local,outputExtent)))return;\n" + " ivec2 pixel = local+outputOffset;\n" + " imageStore(outputImage,pixel,texelFetch(inputImage,pixel,0));}\n"; + static const char * const NETWORK_VIDEO_DEQUANT_SOURCES[] = { NETWORK_VIDEO_DEQUANT_SHADER_0, @@ -351,3 +369,8 @@ static const char * const NETWORK_VIDEO_YUV_SOURCES[] = { NETWORK_VIDEO_YUV_SHADER, }; + +static const char * const NETWORK_VIDEO_COPY_SOURCES[] = +{ + NETWORK_VIDEO_COPY_SHADER, +}; diff --git a/common/include/common/types.h b/common/include/common/types.h index d3f4f9ca..625dbeed 100644 --- a/common/include/common/types.h +++ b/common/include/common/types.h @@ -25,6 +25,8 @@ #include +#define LG_MAX_RUNTIME_DAMAGE_RECTS 256 + typedef void (*LG_FrameReleaseFn)(void * opaque, uint64_t handle); struct Point diff --git a/idd/LGIdd/transport/network/CNetworkVideo.cpp b/idd/LGIdd/transport/network/CNetworkVideo.cpp index 27705d8a..b1105f0c 100644 --- a/idd/LGIdd/transport/network/CNetworkVideo.cpp +++ b/idd/LGIdd/transport/network/CNetworkVideo.cpp @@ -83,8 +83,11 @@ namespace constexpr uint32_t PYROWAVE_ALIGNMENT = 1U << PYROWAVE_LEVELS; constexpr uint32_t PYROWAVE_MIN_DIMENSION = 4U << PYROWAVE_LEVELS; constexpr uint32_t PYROWAVE_PACKET_BYTES = 2488; + constexpr uint32_t PYROWAVE_DAMAGE_BYTES_MAX = + LG_NET_VIDEO_DAMAGE_HEADER_WIRE_SIZE + + LG_NET_VIDEO_MAX_DAMAGE_RECTS * LG_NET_VIDEO_DAMAGE_RECT_WIRE_SIZE; constexpr uint32_t PYROWAVE_BLOCK_BYTES_MAX = - PYROWAVE_PACKET_BYTES + 8U; + PYROWAVE_PACKET_BYTES + 8U + PYROWAVE_DAMAGE_BYTES_MAX; // Reserve an equal fragment budget for every possible codec block so a // negotiated delivery path can always represent the encoded frame. constexpr uint32_t PYROWAVE_FRAGS_PER_BLOCK = @@ -93,7 +96,8 @@ namespace (PYROWAVE_BLOCK_BYTES_MAX + PYROWAVE_FRAGS_PER_BLOCK - 1U) / PYROWAVE_FRAGS_PER_BLOCK; constexpr uint32_t PYROWAVE_FRAGMENT_MIN = - LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE + PYROWAVE_FRAGMENT_DATA_MIN; + LG_NET_VIDEO_FRAME_UPDATE_FRAGMENT_HEADER_WIRE_SIZE + + PYROWAVE_FRAGMENT_DATA_MIN; constexpr uint32_t PYROWAVE_PACKET_META_WORDS = 2; constexpr uint32_t PYROWAVE_SEQUENCE_MASK = 7; constexpr uint32_t PYROWAVE_HEADER_CODE_MASK = @@ -135,6 +139,8 @@ namespace static_assert(PYROWAVE_BLOCK_BYTES_MAX <= PYROWAVE_FRAGS_PER_BLOCK * LG_NET_VIDEO_MAX_FRAGMENT_LENGTH, "The fragment payload limit must represent every codec block"); + static_assert(LG_NET_VIDEO_MAX_DAMAGE_RECTS >= FRAME_DAMAGE_MAX, + "The network protocol must represent every capture damage rectangle"); constexpr LGNetEnvelopeFlags DELIVERY_MASK = LG_NET_ENVELOPE_RESPONSE | LG_NET_ENVELOPE_FINAL | @@ -1502,6 +1508,155 @@ void FinalizeMain() uint32_t index = 0; }; + bool VideoDamageContains(const LGNetVideoDamageRect& outer, + const LGNetVideoDamageRect& inner) + { + return outer.x <= inner.x && outer.y <= inner.y && + static_cast(outer.x) + outer.width >= + static_cast(inner.x) + inner.width && + static_cast(outer.y) + outer.height >= + static_cast(inner.y) + inner.height; + } + + bool MergeVideoDamageRect(const LGNetVideoDamageRect& first, + const LGNetVideoDamageRect& second, LGNetVideoDamageRect& merged) + { + const uint32_t firstRight = + static_cast(first.x) + first.width; + const uint32_t secondRight = + static_cast(second.x) + second.width; + const uint32_t firstBottom = + static_cast(first.y) + first.height; + const uint32_t secondBottom = + static_cast(second.y) + second.height; + if (first.y == second.y && first.height == second.height && + first.x <= secondRight && second.x <= firstRight) + { + const uint32_t left = (std::min)(first.x, second.x); + const uint32_t right = (std::max)(firstRight, secondRight); + merged.x = static_cast(left); + merged.y = first.y; + merged.width = static_cast(right - left); + merged.height = first.height; + return true; + } + if (first.x == second.x && first.width == second.width && + first.y <= secondBottom && second.y <= firstBottom) + { + const uint32_t top = (std::min)(first.y, second.y); + const uint32_t bottom = (std::max)(firstBottom, secondBottom); + merged.x = first.x; + merged.y = static_cast(top); + merged.width = first.width; + merged.height = static_cast(bottom - top); + return true; + } + return false; + } + + bool AddVideoDamageRect(LGNetVideoDamage& damage, + const LGNetVideoDamageRect& rect) + { + LGNetVideoDamageRect current = rect; + for (uint16_t index = 0; index < damage.count;) + { + const LGNetVideoDamageRect existing = damage.rects[index]; + if (VideoDamageContains(existing, current)) + return true; + LGNetVideoDamageRect merged = {}; + if (!VideoDamageContains(current, existing) && + !MergeVideoDamageRect(existing, current, merged)) + { + ++index; + continue; + } + if (VideoDamageContains(current, existing)) + merged = current; + current = merged; + --damage.count; + damage.rects[index] = damage.rects[damage.count]; + index = 0; + } + + if (damage.count == LG_NET_VIDEO_MAX_DAMAGE_RECTS) + return false; + damage.rects[damage.count++] = current; + return true; + } + + void SetFullVideoDamage(LGNetVideoDamage& damage) + { + damage = {}; + damage.flags = static_cast( + LG_NET_VIDEO_DAMAGE_FULL); + } + + bool SetVideoDamage(const FrameDesc& frame, uint32_t width, + uint32_t height, LGNetVideoDamage& damage) + { + damage = {}; + if (frame.damage == FrameDamage::NONE) + return true; + if (frame.damage == FrameDamage::FULL) + { + SetFullVideoDamage(damage); + return true; + } + if (frame.damage != FrameDamage::RECTS || !frame.count || + frame.count > FRAME_DAMAGE_MAX) + return false; + + for (unsigned index = 0; index < frame.count; ++index) + { + const RECT& rect = frame.rects[index]; + if (rect.left < 0 || rect.top < 0 || rect.right <= rect.left || + rect.bottom <= rect.top || + static_cast(rect.right) > width || + static_cast(rect.bottom) > height) + { + SetFullVideoDamage(damage); + return true; + } + const LGNetVideoDamageRect output = + { + static_cast(rect.left), + static_cast(rect.top), + static_cast(rect.right - rect.left), + static_cast(rect.bottom - rect.top), + }; + if (!AddVideoDamageRect(damage, output)) + { + SetFullVideoDamage(damage); + return true; + } + } + return lgNetVideoDamageValid(&damage, width, height); + } + + void MergeVideoDamage(LGNetVideoDamage& destination, + const LGNetVideoDamage& source) + { + if ((destination.flags & LG_NET_VIDEO_DAMAGE_FULL) || + (!source.flags && !source.count)) + return; + if (source.flags & LG_NET_VIDEO_DAMAGE_FULL) + { + SetFullVideoDamage(destination); + return; + } + + bool complete = true; + for (uint16_t index = 0; index < source.count; ++index) + if (!AddVideoDamageRect(destination, source.rects[index])) + { + complete = false; + break; + } + if (complete) + return; + SetFullVideoDamage(destination); + } + struct EncodedFrame { uint64_t configEpoch = 0; @@ -1511,6 +1666,7 @@ void FinalizeMain() uint32_t deadlineMs = 0; uint32_t blockCapacity = 0; uint32_t hardLimit = 0; + LGNetVideoDamage damage = {}; std::vector damagedBlocks; std::vector data; std::vector blocks; @@ -3317,7 +3473,11 @@ void FinalizeMain() output = {}; - const bool result = Compact(*slot, slot->mapped, output); + bool result = Compact(*slot, slot->mapped, output); + + if (result) + result = SetVideoDamage(slot->frame.desc.frame, + m_cfg.width, m_cfg.height, output.damage); if (result) { @@ -3389,8 +3549,9 @@ struct CNetworkVideo::Impl { struct ReferenceFrame { - uint64_t frameID = 0; + uint64_t frameID = 0; std::shared_ptr frame; + LGNetVideoDamage subsequentDamage = {}; std::vector subsequentBlocks; }; @@ -3416,6 +3577,7 @@ struct CNetworkVideo::Impl uint8_t blockedDeliveries = 0; std::shared_ptr referenceFrame; std::deque referenceHistory; + LGNetVideoDamage updateDamage = {}; std::vector updateBlocks; bool statusSent = false; bool subscribed = false; @@ -3604,10 +3766,21 @@ struct CNetworkVideo::Impl peer.referenceFrame.reset(); peer.referenceHistory.clear(); peer.updateBlocks.clear(); + peer.updateDamage = {}; peer.referenceFrameID = 0; peer.referencePending = false; } + static void AccumulatePeerDamageNL(PeerState& peer, + const LGNetVideoDamage& damage) + { + if (!peer.referenceFrame) + return; + MergeVideoDamage(peer.updateDamage, damage); + for (ReferenceFrame& reference : peer.referenceHistory) + MergeVideoDamage(reference.subsequentDamage, damage); + } + static bool PeerNeedsReferenceNL( const PeerState& peer, const EncodedFrame& frame) { @@ -3780,6 +3953,7 @@ struct CNetworkVideo::Impl result->deadlineMs = current.deadlineMs; result->blockCapacity = current.blockCapacity; result->hardLimit = current.hardLimit; + result->damage = current.damage; result->damagedBlocks = std::move(current.damagedBlocks); result->blocks.reserve((std::max)( previous->blocks.size(), current.blocks.size())); @@ -3917,6 +4091,7 @@ struct CNetworkVideo::Impl static std::shared_ptr BuildUpdateFrame( const EncodedFrame& reference, const EncodedFrame& current, const std::vector& previousBlocks, + const LGNetVideoDamage& damage, uint32_t width, uint32_t height, std::vector& updateBlocks) { try @@ -3947,9 +4122,14 @@ struct CNetworkVideo::Impl update->captureTimeNs = current.captureTimeNs; update->deadlineMs = current.deadlineMs; update->blockCapacity = current.blockCapacity; + update->damage = damage; update->blocks.reserve(operationCount); - update->data.reserve(current.data.size()); - update->data.resize(8U); + const size_t damageSize = lgNetVideoDamageSize(&damage); + if (!damageSize || + damageSize > LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH - 8U) + return {}; + update->data.reserve(current.data.size() + damageSize); + update->data.resize(8U + damageSize); const uint32_t firstWord = LoadLE32(current.data.data()); const uint32_t secondWord = @@ -3958,6 +4138,9 @@ struct CNetworkVideo::Impl PYROWAVE_HEADER_CODE_DELTA | operationCount; StoreLE32(update->data.data(), firstWord); StoreLE32(update->data.data() + sizeof(uint32_t), secondWord); + if (!lgNetVideoDamageEncode(update->data.data() + 8U, + damageSize, &damage, width, height)) + return {}; const uint32_t sequence = (firstWord >> 28U) & PYROWAVE_SEQUENCE_MASK; @@ -4007,7 +4190,7 @@ struct CNetworkVideo::Impl return {}; update->blocks[0].offset = 0; - update->blocks[0].length += 8U; + update->blocks[0].length += static_cast(8U + damageSize); return update; } catch (...) @@ -4096,14 +4279,20 @@ struct CNetworkVideo::Impl if (candidate->subsequentBlocks.size() != promotedFrame->blockCapacity) return false; - std::vector subsequentBlocks = + std::vector subsequentBlocks = std::move(candidate->subsequentBlocks); + const LGNetVideoDamage subsequentDamage = candidate->subsequentDamage; peer.referenceFrame = promotedFrame; peer.referenceFrameID = promotedID; peer.referencePending = false; peer.keyframePending = false; + peer.updateDamage = subsequentDamage; peer.updateBlocks = std::move(subsequentBlocks); + // Damage also accumulates across frames intentionally skipped while the + // reference is pending, even though no block comparison is done for them. + // BuildUpdateFrame will authoritatively compare against the promoted + // reference and clear the damage if the encoded state did not change. while (!peer.referenceHistory.empty() && peer.referenceHistory.front().frameID <= promotedID) peer.referenceHistory.pop_front(); @@ -4808,6 +4997,7 @@ struct CNetworkVideo::Impl peer.referenceFrame = frame; peer.referenceFrameID = frameID; peer.referencePending = true; + peer.updateDamage = {}; peer.updateBlocks = std::move(cleanBlocks); RecordReferenceFrameNL(peer, frameID, frame); } @@ -4869,10 +5059,14 @@ struct CNetworkVideo::Impl if (PeerNeedsReferenceNL(peer, *current)) return QueueFrameNL( peer, current, frameID, presentationBaseNs, true); + if (peer.updateDamage.flags & LG_NET_VIDEO_DAMAGE_FULL) + return QueueFrameNL( + peer, current, frameID, presentationBaseNs, true); DropPeerFramesNL(peer); std::vector updateBlocks; const std::shared_ptr update = BuildUpdateFrame( *peer.referenceFrame, *current, peer.updateBlocks, + peer.updateDamage, m_activeCfg.width, m_activeCfg.height, updateBlocks); const bool compared = updateBlocks.size() == current->blockCapacity; const bool changed = std::any_of(updateBlocks.begin(), @@ -4882,6 +5076,7 @@ struct CNetworkVideo::Impl if (compared && !changed) { peer.lastCaptureTimeNs = current->captureTimeNs; + peer.updateDamage = {}; return true; } return QueueFrameNL( @@ -5014,14 +5209,17 @@ struct CNetworkVideo::Impl encoded->sourceSerial == m_latestSourceSerial) m_latestFrame = encoded; for (PeerState& peer : m_peers) - if (peer.subscribed && peer.configured && - PeerFrameDueNL(peer, *encoded)) + if (peer.subscribed && peer.configured) { - if (!QueueCurrentFrameNL(peer, encoded, - frameID, encoded->captureTimeNs)) - QueueStatusNL(peer, LG_NET_VIDEO_STATE_STALLED, 2); - else - QueueStatusNL(peer, LG_NET_VIDEO_STATE_STREAMING, 0); + AccumulatePeerDamageNL(peer, encoded->damage); + if (PeerFrameDueNL(peer, *encoded)) + { + if (!QueueCurrentFrameNL(peer, encoded, + frameID, encoded->captureTimeNs)) + QueueStatusNL(peer, LG_NET_VIDEO_STATE_STALLED, 2); + else + QueueStatusNL(peer, LG_NET_VIDEO_STATE_STREAMING, 0); + } } } diff --git a/repos/LGProtocol b/repos/LGProtocol index a8f5a214..7913a08a 160000 --- a/repos/LGProtocol +++ b/repos/LGProtocol @@ -1 +1 @@ -Subproject commit a8f5a21480472777d4f61ed55202fde03b2f0fc9 +Subproject commit 7913a08abad9d45e8c9954adff7360d35511ed27