[client/idd] network: process only damaged PyroWave blocks

Limit GPU encoding and client decoding to damage-derived regions.
Preserve per-level sparse transform state and cover mirrored inverse
edges.

Keep cumulative canonical packets so dropped frames remain recoverable.

client/transports/Network/video.c

@@ -286,6 +286,66 @@ static void networkVideoFreeOutput(struct NetworkVideoOutput ** output)
   *output = NULL;
 }
 
+static void networkVideoMergeDamage(NetworkVideoDecodedFrame * destination,
+    const NetworkVideoDecodedFrame * source)
+{
+  if (!destination || !source)
+    return;
+
+  if (!destination->damageRectsCount || !source->damageRectsCount ||
+      destination->damageRectsCount > LG_TRANSPORT_MAX_DAMAGE_RECTS ||
+      source->damageRectsCount > LG_TRANSPORT_MAX_DAMAGE_RECTS)
+  {
+    destination->damageRectsCount = 0;
+    return;
+  }
+
+  if (destination->damageRectsCount >=
+      LG_TRANSPORT_MAX_DAMAGE_RECTS - source->damageRectsCount)
+  {
+    uint32_t left   = UINT32_MAX;
+    uint32_t top    = UINT32_MAX;
+    uint64_t right  = 0;
+    uint64_t bottom = 0;
+    for (unsigned set = 0; set < 2; ++set)
+    {
+      const KVMFRFrameDamageRect * rects = set ?
+        source->damageRects : destination->damageRects;
+      const uint32_t               count = set ?
+        source->damageRectsCount : destination->damageRectsCount;
+      for (uint32_t index = 0; index < count; ++index)
+      {
+        left   = rects[index].x < left ? rects[index].x : left;
+        top    = rects[index].y < top ? rects[index].y : top;
+        right  = (uint64_t)rects[index].x + rects[index].width > right ?
+          (uint64_t)rects[index].x + rects[index].width : right;
+        bottom = (uint64_t)rects[index].y + rects[index].height > bottom ?
+          (uint64_t)rects[index].y + rects[index].height : bottom;
+      }
+    }
+    if (right <= left || bottom <= top || right - left > UINT32_MAX ||
+        bottom - top > UINT32_MAX)
+    {
+      destination->damageRectsCount = 0;
+      return;
+    }
+    destination->damageRects[0] = (KVMFRFrameDamageRect)
+    {
+      .x      = left,
+      .y      = top,
+      .width  = (uint32_t)(right - left),
+      .height = (uint32_t)(bottom - top),
+    };
+    destination->damageRectsCount = 1;
+    return;
+  }
+
+  memcpy(destination->damageRects + destination->damageRectsCount,
+      source->damageRects,
+      source->damageRectsCount * sizeof(*source->damageRects));
+  destination->damageRectsCount += source->damageRectsCount;
+}
+
 static void networkVideoClearEncoded(struct NetworkVideoEncoded * encoded)
 {
   if (!encoded)
@@ -902,7 +962,8 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video,
     if (assembly->active)
     {
       if (assembly->update && update &&
-          assembly->baseFrameID != baseFrameID)
+          assembly->baseFrameID != baseFrameID &&
+          (!baseFrameID || baseFrameID > video->lastDecodedFrameID))
         return false;
       video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames);
       if (!assembly->update && !update && *encodedCount < 2U &&
@@ -1080,26 +1141,47 @@ static void networkVideoDecode(NetworkVideo * video,
     return;
   }
 
-  NetworkVideoDecodedFrame       decoded;
-  const NetworkVideoDecodeResult result = networkVideoDecoderDecode(
+  struct NetworkVideoOutput * output = calloc(1, sizeof(*output));
+  if (!output)
+  {
+    free(encoded->ownedData);
+    networkVideoDecodeFailure(
+        video, encoded, NETWORK_VIDEO_DECODE_ERROR);
+    return;
+  }
+
+  NetworkVideoDecodedFrame       decoded = {0};
+  const NetworkVideoDecodeResult result  = networkVideoDecoderDecode(
       video->decoder, &encoded->config, encoded->data,
       encoded->length, encoded->blockCount, encoded->update,
       encoded->complete, encoded->baseFrameID, encoded->frameID, &decoded);
   free(encoded->ownedData);
+  if (result == NETWORK_VIDEO_DECODE_NO_FRAME)
+  {
+    free(output);
+    LG_LOCK(video->stateLock);
+    if (video->decodeGeneration == encoded->generation &&
+        video->wireConfigured &&
+        video->config.configEpoch == encoded->config.configEpoch &&
+        video->rendererAttached && video->decoderReady &&
+        encoded->frameID > video->lastDecodedFrameID)
+    {
+      if (encoded->complete)
+        video->lastCompleteFrameID = encoded->frameID;
+      video->lastDecodedFrameID    = encoded->frameID;
+      networkVideoQueueFeedbackNL(video, 0);
+    }
+    LG_UNLOCK(video->stateLock);
+    return;
+  }
   if (result != NETWORK_VIDEO_DECODE_OK)
   {
+    free(output);
     networkVideoDecodeFailure(video, encoded, result);
     return;
   }
 
-  struct NetworkVideoOutput * output   = calloc(1, sizeof(*output));
   struct NetworkVideoOutput * replaced = NULL;
-  if (!output)
-  {
-    networkVideoDecoderRelease(&decoded);
-    networkVideoDecodeFailure(video, encoded, NETWORK_VIDEO_DECODE_ERROR);
-    return;
-  }
   output->decoded                 = decoded;
   output->frameID                 = encoded->frameID;
   output->timestamp               = encoded->presentationTimestampNs;
@@ -1122,6 +1204,7 @@ static void networkVideoDecode(NetworkVideo * video,
   {
     video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames);
     replaced          = video->output;
+    networkVideoMergeDamage(&output->decoded, &replaced->decoded);
   }
   if (encoded->complete)
     video->lastCompleteFrameID   = output->frameID;
@@ -1150,13 +1233,12 @@ static void networkVideoQueueDecode(NetworkVideo * video,
   bool                       queued      = false;
   bool                       replaceable;
   LG_LOCK(video->stateLock);
-  baseMismatch = encoded->update && video->decodePending &&
-    video->pendingDecode.update &&
-    video->pendingDecode.baseFrameID != encoded->baseFrameID;
+  baseMismatch = encoded->update &&
+    (!encoded->baseFrameID ||
+      encoded->baseFrameID > video->lastDecodedFrameID);
   replaceable  = !video->decodePending ||
     !encoded->update ||
-    (video->pendingDecode.update &&
-      video->pendingDecode.baseFrameID == encoded->baseFrameID);
+    (video->pendingDecode.update && !baseMismatch);
   if (baseMismatch)
   {
     video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames);
@@ -1494,8 +1576,8 @@ static bool networkVideoHandleFrameUpdate(NetworkVideo * video,
       video->assembly.frameID <= update.frameID)
   {
     video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames);
-    baseMismatch      = video->assembly.update &&
-      video->assembly.baseFrameID != update.baseFrameID;
+    baseMismatch      = !update.baseFrameID ||
+      update.baseFrameID > video->lastDecodedFrameID;
     retiredAssembly   = networkVideoTakeAssemblyNL(video);
     if (baseMismatch)
       networkVideoRequestKeyframeNL(video,
@@ -2234,6 +2316,9 @@ LG_TransportStatus networkVideoNextFrame(NetworkVideo * video, bool useDMA,
         frame->format                 = &output->decoded.format;
         frame->externalFrame          =
           lgExternalFrameRetain(output->decoded.externalFrame);
+        frame->damageRects            = output->decoded.damageRectsCount ?
+          output->decoded.damageRects : NULL;
+        frame->damageRectsCount       = output->decoded.damageRectsCount;
         frame->releaseFn              = networkVideoReleaseLease;
         frame->releaseOpaque          = video;
         frame->releaseHandle          = output->handle;

client/transports/Network/video/decoder.h

@@ -35,6 +35,7 @@ typedef struct NetworkVideoDecoder NetworkVideoDecoder;
 typedef enum NetworkVideoDecodeResult
 {
   NETWORK_VIDEO_DECODE_OK,
+  NETWORK_VIDEO_DECODE_NO_FRAME,
   NETWORK_VIDEO_DECODE_UNSUPPORTED,
   NETWORK_VIDEO_DECODE_CORRUPT,
   NETWORK_VIDEO_DECODE_ERROR,
@@ -43,10 +44,12 @@ NetworkVideoDecodeResult;
 
 typedef struct NetworkVideoDecodedFrame
 {
-  LG_ExternalFrame          * externalFrame;
-  LG_TransportFrameFormat     format;
-  LG_TransportFrameTiming     timing;
-  uint64_t                    decodeTimeNs;
+  LG_ExternalFrame         * externalFrame;
+  LG_TransportFrameFormat    format;
+  LG_TransportFrameTiming    timing;
+  KVMFRFrameDamageRect       damageRects[LG_TRANSPORT_MAX_DAMAGE_RECTS];
+  uint32_t                   damageRectsCount;
+  uint64_t                   decodeTimeNs;
 }
 NetworkVideoDecodedFrame;
 

client/transports/Network/video/shaders.h

@@ -55,6 +55,7 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_0[] =
   "layout(std430,binding=0) readonly buffer OffsetBuffer { uint offsets[]; };\n"
   "layout(std430,binding=1) readonly buffer PayloadBuffer { uint payload[]; };\n"
   "uniform ivec2 resolution;\n"
+  "uniform ivec2 workGroupOffset;\n"
   "uniform int outputLayer;\n"
   "uniform int blockOffset;\n"
   "uniform int blockStride;\n"
@@ -88,21 +89,22 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_0[] =
   "  }\n"
   "}\n"
   "void main() {\n"
-  "  uint localIndex   = gl_LocalInvocationIndex;\n"
-  "  int  blockIndex   = blockOffset + int(gl_WorkGroupID.y) * blockStride +\n"
-  "    int(gl_WorkGroupID.x);\n"
-  "  uint sourceWord   = offsets[blockIndex];\n"
-  "  uint blockPresent = uint(sourceWord != 0xffffffffu);\n"
+  "  uint  localIndex   = gl_LocalInvocationIndex;\n"
+  "  ivec2 logicalGroup = ivec2(gl_WorkGroupID.xy) + workGroupOffset;\n"
+  "  int   blockIndex   = blockOffset + logicalGroup.y * blockStride +\n"
+  "    logicalGroup.x;\n"
+  "  uint  sourceWord   = offsets[blockIndex];\n"
+  "  uint  blockPresent = uint(sourceWord != 0xffffffffu);\n"
   "  // Missing blocks must still reach every workgroup barrier.\n"
   "  sourceWord *= blockPresent;\n"
-  "  uint  subblock = bitfieldExtract(localIndex,0,3);\n"
-  "  uint  blockX   = bitfieldExtract(localIndex,3,2);\n"
-  "  uint  blockY   = bitfieldExtract(localIndex,5,2);\n"
-  "  uint  group    = blockY * 4u + blockX;\n"
-  "  ivec2 coord    = ivec2(gl_WorkGroupID.xy) * 32 +\n"
+  "  uint  subblock   = bitfieldExtract(localIndex,0,3);\n"
+  "  uint  blockX     = bitfieldExtract(localIndex,3,2);\n"
+  "  uint  blockY     = bitfieldExtract(localIndex,5,2);\n"
+  "  uint  group      = blockY * 4u + blockX;\n"
+  "  ivec2 coord      = logicalGroup * 32 +\n"
   "    8 * ivec2(blockX,blockY) + unswizzle8x8(subblock << 3u);\n"
-  "  uint sourceByte = sourceWord * 4u;\n"
-  "  uint ballot     = (payload[sourceWord] & 65535u) * blockPresent;\n"
+  "  uint  sourceByte = sourceWord * 4u;\n"
+  "  uint  ballot     = (payload[sourceWord] & 65535u) * blockPresent;\n"
   "  if (localIndex == 0u) {\n"
   "    uint cursor = sourceByte + 8u + 3u * uint(bitCount(ballot));\n"
   "    for (uint index=0u; index<16u; ++index) {\n"
@@ -192,6 +194,7 @@ static const char NETWORK_VIDEO_IDWT_SHADER_0[] =
   "layout(binding=0) uniform highp sampler2DArray coefficients;\n"
   "layout(r32f,binding=1) writeonly uniform highp image2D outputImage;\n"
   "uniform ivec2 resolution;\n"
+  "uniform ivec2 workGroupOffset;\n"
   "uniform vec2 invResolution;\n"
   "uniform int baseLayer;\n"
   "uniform bool dcShift;\n"
@@ -230,8 +233,10 @@ static const char NETWORK_VIDEO_IDWT_SHADER_0[] =
   "  sharedBlock[coord.y+1][2*coord.x+3]=vec2(b.w,d.w);\n"
   "}\n"
   "void loadApron(uint index) {\n"
-  "  ivec2 base=ivec2(gl_WorkGroupID.xy)*HALF_BLOCK-HALF_APRON;\n"
-  "  ivec2 local=2*unswizzle8x8(index); ivec2 coord=base+local;\n"
+  "  ivec2 workGroup = ivec2(gl_WorkGroupID.xy)+workGroupOffset;\n"
+  "  ivec2 base      = workGroup*HALF_BLOCK-HALF_APRON;\n"
+  "  ivec2 local     = 2*unswizzle8x8(index);\n"
+  "  ivec2 coord     = base+local;\n"
   "  store4(local,gatherBand(coord,true,true,0).wxzy,\n"
   "    gatherBand(coord,false,true,2).wxzy,\n"
   "    gatherBand(coord,true,false,1).wxzy,\n"
@@ -280,11 +285,13 @@ static const char NETWORK_VIDEO_IDWT_SHADER_1[] =
   "}\n"
   "void main(){uint index=gl_LocalInvocationIndex;loadApron(index);\n"
   "  inverse8x2(index);inverse4x2(index,index<32u,HALF_BLOCK);barrier();\n"
-  "  inverse8x2(index);barrier();ivec2 local=unswizzle8x8(index);\n"
-  "  ivec2 outputSize=imageSize(outputImage);\n"
+  "  inverse8x2(index);barrier();\n"
+  "  ivec2 local      = unswizzle8x8(index);\n"
+  "  ivec2 outputSize = imageSize(outputImage);\n"
+  "  ivec2 workGroup  = ivec2(gl_WorkGroupID.xy)+workGroupOffset;\n"
   "  for(int y=local.y;y<HALF_BLOCK;y+=8)for(int x=local.x;x<BLOCK;x+=8){\n"
   "    vec2 value=sharedBlock[y][x];if(dcShift)value+=vec2(0.5);\n"
-  "    ivec2 a=ivec2(2*y,x)+BLOCK*ivec2(gl_WorkGroupID.yx);\n"
+  "    ivec2 a=ivec2(2*y,x)+BLOCK*workGroup.yx;\n"
   "    ivec2 b=a+ivec2(1,0);if(all(lessThan(a,outputSize)))\n"
   "      imageStore(outputImage,a,vec4(value.x));\n"
   "    if(all(lessThan(b,outputSize)))imageStore(outputImage,b,vec4(value.y));}\n"
@@ -300,6 +307,7 @@ static const char NETWORK_VIDEO_YUV_SHADER[] =
   "layout(binding=2) uniform highp sampler2D crPlane;\n"
   "layout(rgba16f,binding=0) writeonly uniform highp image2D outputImage;\n"
   "uniform ivec2 outputSize;\n"
+  "uniform ivec2 outputOffset;\n"
   "uniform bool bt2020;\n"
   "uniform bool chromaLeft;\n"
   "uniform float lumaOffset;\n"
@@ -307,7 +315,7 @@ static const char NETWORK_VIDEO_YUV_SHADER[] =
   "uniform float chromaMidpoint;\n"
   "uniform float chromaScale;\n"
   "void main(){\n"
-  "  ivec2 pixel     = ivec2(gl_GlobalInvocationID.xy);\n"
+  "  ivec2 pixel     = ivec2(gl_GlobalInvocationID.xy)+outputOffset;\n"
   "  if(any(greaterThanEqual(pixel,outputSize)))return;\n"
   "  ivec2 planeSize = textureSize(lumaPlane,0);\n"
   "  vec2  uv        = (vec2(pixel)+0.5)/vec2(planeSize);\n"