Mirrored_Repos/LookingGlass
1
0
Fork 0
mirror of https://github.com/gnif/LookingGlass.git synced 2026-11-12 14:57:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[client/idd] network: preserve exact frame damage

Browse Source
This commit is contained in:
Geoffrey McRae
2026-10-08 14:18:46 +11:00
parent ae031f6064
commit ccff282cd0
14 changed files with 519 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
client/include/interface/transport.h
View File

@@ -33,7 +33,7 @@
#include "interface/external_frame.h"
#include "interface/input.h"
#define LG_TRANSPORT_MAX_DAMAGE_RECTS LG_MAX_FRAME_DAMAGE_RECTS
#define LG_TRANSPORT_MAX_DAMAGE_RECTS LG_MAX_RUNTIME_DAMAGE_RECTS
typedef struct LG_Transport LG_Transport;
typedef struct LG_RendererInterop LG_RendererInterop;

3
client/renderers/EGL/damage.c
View File

@@ -88,7 +88,8 @@ bool egl_damageInit(EGL_Damage ** damage)
return false;
}
if (!egl_desktopRectsInit(&(*damage)->mesh, LG_MAX_FRAME_DAMAGE_RECTS))
if (!egl_desktopRectsInit(&(*damage)->mesh,
LG_MAX_RUNTIME_DAMAGE_RECTS))
{
DEBUG_ERROR("Failed to initialize the mesh");
return false;

2
client/renderers/EGL/damage.h
View File

@@ -29,7 +29,7 @@ struct DesktopDamage
{
LG_RendererFrameToken frameToken;
int count;
KVMFRFrameDamageRect rects[LG_MAX_FRAME_DAMAGE_RECTS];
KVMFRFrameDamageRect rects[LG_MAX_RUNTIME_DAMAGE_RECTS];
};
typedef struct EGL_Damage EGL_Damage;

19
client/renderers/EGL/egl.c
View File

@@ -58,10 +58,11 @@
#include "postprocess.h"
#include "util.h"
#define MAX_BUFFER_AGE 3
#define DESKTOP_DAMAGE_COUNT 4
#define DESKTOP_DAMAGE_MARGIN 1
#define MAX_ACCUMULATED_DAMAGE ((LG_MAX_FRAME_DAMAGE_RECTS + MAX_OVERLAY_RECTS + 2) * MAX_BUFFER_AGE)
#define MAX_BUFFER_AGE 3
#define DESKTOP_DAMAGE_COUNT 4
#define DESKTOP_DAMAGE_MARGIN 1
#define MAX_ACCUMULATED_DAMAGE \
((LG_MAX_RUNTIME_DAMAGE_RECTS + MAX_OVERLAY_RECTS + 2) * MAX_BUFFER_AGE)
#define IDX_AGO(counter, i, total) (((counter) + (total) - (i)) % (total))
struct Options
@@ -474,7 +475,7 @@ static void egl_addSwSurfaceDamage(
damage->frameToken = LG_RENDERER_FRAME_TOKEN_NONE;
if (damage->count >= 0)
{
if (damage->count + 1 >= LG_MAX_FRAME_DAMAGE_RECTS)
if (damage->count + 1 > LG_MAX_RUNTIME_DAMAGE_RECTS)
damage->count = -1;
else
damage->rects[damage->count++] = rect;
@@ -880,7 +881,7 @@ static bool egl_onFrame(LG_Renderer * renderer,
if (unlikely(
damage->count == -1 ||
damageRectsCount == 0 ||
damage->count + damageRectsCount >= LG_MAX_FRAME_DAMAGE_RECTS))
damage->count + damageRectsCount > LG_MAX_RUNTIME_DAMAGE_RECTS))
{
damage->count = -1;
}
@@ -1443,7 +1444,7 @@ static void egl_requeueDesktopDamage(
this->desktopDamage + this->desktopDamageIdx;
if (damage->count < 0 || pending->count < 0 ||
damage->count + pending->count >= LG_MAX_FRAME_DAMAGE_RECTS)
damage->count + pending->count > LG_MAX_RUNTIME_DAMAGE_RECTS)
pending->count = -1;
else
{
@@ -1698,8 +1699,8 @@ static bool egl_render(LG_Renderer * renderer, LG_RendererRotate rotate,
/* The diagnostics being displayed must not contribute to Compose. */
timing->composeTime = nanotime() - composeStart;
struct Rect damage[LG_MAX_FRAME_DAMAGE_RECTS + MAX_OVERLAY_RECTS + 2];
int damageIdx = app_renderOverlay(damage, MAX_OVERLAY_RECTS);
struct Rect damage[LG_MAX_RUNTIME_DAMAGE_RECTS + MAX_OVERLAY_RECTS + 2];
int damageIdx = app_renderOverlay(damage, MAX_OVERLAY_RECTS);
if (unlikely(damageIdx != 0))
{
if (damageIdx == -1)

13
client/renderers/EGL/texture_framebuffer.c
View File

@@ -27,7 +27,7 @@
struct TexDamage
{
int count;
KVMFRFrameDamageRect rects[LG_MAX_FRAME_DAMAGE_RECTS];
KVMFRFrameDamageRect rects[LG_MAX_RUNTIME_DAMAGE_RECTS];
};
typedef struct TexFB
@@ -87,10 +87,11 @@ static bool egl_texFBUpdate(EGL_Texture * texture, const EGL_TexUpdate * update)
if (!egl_texBufferStreamLock(parent))
return false;
struct TexDamage * damage = this->damage + parent->bufIndex;
bool damageAll = !update->rects || update->rectCount == 0 || damage->count < 0 ||
damage->count + update->rectCount > LG_MAX_FRAME_DAMAGE_RECTS;
bool complete;
struct TexDamage * damage = this->damage + parent->bufIndex;
bool damageAll =
!update->rects || update->rectCount == 0 || damage->count < 0 ||
damage->count + update->rectCount > LG_MAX_RUNTIME_DAMAGE_RECTS;
bool complete;
if (damageAll)
{
@@ -172,7 +173,7 @@ static bool egl_texFBUpdate(EGL_Texture * texture, const EGL_TexUpdate * update)
if (i == parent->bufIndex)
damage->count = 0;
else if (update->rects && update->rectCount > 0 && damage->count >= 0 &&
damage->count + update->rectCount <= LG_MAX_FRAME_DAMAGE_RECTS)
damage->count + update->rectCount <= LG_MAX_RUNTIME_DAMAGE_RECTS)
{
memcpy(damage->rects + damage->count, update->rects,
update->rectCount * sizeof(KVMFRFrameDamageRect));

4
client/src/render_queue.c
View File

@@ -147,7 +147,7 @@ typedef struct RenderQueueSwSurface
bool damageFull;
int damageCount;
KVMFRFrameDamageRect damage[LG_MAX_FRAME_DAMAGE_RECTS];
KVMFRFrameDamageRect damage[LG_MAX_RUNTIME_DAMAGE_RECTS];
bool updateQueued;
RenderCommand updateCommand;
@@ -361,7 +361,7 @@ static void swSurfaceDamageAdd(RenderQueueSwSurface * surface,
i = 0;
}
if (surface->damageCount < LG_MAX_FRAME_DAMAGE_RECTS)
if (surface->damageCount < LG_MAX_RUNTIME_DAMAGE_RECTS)
break;
int best = 0;

2
client/transports/Network/video.c
View File

@@ -300,7 +300,7 @@ static void networkVideoMergeDamage(NetworkVideoDecodedFrame * destination,
return;
}
if (destination->damageRectsCount >=
if (destination->damageRectsCount >
LG_TRANSPORT_MAX_DAMAGE_RECTS - source->damageRectsCount)
{
uint32_t left = UINT32_MAX;

462
client/transports/Network/video/decoder_egl.c
View File

@@ -68,10 +68,10 @@
#define NETWORK_VIDEO_MIN_SIZE (4U << NETWORK_VIDEO_LEVELS)
#define NETWORK_VIDEO_UPLOADS 3U
#define NETWORK_VIDEO_POOL_SIZE 3U
#define NETWORK_VIDEO_DAMAGE_HISTORY 64U
#define NETWORK_VIDEO_POOL_BYTE_LIMIT \
(UINT64_C(192) * 1024U * 1024U)
#define NETWORK_VIDEO_PIXEL_BYTES 8U
#define NETWORK_VIDEO_TILE_SIZE 32U
#define NETWORK_VIDEO_SEQUENCE_BITS (UINT32_C(7) << 28U)
struct NetworkVideoBlockMeta
@@ -119,6 +119,12 @@ struct NetworkVideoTexture
uint64_t generation;
};
struct NetworkVideoDamageHistory
{
uint64_t generation;
LGNetVideoDamage damage;
};
struct NetworkVideoTexturePool
{
LG_Lock lock;
@@ -165,6 +171,7 @@ struct NetworkVideoUniforms
GLint crPlane;
GLint outputSize;
GLint outputOffset;
GLint outputExtent;
GLint bt2020;
GLint chromaLeft;
GLint lumaOffset;
@@ -173,6 +180,14 @@ struct NetworkVideoUniforms
GLint chromaScale;
}
yuv;
struct
{
GLint inputImage;
GLint outputOffset;
GLint outputExtent;
}
copy;
};
struct NetworkVideoDecoder
@@ -187,8 +202,10 @@ struct NetworkVideoDecoder
GLuint dequantProgram;
GLuint idwtProgram;
GLuint yuvProgram;
GLuint copyProgram;
GLuint coefficientTextures[NETWORK_VIDEO_LEVELS];
GLuint planeTextures [NETWORK_VIDEO_COMPONENTS];
GLuint canonicalTexture;
struct NetworkVideoUpload uploads [NETWORK_VIDEO_UPLOADS];
struct NetworkVideoTexturePool * texturePool;
struct NetworkVideoUniforms uniforms;
@@ -198,8 +215,8 @@ struct NetworkVideoDecoder
uint8_t * dirtyBlocks;
struct NetworkVideoDirtyGrid
dirty[NETWORK_VIDEO_COMPONENTS][NETWORK_VIDEO_LEVELS];
uint8_t * outputDirty;
uint64_t * outputTileGenerations;
struct NetworkVideoDamageHistory
damageHistory[NETWORK_VIDEO_DAMAGE_HISTORY];
LGNetVideoStreamConfig config;
struct NetworkVideoBlockMeta
@@ -209,8 +226,6 @@ struct NetworkVideoDecoder
uint32_t alignedWidth;
uint32_t alignedHeight;
uint32_t blockCapacity;
uint32_t outputTileWidth;
uint32_t outputTileHeight;
uint64_t referenceFrameID;
uint64_t outputGeneration;
unsigned uploadIndex;
@@ -518,6 +533,8 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder)
{
glDeleteTextures(NETWORK_VIDEO_LEVELS, decoder->coefficientTextures);
glDeleteTextures(NETWORK_VIDEO_COMPONENTS, decoder->planeTextures);
if (decoder->canonicalTexture)
glDeleteTextures(1, &decoder->canonicalTexture);
for (unsigned index = 0; index < NETWORK_VIDEO_UPLOADS; ++index)
{
struct NetworkVideoUpload * upload = &decoder->uploads[index];
@@ -537,28 +554,24 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder)
component < NETWORK_VIDEO_COMPONENTS; ++component)
for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level)
free(decoder->dirty[component][level].tiles);
free(decoder->outputDirty);
free(decoder->outputTileGenerations);
free(decoder->offsets);
memset(decoder->uploads, 0, sizeof(decoder->uploads));
decoder->texturePool = NULL;
decoder->offsets = NULL;
decoder->packetStates = NULL;
decoder->pendingPackets = NULL;
decoder->dirtyBlocks = NULL;
decoder->outputDirty = NULL;
decoder->outputTileGenerations = NULL;
decoder->alignedWidth = 0;
decoder->alignedHeight = 0;
decoder->blockCapacity = 0;
decoder->outputTileWidth = 0;
decoder->outputTileHeight = 0;
decoder->referenceFrameID = 0;
decoder->outputGeneration = 0;
decoder->uploadIndex = 0;
decoder->configured = false;
decoder->referenceValid = false;
memset(decoder->damageHistory, 0, sizeof(decoder->damageHistory));
decoder->texturePool = NULL;
decoder->offsets = NULL;
decoder->packetStates = NULL;
decoder->pendingPackets = NULL;
decoder->dirtyBlocks = NULL;
decoder->canonicalTexture = 0;
decoder->alignedWidth = 0;
decoder->alignedHeight = 0;
decoder->blockCapacity = 0;
decoder->referenceFrameID = 0;
decoder->outputGeneration = 0;
decoder->uploadIndex = 0;
decoder->configured = false;
decoder->referenceValid = false;
memset(decoder->coefficientTextures, 0,
sizeof(decoder->coefficientTextures));
memset(decoder->planeTextures, 0, sizeof(decoder->planeTextures));
@@ -575,9 +588,12 @@ static void networkVideoDeleteProgramsNL(NetworkVideoDecoder * decoder)
glDeleteProgram(decoder->idwtProgram);
if (decoder->yuvProgram)
glDeleteProgram(decoder->yuvProgram);
if (decoder->copyProgram)
glDeleteProgram(decoder->copyProgram);
decoder->dequantProgram = 0;
decoder->idwtProgram = 0;
decoder->yuvProgram = 0;
decoder->copyProgram = 0;
memset(&decoder->uniforms, 0, sizeof(decoder->uniforms));
}
@@ -619,6 +635,8 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder)
decoder->yuvProgram, "outputSize");
decoder->uniforms.yuv.outputOffset = glGetUniformLocation(
decoder->yuvProgram, "outputOffset");
decoder->uniforms.yuv.outputExtent = glGetUniformLocation(
decoder->yuvProgram, "outputExtent");
decoder->uniforms.yuv.bt2020 = glGetUniformLocation(
decoder->yuvProgram, "bt2020");
decoder->uniforms.yuv.chromaLeft = glGetUniformLocation(
@@ -632,6 +650,13 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder)
decoder->uniforms.yuv.chromaScale = glGetUniformLocation(
decoder->yuvProgram, "chromaScale");
decoder->uniforms.copy.inputImage = glGetUniformLocation(
decoder->copyProgram, "inputImage");
decoder->uniforms.copy.outputOffset = glGetUniformLocation(
decoder->copyProgram, "outputOffset");
decoder->uniforms.copy.outputExtent = glGetUniformLocation(
decoder->copyProgram, "outputExtent");
const struct NetworkVideoUniforms * uniforms = &decoder->uniforms;
if (uniforms->dequant.resolution < 0 ||
uniforms->dequant.outputLayer < 0 ||
@@ -648,11 +673,15 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder)
uniforms->yuv.lumaPlane < 0 ||
uniforms->yuv.cbPlane < 0 || uniforms->yuv.crPlane < 0 ||
uniforms->yuv.outputSize < 0 || uniforms->yuv.outputOffset < 0 ||
uniforms->yuv.outputExtent < 0 ||
uniforms->yuv.bt2020 < 0 ||
uniforms->yuv.chromaLeft < 0 || uniforms->yuv.lumaOffset < 0 ||
uniforms->yuv.lumaScale < 0 ||
uniforms->yuv.chromaMidpoint < 0 ||
uniforms->yuv.chromaScale < 0)
uniforms->yuv.chromaScale < 0 ||
uniforms->copy.inputImage < 0 ||
uniforms->copy.outputOffset < 0 ||
uniforms->copy.outputExtent < 0)
return false;
glUseProgram(decoder->idwtProgram);
@@ -661,6 +690,8 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder)
glUniform1i(uniforms->yuv.lumaPlane, 0);
glUniform1i(uniforms->yuv.cbPlane, 1);
glUniform1i(uniforms->yuv.crPlane, 2);
glUseProgram(decoder->copyProgram);
glUniform1i(uniforms->copy.inputImage, 0);
return glGetError() == GL_NO_ERROR;
}
@@ -824,31 +855,22 @@ static bool networkVideoAllocateNL(NetworkVideoDecoder * decoder)
return false;
}
decoder->offsets = malloc(
if (!networkVideoCreateTexture2D(&decoder->canonicalTexture, GL_RGBA16F,
decoder->config.width, decoder->config.height, GL_LINEAR))
return false;
decoder->offsets = malloc(
(size_t)decoder->blockCapacity * sizeof(*decoder->offsets));
decoder->packetStates = calloc(
decoder->packetStates = calloc(
decoder->blockCapacity, sizeof(*decoder->packetStates));
decoder->pendingPackets = calloc(
decoder->pendingPackets = calloc(
decoder->blockCapacity, sizeof(*decoder->pendingPackets));
decoder->dirtyBlocks = calloc(
decoder->dirtyBlocks = calloc(
decoder->blockCapacity, sizeof(*decoder->dirtyBlocks));
decoder->texturePool = networkVideoTexturePoolCreate(decoder);
decoder->outputTileWidth =
(decoder->config.width + NETWORK_VIDEO_TILE_SIZE - 1U) /
NETWORK_VIDEO_TILE_SIZE;
decoder->outputTileHeight =
(decoder->config.height + NETWORK_VIDEO_TILE_SIZE - 1U) /
NETWORK_VIDEO_TILE_SIZE;
const size_t outputTileCount =
(size_t)decoder->outputTileWidth * decoder->outputTileHeight;
decoder->outputDirty = calloc(
outputTileCount, sizeof(*decoder->outputDirty));
decoder->outputTileGenerations = calloc(
outputTileCount, sizeof(*decoder->outputTileGenerations));
decoder->texturePool = networkVideoTexturePoolCreate(decoder);
if (!decoder->offsets || !decoder->packetStates ||
!decoder->pendingPackets || !decoder->dirtyBlocks ||
!decoder->texturePool || !decoder->outputDirty ||
!decoder->outputTileGenerations)
!decoder->texturePool)
return false;
for (unsigned component = 0;
@@ -1105,8 +1127,12 @@ bool networkVideoDecoderAttach(NetworkVideoDecoder * decoder,
NETWORK_VIDEO_YUV_SOURCES,
sizeof(NETWORK_VIDEO_YUV_SOURCES) /
sizeof(NETWORK_VIDEO_YUV_SOURCES[0]));
decoder->copyProgram = networkVideoCreateProgram(
NETWORK_VIDEO_COPY_SOURCES,
sizeof(NETWORK_VIDEO_COPY_SOURCES) /
sizeof(NETWORK_VIDEO_COPY_SOURCES[0]));
const bool programsReady = decoder->dequantProgram &&
decoder->idwtProgram && decoder->yuvProgram &&
decoder->idwtProgram && decoder->yuvProgram && decoder->copyProgram &&
networkVideoCacheUniformsNL(decoder);
networkVideoRestoreCurrent(decoder, &previous);
if (!programsReady)
@@ -1304,8 +1330,6 @@ static void networkVideoClearDirtyNL(NetworkVideoDecoder * decoder)
grid->count = 0;
}
}
memset(decoder->outputDirty, 0,
(size_t)decoder->outputTileWidth * decoder->outputTileHeight);
}
static uint32_t networkVideoMirrorTexel(
@@ -1383,44 +1407,6 @@ static void networkVideoMarkGridRect(struct NetworkVideoDirtyGrid * grid,
}
}
static void networkVideoMarkOutputRect(NetworkVideoDecoder * decoder,
unsigned component, uint32_t x, uint32_t y,
uint32_t width, uint32_t height)
{
uint32_t expansion = component ? 1U : 0U;
if (component &&
decoder->config.chromaSubsampling == LG_NET_VIDEO_CHROMA_420)
{
x *= 2U;
y *= 2U;
width *= 2U;
height *= 2U;
expansion = 2U;
}
const uint32_t startX = x > expansion ? x - expansion : 0U;
const uint32_t startY = y > expansion ? y - expansion : 0U;
const uint64_t rawEndX = (uint64_t)x + width + expansion;
const uint64_t rawEndY = (uint64_t)y + height + expansion;
const uint32_t endX = rawEndX < decoder->config.width ?
(uint32_t)rawEndX : decoder->config.width;
const uint32_t endY = rawEndY < decoder->config.height ?
(uint32_t)rawEndY : decoder->config.height;
if (startX >= endX || startY >= endY)
return;
const uint32_t tileStartX = startX / NETWORK_VIDEO_TILE_SIZE;
const uint32_t tileStartY = startY / NETWORK_VIDEO_TILE_SIZE;
const uint32_t tileEndX =
(endX + NETWORK_VIDEO_TILE_SIZE - 1U) / NETWORK_VIDEO_TILE_SIZE;
const uint32_t tileEndY =
(endY + NETWORK_VIDEO_TILE_SIZE - 1U) / NETWORK_VIDEO_TILE_SIZE;
for (uint32_t tileY = tileStartY; tileY < tileEndY; ++tileY)
for (uint32_t tileX = tileStartX; tileX < tileEndX; ++tileX)
decoder->outputDirty[
(size_t)tileY * decoder->outputTileWidth + tileX] = 1U;
}
static void networkVideoPrepareDirtyNL(NetworkVideoDecoder * decoder)
{
networkVideoClearDirtyNL(decoder);
@@ -1573,6 +1559,9 @@ static void networkVideoDequantNL(NetworkVideoDecoder * decoder,
static void networkVideoPropagateDirtyNL(NetworkVideoDecoder * decoder,
unsigned component, unsigned level, bool final)
{
if (final)
return;
const struct NetworkVideoDirtyGrid * grid =
&decoder->dirty[component][level];
const uint32_t outputWidth = decoder->alignedWidth >> level;
@@ -1589,13 +1578,9 @@ static void networkVideoPropagateDirtyNL(NetworkVideoDecoder * decoder,
(outputWidth - outputX < 32U ? outputWidth - outputX : 32U) : 0U;
const uint32_t height = outputY < outputHeight ?
(outputHeight - outputY < 32U ? outputHeight - outputY : 32U) : 0U;
if (final)
networkVideoMarkOutputRect(
decoder, component, outputX, outputY, width, height);
else
networkVideoMarkGridRect(&decoder->dirty[component][level - 1U],
outputWidth, outputHeight,
outputX, outputY, width, height);
networkVideoMarkGridRect(&decoder->dirty[component][level - 1U],
outputWidth, outputHeight,
outputX, outputY, width, height);
}
}
}
@@ -1676,10 +1661,6 @@ static void networkVideoIDWTNL(NetworkVideoDecoder * decoder, bool update)
GL_TEXTURE_FETCH_BARRIER_BIT);
}
}
if (!update)
memset(decoder->outputDirty, 1,
(size_t)decoder->outputTileWidth * decoder->outputTileHeight);
}
static void networkVideoColorRange(
@@ -1716,120 +1697,125 @@ static void networkVideoColorRange(
}
}
static bool networkVideoTextureTileDirty(
const NetworkVideoDecoder * decoder, size_t index,
uint64_t textureGeneration)
static bool networkVideoDamageFull(const LGNetVideoDamage * damage)
{
return decoder->outputDirty[index] ||
decoder->outputTileGenerations[index] > textureGeneration;
return (damage->flags & LG_NET_VIDEO_DAMAGE_FULL) != 0;
}
static void networkVideoBuildDamageNL(NetworkVideoDecoder * decoder,
NetworkVideoDecodedFrame * result)
static void networkVideoDispatchRect(GLint offsetUniform,
GLint extentUniform, uint32_t x, uint32_t y,
uint32_t width, uint32_t height)
{
uint32_t count = 0;
bool overflow = false;
for (uint32_t tileY = 0;
tileY < decoder->outputTileHeight && !overflow; ++tileY)
{
uint32_t tileX = 0;
while (tileX < decoder->outputTileWidth)
{
const size_t row = (size_t)tileY * decoder->outputTileWidth;
while (tileX < decoder->outputTileWidth &&
!decoder->outputDirty[row + tileX])
++tileX;
const uint32_t start = tileX;
while (tileX < decoder->outputTileWidth &&
decoder->outputDirty[row + tileX])
++tileX;
if (start == tileX)
continue;
glUniform2i(offsetUniform, (GLint)x, (GLint)y);
glUniform2i(extentUniform, (GLint)width, (GLint)height);
glDispatchCompute((width + 7U) / 8U, (height + 7U) / 8U, 1);
}
const uint32_t x = start * NETWORK_VIDEO_TILE_SIZE;
const uint32_t y = tileY * NETWORK_VIDEO_TILE_SIZE;
const uint32_t endX = tileX * NETWORK_VIDEO_TILE_SIZE <
decoder->config.width ?
tileX * NETWORK_VIDEO_TILE_SIZE : decoder->config.width;
const uint32_t endY = y + NETWORK_VIDEO_TILE_SIZE <
decoder->config.height ?
y + NETWORK_VIDEO_TILE_SIZE : decoder->config.height;
bool merged = false;
for (uint32_t index = 0; index < count; ++index)
{
KVMFRFrameDamageRect * rect = &result->damageRects[index];
if (rect->x == x && rect->width == endX - x &&
rect->y + rect->height == y)
{
rect->height += endY - y;
merged = true;
break;
}
}
if (merged)
continue;
if (count + 1U >= LG_TRANSPORT_MAX_DAMAGE_RECTS)
{
overflow = true;
break;
}
result->damageRects[count++] = (KVMFRFrameDamageRect)
{
.x = x,
.y = y,
.width = endX - x,
.height = endY - y,
};
}
}
if (!overflow)
static void networkVideoDispatchDamageNL(NetworkVideoDecoder * decoder,
GLint offsetUniform, GLint extentUniform,
const LGNetVideoDamage * damage)
{
if (networkVideoDamageFull(damage))
{
result->damageRectsCount = count;
networkVideoDispatchRect(offsetUniform, extentUniform, 0, 0,
decoder->config.width, decoder->config.height);
return;
}
uint32_t minX = decoder->config.width;
uint32_t minY = decoder->config.height;
uint32_t maxX = 0;
uint32_t maxY = 0;
for (uint32_t tileY = 0; tileY < decoder->outputTileHeight; ++tileY)
for (uint16_t index = 0; index < damage->count; ++index)
{
for (uint32_t tileX = 0; tileX < decoder->outputTileWidth; ++tileX)
{
if (!decoder->outputDirty[
(size_t)tileY * decoder->outputTileWidth + tileX])
continue;
const uint32_t x = tileX * NETWORK_VIDEO_TILE_SIZE;
const uint32_t y = tileY * NETWORK_VIDEO_TILE_SIZE;
minX = x < minX ? x : minX;
minY = y < minY ? y : minY;
maxX = x + NETWORK_VIDEO_TILE_SIZE > maxX ?
x + NETWORK_VIDEO_TILE_SIZE : maxX;
maxY = y + NETWORK_VIDEO_TILE_SIZE > maxY ?
y + NETWORK_VIDEO_TILE_SIZE : maxY;
}
const LGNetVideoDamageRect * rect = &damage->rects[index];
networkVideoDispatchRect(offsetUniform, extentUniform,
rect->x, rect->y, rect->width, rect->height);
}
maxX = maxX < decoder->config.width ? maxX : decoder->config.width;
maxY = maxY < decoder->config.height ? maxY : decoder->config.height;
result->damageRects[0] = (KVMFRFrameDamageRect)
}
static const LGNetVideoDamage * networkVideoFindDamageNL(
const NetworkVideoDecoder * decoder, uint64_t generation)
{
const struct NetworkVideoDamageHistory * history =
&decoder->damageHistory[generation % NETWORK_VIDEO_DAMAGE_HISTORY];
return history->generation == generation ? &history->damage : NULL;
}
static bool networkVideoCopyNeedsFullNL(
const NetworkVideoDecoder * decoder, uint64_t textureGeneration,
uint64_t generation, const LGNetVideoDamage * damage)
{
if (!textureGeneration || textureGeneration >= generation ||
generation - textureGeneration > NETWORK_VIDEO_DAMAGE_HISTORY ||
networkVideoDamageFull(damage))
return true;
for (uint64_t previous = textureGeneration + 1U;
previous < generation; ++previous)
{
.x = minX,
.y = minY,
.width = maxX - minX,
.height = maxY - minY,
};
result->damageRectsCount = 1;
const LGNetVideoDamage * prior =
networkVideoFindDamageNL(decoder, previous);
if (!prior || networkVideoDamageFull(prior))
return true;
}
return false;
}
static void networkVideoCopyCanonicalNL(NetworkVideoDecoder * decoder,
GLuint texture, uint64_t textureGeneration, uint64_t generation,
const LGNetVideoDamage * damage)
{
glUseProgram(decoder->copyProgram);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, decoder->canonicalTexture);
glBindImageTexture(0, texture, 0, GL_FALSE, 0,
GL_WRITE_ONLY, GL_RGBA16F);
if (networkVideoCopyNeedsFullNL(
decoder, textureGeneration, generation, damage))
{
networkVideoDispatchRect(decoder->uniforms.copy.outputOffset,
decoder->uniforms.copy.outputExtent, 0, 0,
decoder->config.width, decoder->config.height);
}
else
{
for (uint64_t previous = textureGeneration + 1U;
previous < generation; ++previous)
networkVideoDispatchDamageNL(decoder,
decoder->uniforms.copy.outputOffset,
decoder->uniforms.copy.outputExtent,
networkVideoFindDamageNL(decoder, previous));
networkVideoDispatchDamageNL(decoder,
decoder->uniforms.copy.outputOffset,
decoder->uniforms.copy.outputExtent, damage);
}
glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT |
GL_TEXTURE_FETCH_BARRIER_BIT);
}
static void networkVideoBuildDamage(const LGNetVideoDamage * damage,
NetworkVideoDecodedFrame * result)
{
if (networkVideoDamageFull(damage))
return;
for (uint16_t index = 0; index < damage->count; ++index)
{
const LGNetVideoDamageRect * source = &damage->rects[index];
result->damageRects[index] = (KVMFRFrameDamageRect)
{
.x = source->x,
.y = source->y,
.width = source->width,
.height = source->height,
};
}
result->damageRectsCount = damage->count;
}
static struct NetworkVideoExternalFrame * networkVideoConvertNL(
NetworkVideoDecoder * decoder, bool chromaLeft, uint64_t generation)
NetworkVideoDecoder * decoder, bool chromaLeft, uint64_t generation,
const LGNetVideoDamage * damage)
{
struct NetworkVideoTexture pooled =
networkVideoTexturePoolAcquire(decoder->texturePool);
if (!pooled.texture)
return NULL;
glUseProgram(decoder->yuvProgram);
for (unsigned component = 0;
component < NETWORK_VIDEO_COMPONENTS; ++component)
@@ -1852,46 +1838,21 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL(
glUniform1f(decoder->uniforms.yuv.lumaScale, lumaScale);
glUniform1f(decoder->uniforms.yuv.chromaMidpoint, chromaMidpoint);
glUniform1f(decoder->uniforms.yuv.chromaScale, chromaScale);
glBindImageTexture(0, pooled.texture, 0, GL_FALSE, 0,
glBindImageTexture(0, decoder->canonicalTexture, 0, GL_FALSE, 0,
GL_WRITE_ONLY, GL_RGBA16F);
if (!pooled.generation)
{
glUniform2i(decoder->uniforms.yuv.outputOffset, 0, 0);
glDispatchCompute((decoder->config.width + 7U) / 8U,
(decoder->config.height + 7U) / 8U, 1);
}
else
{
for (uint32_t tileY = 0; tileY < decoder->outputTileHeight; ++tileY)
{
uint32_t tileX = 0;
while (tileX < decoder->outputTileWidth)
{
const size_t row = (size_t)tileY * decoder->outputTileWidth;
while (tileX < decoder->outputTileWidth &&
!networkVideoTextureTileDirty(
decoder, row + tileX, pooled.generation))
++tileX;
const uint32_t start = tileX;
while (tileX < decoder->outputTileWidth &&
networkVideoTextureTileDirty(
decoder, row + tileX, pooled.generation))
++tileX;
if (start == tileX)
continue;
const uint32_t pixelWidth =
(tileX - start) * NETWORK_VIDEO_TILE_SIZE;
glUniform2i(decoder->uniforms.yuv.outputOffset,
(GLint)(start * NETWORK_VIDEO_TILE_SIZE),
(GLint)(tileY * NETWORK_VIDEO_TILE_SIZE));
glDispatchCompute((pixelWidth + 7U) / 8U,
NETWORK_VIDEO_TILE_SIZE / 8U, 1);
}
}
}
networkVideoDispatchDamageNL(decoder,
decoder->uniforms.yuv.outputOffset,
decoder->uniforms.yuv.outputExtent, damage);
glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT |
GL_TEXTURE_FETCH_BARRIER_BIT);
struct NetworkVideoTexture pooled =
networkVideoTexturePoolAcquire(decoder->texturePool);
if (!pooled.texture)
return NULL;
networkVideoCopyCanonicalNL(decoder, pooled.texture,
pooled.generation, generation, damage);
GLsync readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
glFlush();
if (!readySync || glGetError() != GL_NO_ERROR)
@@ -1956,6 +1917,17 @@ NetworkVideoDecodeResult networkVideoDecoderDecode(
NETWORK_VIDEO_DECODE_UNSUPPORTED : NETWORK_VIDEO_DECODE_CORRUPT;
}
LGNetVideoDamage damage = layout.damage;
if (!update)
{
damage = (LGNetVideoDamage)
{
.flags = (LGNetVideoDamageFlags)LG_NET_VIDEO_DAMAGE_FULL,
};
}
const bool hasPresentationDamage =
networkVideoDamageFull(&damage) || damage.count;
uint32_t changedCount = 0;
if (!networkVideoStagePacketsNL(
decoder, data, size, update, &changedCount))
@@ -2004,10 +1976,15 @@ NetworkVideoDecodeResult networkVideoDecoderDecode(
networkVideoDequantNL(decoder, upload, update);
upload->readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
networkVideoIDWTNL(decoder, update);
struct NetworkVideoExternalFrame * external =
networkVideoConvertNL(decoder, layout.chromaLeft, generation);
struct NetworkVideoExternalFrame * external = NULL;
if (hasPresentationDamage)
external = networkVideoConvertNL(decoder,
layout.chromaLeft, generation, &damage);
else
glFlush();
const uint64_t decodeTime = nanotime() - decodeStart;
const bool success = external && glGetError() == GL_NO_ERROR;
const bool success =
(!hasPresentationDamage || external) && glGetError() == GL_NO_ERROR;
networkVideoRestoreCurrent(decoder, &previous);
if (!success)
{
@@ -2021,14 +1998,19 @@ NetworkVideoDecodeResult networkVideoDecoderDecode(
}
networkVideoCommitPendingPacketsNL(decoder);
const size_t outputTileCount =
(size_t)decoder->outputTileWidth * decoder->outputTileHeight;
for (size_t index = 0; index < outputTileCount; ++index)
if (decoder->outputDirty[index])
decoder->outputTileGenerations[index] = generation;
decoder->outputGeneration = generation;
decoder->referenceFrameID = complete ? frameID : 0;
decoder->referenceValid = complete;
if (!hasPresentationDamage)
{
LG_UNLOCK(decoder->lock);
return NETWORK_VIDEO_DECODE_NO_FRAME;
}
struct NetworkVideoDamageHistory * history =
&decoder->damageHistory[generation % NETWORK_VIDEO_DAMAGE_HISTORY];
history->generation = generation;
history->damage = damage;
decoder->outputGeneration = generation;
uint32_t formatVersion = (uint32_t)config->configEpoch;
if (!formatVersion)
@@ -2037,7 +2019,7 @@ NetworkVideoDecodeResult networkVideoDecoderDecode(
result->externalFrame = &external->base;
result->decodeTimeNs = decodeTime;
if (update)
networkVideoBuildDamageNL(decoder, result);
networkVideoBuildDamage(&damage, result);
result->format = (LG_TransportFrameFormat)
{
.version = formatVersion,

41
client/transports/Network/video/pyrowave.c
View File

@@ -247,11 +247,11 @@ NetworkPyroWaveResult networkPyroWaveParse(
size_t cursor = 0;
uint32_t uniqueBlocks = 0;
uint32_t declaredBlocks = 0;
uint32_t headerFirst = 0;
uint32_t headerSecond = 0;
uint8_t frameSequence = 0;
bool haveSequence = false;
bool haveHeader = false;
bool haveDamage = false;
bool haveClear = false;
bool valid = true;
@@ -285,21 +285,39 @@ NetworkPyroWaveResult networkPyroWaveParse(
width != config->width || height != config->height ||
!blocks || blocks > capacity ||
!networkPyroWaveSequenceProfile(config, second) ||
(haveHeader &&
(first != headerFirst || second != headerSecond)))
haveHeader)
{
valid = false;
break;
}
declaredBlocks = blocks;
headerFirst = first;
headerSecond = second;
haveHeader = true;
cursor += NETWORK_PYROWAVE_HEADER_SIZE;
if (code == LG_NET_VIDEO_PYROWAVE_MODE_DELTA)
{
size_t damageSize = 0;
if (lgNetVideoDamageDecode(&layout->damage,
bytes + cursor, size - cursor,
config->width, config->height, &damageSize) != LG_NET_PARSE_OK)
{
valid = false;
break;
}
cursor += damageSize;
haveDamage = true;
}
continue;
}
if (!haveHeader)
{
valid = false;
break;
}
const uint32_t ballot = first & UINT32_C(0x0000FFFF);
const uint32_t payloadWords = (first >> 16) & UINT32_C(0x00000FFF);
const uint32_t blockIndex = second >> 8;
@@ -322,20 +340,19 @@ NetworkPyroWaveResult networkPyroWaveParse(
cursor += packetSize;
}
const bool delta =
((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) &
LG_NET_VIDEO_PYROWAVE_MODE_MASK) ==
LG_NET_VIDEO_PYROWAVE_MODE_DELTA;
valid = valid && cursor == size && haveHeader && haveSequence &&
haveDamage == delta &&
declaredBlocks == uniqueBlocks && expectedBlocks == uniqueBlocks &&
(!haveClear ||
((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) &
LG_NET_VIDEO_PYROWAVE_MODE_MASK) ==
LG_NET_VIDEO_PYROWAVE_MODE_DELTA);
(!haveClear || delta);
if (!valid)
return NETWORK_PYROWAVE_INVALID;
layout->sequence = frameSequence;
layout->chromaLeft = (headerSecond >> 31) != 0;
layout->delta =
((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) &
LG_NET_VIDEO_PYROWAVE_MODE_MASK) ==
LG_NET_VIDEO_PYROWAVE_MODE_DELTA;
layout->delta = delta;
return NETWORK_PYROWAVE_OK;
}

7
client/transports/Network/video/pyrowave.h
View File

@@ -36,9 +36,10 @@ NetworkPyroWaveResult;
typedef struct NetworkPyroWaveLayout
{
uint8_t sequence;
bool chromaLeft;
bool delta;
uint8_t sequence;
bool chromaLeft;
bool delta;
LGNetVideoDamage damage;
}
NetworkPyroWaveLayout;

25
client/transports/Network/video/shaders.h
View File

@@ -308,6 +308,7 @@ static const char NETWORK_VIDEO_YUV_SHADER[] =
"layout(rgba16f,binding=0) writeonly uniform highp image2D outputImage;\n"
"uniform ivec2 outputSize;\n"
"uniform ivec2 outputOffset;\n"
"uniform ivec2 outputExtent;\n"
"uniform bool bt2020;\n"
"uniform bool chromaLeft;\n"
"uniform float lumaOffset;\n"
@@ -315,7 +316,9 @@ static const char NETWORK_VIDEO_YUV_SHADER[] =
"uniform float chromaMidpoint;\n"
"uniform float chromaScale;\n"
"void main(){\n"
" ivec2 pixel = ivec2(gl_GlobalInvocationID.xy)+outputOffset;\n"
" ivec2 local = ivec2(gl_GlobalInvocationID.xy);\n"
" if(any(greaterThanEqual(local,outputExtent)))return;\n"
" ivec2 pixel = local+outputOffset;\n"
" if(any(greaterThanEqual(pixel,outputSize)))return;\n"
" ivec2 planeSize = textureSize(lumaPlane,0);\n"
" vec2 uv = (vec2(pixel)+0.5)/vec2(planeSize);\n"
@@ -335,6 +338,21 @@ static const char NETWORK_VIDEO_YUV_SHADER[] =
" vec3 rgb = clamp(m*vec3(y,cb,cr),vec3(0.0),vec3(1.0));\n"
" imageStore(outputImage,pixel,vec4(rgb,1.0));}\n";
static const char NETWORK_VIDEO_COPY_SHADER[] =
"#version 310 es\n"
"precision highp float;\n"
"precision highp int;\n"
"layout(local_size_x=8,local_size_y=8) in;\n"
"layout(binding=0) uniform highp sampler2D inputImage;\n"
"layout(rgba16f,binding=0) writeonly uniform highp image2D outputImage;\n"
"uniform ivec2 outputOffset;\n"
"uniform ivec2 outputExtent;\n"
"void main(){\n"
" ivec2 local = ivec2(gl_GlobalInvocationID.xy);\n"
" if(any(greaterThanEqual(local,outputExtent)))return;\n"
" ivec2 pixel = local+outputOffset;\n"
" imageStore(outputImage,pixel,texelFetch(inputImage,pixel,0));}\n";
static const char * const NETWORK_VIDEO_DEQUANT_SOURCES[] =
{
NETWORK_VIDEO_DEQUANT_SHADER_0,
@@ -351,3 +369,8 @@ static const char * const NETWORK_VIDEO_YUV_SOURCES[] =
{
NETWORK_VIDEO_YUV_SHADER,
};
static const char * const NETWORK_VIDEO_COPY_SOURCES[] =
{
NETWORK_VIDEO_COPY_SHADER,
};

2
common/include/common/types.h
View File

@@ -25,6 +25,8 @@
#include <LGProtocol/KVMFRTypes.h>
#define LG_MAX_RUNTIME_DAMAGE_RECTS 256
typedef void (*LG_FrameReleaseFn)(void * opaque, uint64_t handle);
struct Point

228
idd/LGIdd/transport/network/CNetworkVideo.cpp
View File

@@ -83,8 +83,11 @@ namespace
constexpr uint32_t PYROWAVE_ALIGNMENT = 1U << PYROWAVE_LEVELS;
constexpr uint32_t PYROWAVE_MIN_DIMENSION = 4U << PYROWAVE_LEVELS;
constexpr uint32_t PYROWAVE_PACKET_BYTES = 2488;
constexpr uint32_t PYROWAVE_DAMAGE_BYTES_MAX =
LG_NET_VIDEO_DAMAGE_HEADER_WIRE_SIZE +
LG_NET_VIDEO_MAX_DAMAGE_RECTS * LG_NET_VIDEO_DAMAGE_RECT_WIRE_SIZE;
constexpr uint32_t PYROWAVE_BLOCK_BYTES_MAX =
PYROWAVE_PACKET_BYTES + 8U;
PYROWAVE_PACKET_BYTES + 8U + PYROWAVE_DAMAGE_BYTES_MAX;
// Reserve an equal fragment budget for every possible codec block so a
// negotiated delivery path can always represent the encoded frame.
constexpr uint32_t PYROWAVE_FRAGS_PER_BLOCK =
@@ -93,7 +96,8 @@ namespace
(PYROWAVE_BLOCK_BYTES_MAX + PYROWAVE_FRAGS_PER_BLOCK - 1U) /
PYROWAVE_FRAGS_PER_BLOCK;
constexpr uint32_t PYROWAVE_FRAGMENT_MIN =
LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE + PYROWAVE_FRAGMENT_DATA_MIN;
LG_NET_VIDEO_FRAME_UPDATE_FRAGMENT_HEADER_WIRE_SIZE +
PYROWAVE_FRAGMENT_DATA_MIN;
constexpr uint32_t PYROWAVE_PACKET_META_WORDS = 2;
constexpr uint32_t PYROWAVE_SEQUENCE_MASK = 7;
constexpr uint32_t PYROWAVE_HEADER_CODE_MASK =
@@ -135,6 +139,8 @@ namespace
static_assert(PYROWAVE_BLOCK_BYTES_MAX <=
PYROWAVE_FRAGS_PER_BLOCK * LG_NET_VIDEO_MAX_FRAGMENT_LENGTH,
"The fragment payload limit must represent every codec block");
static_assert(LG_NET_VIDEO_MAX_DAMAGE_RECTS >= FRAME_DAMAGE_MAX,
"The network protocol must represent every capture damage rectangle");
constexpr LGNetEnvelopeFlags DELIVERY_MASK =
LG_NET_ENVELOPE_RESPONSE | LG_NET_ENVELOPE_FINAL |
@@ -1502,6 +1508,155 @@ void FinalizeMain()
uint32_t index = 0;
};
bool VideoDamageContains(const LGNetVideoDamageRect& outer,
const LGNetVideoDamageRect& inner)
{
return outer.x <= inner.x && outer.y <= inner.y &&
static_cast<uint32_t>(outer.x) + outer.width >=
static_cast<uint32_t>(inner.x) + inner.width &&
static_cast<uint32_t>(outer.y) + outer.height >=
static_cast<uint32_t>(inner.y) + inner.height;
}
bool MergeVideoDamageRect(const LGNetVideoDamageRect& first,
const LGNetVideoDamageRect& second, LGNetVideoDamageRect& merged)
{
const uint32_t firstRight =
static_cast<uint32_t>(first.x) + first.width;
const uint32_t secondRight =
static_cast<uint32_t>(second.x) + second.width;
const uint32_t firstBottom =
static_cast<uint32_t>(first.y) + first.height;
const uint32_t secondBottom =
static_cast<uint32_t>(second.y) + second.height;
if (first.y == second.y && first.height == second.height &&
first.x <= secondRight && second.x <= firstRight)
{
const uint32_t left = (std::min)(first.x, second.x);
const uint32_t right = (std::max)(firstRight, secondRight);
merged.x = static_cast<uint16_t>(left);
merged.y = first.y;
merged.width = static_cast<uint16_t>(right - left);
merged.height = first.height;
return true;
}
if (first.x == second.x && first.width == second.width &&
first.y <= secondBottom && second.y <= firstBottom)
{
const uint32_t top = (std::min)(first.y, second.y);
const uint32_t bottom = (std::max)(firstBottom, secondBottom);
merged.x = first.x;
merged.y = static_cast<uint16_t>(top);
merged.width = first.width;
merged.height = static_cast<uint16_t>(bottom - top);
return true;
}
return false;
}
bool AddVideoDamageRect(LGNetVideoDamage& damage,
const LGNetVideoDamageRect& rect)
{
LGNetVideoDamageRect current = rect;
for (uint16_t index = 0; index < damage.count;)
{
const LGNetVideoDamageRect existing = damage.rects[index];
if (VideoDamageContains(existing, current))
return true;
LGNetVideoDamageRect merged = {};
if (!VideoDamageContains(current, existing) &&
!MergeVideoDamageRect(existing, current, merged))
{
++index;
continue;
}
if (VideoDamageContains(current, existing))
merged = current;
current = merged;
--damage.count;
damage.rects[index] = damage.rects[damage.count];
index = 0;
}
if (damage.count == LG_NET_VIDEO_MAX_DAMAGE_RECTS)
return false;
damage.rects[damage.count++] = current;
return true;
}
void SetFullVideoDamage(LGNetVideoDamage& damage)
{
damage = {};
damage.flags = static_cast<LGNetVideoDamageFlags>(
LG_NET_VIDEO_DAMAGE_FULL);
}
bool SetVideoDamage(const FrameDesc& frame, uint32_t width,
uint32_t height, LGNetVideoDamage& damage)
{
damage = {};
if (frame.damage == FrameDamage::NONE)
return true;
if (frame.damage == FrameDamage::FULL)
{
SetFullVideoDamage(damage);
return true;
}
if (frame.damage != FrameDamage::RECTS || !frame.count ||
frame.count > FRAME_DAMAGE_MAX)
return false;
for (unsigned index = 0; index < frame.count; ++index)
{
const RECT& rect = frame.rects[index];
if (rect.left < 0 || rect.top < 0 || rect.right <= rect.left ||
rect.bottom <= rect.top ||
static_cast<uint32_t>(rect.right) > width ||
static_cast<uint32_t>(rect.bottom) > height)
{
SetFullVideoDamage(damage);
return true;
}
const LGNetVideoDamageRect output =
{
static_cast<uint16_t>(rect.left),
static_cast<uint16_t>(rect.top),
static_cast<uint16_t>(rect.right - rect.left),
static_cast<uint16_t>(rect.bottom - rect.top),
};
if (!AddVideoDamageRect(damage, output))
{
SetFullVideoDamage(damage);
return true;
}
}
return lgNetVideoDamageValid(&damage, width, height);
}
void MergeVideoDamage(LGNetVideoDamage& destination,
const LGNetVideoDamage& source)
{
if ((destination.flags & LG_NET_VIDEO_DAMAGE_FULL) ||
(!source.flags && !source.count))
return;
if (source.flags & LG_NET_VIDEO_DAMAGE_FULL)
{
SetFullVideoDamage(destination);
return;
}
bool complete = true;
for (uint16_t index = 0; index < source.count; ++index)
if (!AddVideoDamageRect(destination, source.rects[index]))
{
complete = false;
break;
}
if (complete)
return;
SetFullVideoDamage(destination);
}
struct EncodedFrame
{
uint64_t configEpoch = 0;
@@ -1511,6 +1666,7 @@ void FinalizeMain()
uint32_t deadlineMs = 0;
uint32_t blockCapacity = 0;
uint32_t hardLimit = 0;
LGNetVideoDamage damage = {};
std::vector<uint8_t> damagedBlocks;
std::vector<uint8_t> data;
std::vector<EncodedBlock> blocks;
@@ -3317,7 +3473,11 @@ void FinalizeMain()
output = {};
const bool result = Compact(*slot, slot->mapped, output);
bool result = Compact(*slot, slot->mapped, output);
if (result)
result = SetVideoDamage(slot->frame.desc.frame,
m_cfg.width, m_cfg.height, output.damage);
if (result)
{
@@ -3389,8 +3549,9 @@ struct CNetworkVideo::Impl
{
struct ReferenceFrame
{
uint64_t frameID = 0;
uint64_t frameID = 0;
std::shared_ptr<const EncodedFrame> frame;
LGNetVideoDamage subsequentDamage = {};
std::vector<uint8_t> subsequentBlocks;
};
@@ -3416,6 +3577,7 @@ struct CNetworkVideo::Impl
uint8_t blockedDeliveries = 0;
std::shared_ptr<const EncodedFrame> referenceFrame;
std::deque<ReferenceFrame> referenceHistory;
LGNetVideoDamage updateDamage = {};
std::vector<uint8_t> updateBlocks;
bool statusSent = false;
bool subscribed = false;
@@ -3604,10 +3766,21 @@ struct CNetworkVideo::Impl
peer.referenceFrame.reset();
peer.referenceHistory.clear();
peer.updateBlocks.clear();
peer.updateDamage = {};
peer.referenceFrameID = 0;
peer.referencePending = false;
}
static void AccumulatePeerDamageNL(PeerState& peer,
const LGNetVideoDamage& damage)
{
if (!peer.referenceFrame)
return;
MergeVideoDamage(peer.updateDamage, damage);
for (ReferenceFrame& reference : peer.referenceHistory)
MergeVideoDamage(reference.subsequentDamage, damage);
}
static bool PeerNeedsReferenceNL(
const PeerState& peer, const EncodedFrame& frame)
{
@@ -3780,6 +3953,7 @@ struct CNetworkVideo::Impl
result->deadlineMs = current.deadlineMs;
result->blockCapacity = current.blockCapacity;
result->hardLimit = current.hardLimit;
result->damage = current.damage;
result->damagedBlocks = std::move(current.damagedBlocks);
result->blocks.reserve((std::max)(
previous->blocks.size(), current.blocks.size()));
@@ -3917,6 +4091,7 @@ struct CNetworkVideo::Impl
static std::shared_ptr<const EncodedFrame> BuildUpdateFrame(
const EncodedFrame& reference, const EncodedFrame& current,
const std::vector<uint8_t>& previousBlocks,
const LGNetVideoDamage& damage, uint32_t width, uint32_t height,
std::vector<uint8_t>& updateBlocks)
{
try
@@ -3947,9 +4122,14 @@ struct CNetworkVideo::Impl
update->captureTimeNs = current.captureTimeNs;
update->deadlineMs = current.deadlineMs;
update->blockCapacity = current.blockCapacity;
update->damage = damage;
update->blocks.reserve(operationCount);
update->data.reserve(current.data.size());
update->data.resize(8U);
const size_t damageSize = lgNetVideoDamageSize(&damage);
if (!damageSize ||
damageSize > LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH - 8U)
return {};
update->data.reserve(current.data.size() + damageSize);
update->data.resize(8U + damageSize);
const uint32_t firstWord = LoadLE32(current.data.data());
const uint32_t secondWord =
@@ -3958,6 +4138,9 @@ struct CNetworkVideo::Impl
PYROWAVE_HEADER_CODE_DELTA | operationCount;
StoreLE32(update->data.data(), firstWord);
StoreLE32(update->data.data() + sizeof(uint32_t), secondWord);
if (!lgNetVideoDamageEncode(update->data.data() + 8U,
damageSize, &damage, width, height))
return {};
const uint32_t sequence =
(firstWord >> 28U) & PYROWAVE_SEQUENCE_MASK;
@@ -4007,7 +4190,7 @@ struct CNetworkVideo::Impl
return {};
update->blocks[0].offset = 0;
update->blocks[0].length += 8U;
update->blocks[0].length += static_cast<uint32_t>(8U + damageSize);
return update;
}
catch (...)
@@ -4096,14 +4279,20 @@ struct CNetworkVideo::Impl
if (candidate->subsequentBlocks.size() != promotedFrame->blockCapacity)
return false;
std::vector<uint8_t> subsequentBlocks =
std::vector<uint8_t> subsequentBlocks =
std::move(candidate->subsequentBlocks);
const LGNetVideoDamage subsequentDamage = candidate->subsequentDamage;
peer.referenceFrame = promotedFrame;
peer.referenceFrameID = promotedID;
peer.referencePending = false;
peer.keyframePending = false;
peer.updateDamage = subsequentDamage;
peer.updateBlocks = std::move(subsequentBlocks);
// Damage also accumulates across frames intentionally skipped while the
// reference is pending, even though no block comparison is done for them.
// BuildUpdateFrame will authoritatively compare against the promoted
// reference and clear the damage if the encoded state did not change.
while (!peer.referenceHistory.empty() &&
peer.referenceHistory.front().frameID <= promotedID)
peer.referenceHistory.pop_front();
@@ -4808,6 +4997,7 @@ struct CNetworkVideo::Impl
peer.referenceFrame = frame;
peer.referenceFrameID = frameID;
peer.referencePending = true;
peer.updateDamage = {};
peer.updateBlocks = std::move(cleanBlocks);
RecordReferenceFrameNL(peer, frameID, frame);
}
@@ -4869,10 +5059,14 @@ struct CNetworkVideo::Impl
if (PeerNeedsReferenceNL(peer, *current))
return QueueFrameNL(
peer, current, frameID, presentationBaseNs, true);
if (peer.updateDamage.flags & LG_NET_VIDEO_DAMAGE_FULL)
return QueueFrameNL(
peer, current, frameID, presentationBaseNs, true);
DropPeerFramesNL(peer);
std::vector<uint8_t> updateBlocks;
const std::shared_ptr<const EncodedFrame> update = BuildUpdateFrame(
*peer.referenceFrame, *current, peer.updateBlocks,
peer.updateDamage, m_activeCfg.width, m_activeCfg.height,
updateBlocks);
const bool compared = updateBlocks.size() == current->blockCapacity;
const bool changed = std::any_of(updateBlocks.begin(),
@@ -4882,6 +5076,7 @@ struct CNetworkVideo::Impl
if (compared && !changed)
{
peer.lastCaptureTimeNs = current->captureTimeNs;
peer.updateDamage = {};
return true;
}
return QueueFrameNL(
@@ -5014,14 +5209,17 @@ struct CNetworkVideo::Impl
encoded->sourceSerial == m_latestSourceSerial)
m_latestFrame = encoded;
for (PeerState& peer : m_peers)
if (peer.subscribed && peer.configured &&
PeerFrameDueNL(peer, *encoded))
if (peer.subscribed && peer.configured)
{
if (!QueueCurrentFrameNL(peer, encoded,
frameID, encoded->captureTimeNs))
QueueStatusNL(peer, LG_NET_VIDEO_STATE_STALLED, 2);
else
QueueStatusNL(peer, LG_NET_VIDEO_STATE_STREAMING, 0);
AccumulatePeerDamageNL(peer, encoded->damage);
if (PeerFrameDueNL(peer, *encoded))
{
if (!QueueCurrentFrameNL(peer, encoded,
frameID, encoded->captureTimeNs))
QueueStatusNL(peer, LG_NET_VIDEO_STATE_STALLED, 2);
else
QueueStatusNL(peer, LG_NET_VIDEO_STATE_STREAMING, 0);
}
}
}

2
repos/LGProtocol

Submodule repos/LGProtocol updated: a8f5a21480...7913a08aba