Mirrored_Repos/LookingGlass
1
0
Fork 0
mirror of https://github.com/gnif/LookingGlass.git synced 2026-11-12 14:57:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[client/idd] network: send cumulative PyroWave updates

Browse Source
This commit is contained in:
Geoffrey McRae
2026-10-08 11:02:21 +11:00
parent 2575f0791a
commit ccdcad14ca
8 changed files with 842 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

283
client/transports/Network/video.c
View File

@@ -45,6 +45,8 @@
#define NETWORK_VIDEO_MAX_HEIGHT 8192U
#define NETWORK_VIDEO_MAX_FRAME_LENGTH \
LG_NET_VIDEO_MAX_FRAME_LENGTH
#define NETWORK_VIDEO_MAX_FRAME_UPDATE_LENGTH \
LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH
#define NETWORK_VIDEO_MAX_FRAGMENTS LG_NET_VIDEO_MAX_FRAGMENTS
#define NETWORK_VIDEO_MAX_BLOCKS LG_NET_VIDEO_MAX_BLOCKS
#define NETWORK_VIDEO_MAX_FRAME_LATENCY_MS 50U
@@ -73,15 +75,18 @@ struct NetworkVideoAssembly
{
bool active;
bool sparseAllowed;
bool update;
uint64_t generation;
uint64_t configEpoch;
uint64_t frameID;
uint64_t baseFrameID;
uint64_t firstReceiveNs;
uint64_t deadlineNs;
uint32_t frameLength;
uint32_t fragmentCount;
uint32_t blockCount;
uint32_t receivedBytes;
uint32_t receivedFragments;
uint8_t * data;
uint64_t * coverage;
@@ -127,10 +132,12 @@ struct NetworkVideoEncoded
uint32_t blockCount;
uint64_t generation;
uint64_t frameID;
uint64_t baseFrameID;
uint64_t captureTimestampNs;
uint64_t presentationTimestampNs;
uint64_t receiveTimeNs;
LGNetVideoFrameFlags flags;
bool update;
bool complete;
bool scheduleValid;
LGNetVideoSchedule schedule;
@@ -589,11 +596,11 @@ static void networkVideoSendFeedbackNL(NetworkVideo * video, uint64_t now)
if (!lgNetVideoFeedbackEncode(payload, sizeof(payload), &feedback))
return;
const LG_TransportStatus status = video->datagramsAvailable ?
networkVideoSendDatagramNL(video, LG_NET_VIDEO_MESSAGE_FEEDBACK,
video->config.configEpoch, payload, sizeof(payload)) :
networkVideoSendReliableNL(video, LG_NET_VIDEO_MESSAGE_FEEDBACK, 0,
video->config.configEpoch, 0, payload, sizeof(payload));
/* Feedback acknowledges the reference frame that future cumulative
* updates depend on, so it must not be lossy. */
const LG_TransportStatus status = networkVideoSendReliableNL(video,
LG_NET_VIDEO_MESSAGE_FEEDBACK, 0, video->config.configEpoch, 0,
payload, sizeof(payload));
if (status == LG_TRANSPORT_OK)
{
video->feedbackFlags = 0;
@@ -708,25 +715,30 @@ static bool networkVideoFindScheduleNL(NetworkVideo * video,
}
static bool networkVideoAllocateAssemblyNL(NetworkVideo * video,
const LGNetVideoFragment * fragment, uint64_t now,
const LGNetVideoFragment * fragment, bool update, uint64_t baseFrameID,
uint64_t now,
struct NetworkVideoAssembly * failed)
{
const uint32_t maxFrameLength = update ?
NETWORK_VIDEO_MAX_FRAME_UPDATE_LENGTH : NETWORK_VIDEO_MAX_FRAME_LENGTH;
*failed = (struct NetworkVideoAssembly){0};
if (fragment->frameLength > NETWORK_VIDEO_MAX_FRAME_LENGTH ||
if (fragment->frameLength > maxFrameLength ||
fragment->frameLength > video->config.maxFrameLength ||
fragment->fragmentCount > NETWORK_VIDEO_MAX_FRAGMENTS ||
fragment->blockCount > NETWORK_VIDEO_MAX_BLOCKS)
return false;
const bool sparseAllowed =
(video->config.flags & LG_NET_VIDEO_STREAM_DATAGRAMS) != 0;
!update && (video->config.flags & LG_NET_VIDEO_STREAM_DATAGRAMS) != 0;
struct NetworkVideoAssembly assembly =
{
.active = true,
.sparseAllowed = sparseAllowed,
.update = update,
.generation = video->decodeGeneration,
.configEpoch = fragment->configEpoch,
.frameID = fragment->frameID,
.baseFrameID = baseFrameID,
.firstReceiveNs = now,
.deadlineNs = sparseAllowed ?
now + (uint64_t)video->config.maxFrameLatencyMs * UINT64_C(1000000) :
@@ -775,7 +787,9 @@ static bool networkVideoBlockComplete(
static bool networkVideoAssemblyComplete(
const struct NetworkVideoAssembly * assembly)
{
if (assembly->receivedBytes != assembly->frameLength)
if (assembly->receivedBytes != assembly->frameLength ||
(assembly->update &&
assembly->receivedFragments != assembly->fragmentCount))
return false;
for (uint32_t index = 0; index < assembly->blockCount; ++index)
if (!networkVideoBlockComplete(&assembly->blocks[index]))
@@ -790,7 +804,7 @@ static bool networkVideoCompactAssemblyNL(NetworkVideo * video,
/* Transport block zero contains the sequence header and first codec block.
* Complete later blocks remain independently decodable; absent codec block
* indices are zero-filled by the decoder. */
if (!assembly->active || !assembly->sparseAllowed ||
if (!assembly->active || assembly->update || !assembly->sparseAllowed ||
networkVideoAssemblyComplete(assembly) || !assembly->blockCount ||
assembly->blockCount > NETWORK_VIDEO_PYROWAVE_COUNT_MASK)
return false;
@@ -861,12 +875,14 @@ static bool networkVideoCompactAssemblyNL(NetworkVideo * video,
.blockCount = compactBlocks,
.generation = assembly->generation,
.frameID = assembly->frameID,
.baseFrameID = 0,
.captureTimestampNs = assembly->scheduleValid ?
assembly->schedule.captureTimestampNs : now,
.presentationTimestampNs = assembly->scheduleValid ?
assembly->schedule.presentationTimestampNs : now,
.receiveTimeNs = now - assembly->firstReceiveNs,
.flags = LG_NET_VIDEO_FRAME_KEYFRAME,
.update = false,
.complete = false,
.scheduleValid = assembly->scheduleValid,
.schedule = assembly->schedule,
@@ -875,7 +891,8 @@ static bool networkVideoCompactAssemblyNL(NetworkVideo * video,
}
static bool networkVideoAddFragmentNL(NetworkVideo * video,
const LGNetVideoFragment * fragment, uint64_t now,
const LGNetVideoFragment * fragment, bool update, uint64_t baseFrameID,
uint64_t now,
struct NetworkVideoEncoded encoded[2], size_t * encodedCount,
struct NetworkVideoAssembly retired[2], size_t * retiredCount)
{
@@ -884,14 +901,18 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video,
{
if (assembly->active)
{
if (assembly->update && update &&
assembly->baseFrameID != baseFrameID)
return false;
video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames);
if (*encodedCount < 2U &&
if (!assembly->update && !update && *encodedCount < 2U &&
networkVideoCompactAssemblyNL(video, assembly, now,
&encoded[*encodedCount]))
++*encodedCount;
retired[(*retiredCount)++] = networkVideoTakeAssemblyNL(video);
}
if (!networkVideoAllocateAssemblyNL(video, fragment, now,
if (!networkVideoAllocateAssemblyNL(video, fragment,
update, baseFrameID, now,
&retired[*retiredCount]))
{
++*retiredCount;
@@ -902,7 +923,9 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video,
else if (fragment->frameID < assembly->frameID)
return true;
if (assembly->configEpoch != fragment->configEpoch ||
if (assembly->update != update ||
assembly->baseFrameID != baseFrameID ||
assembly->configEpoch != fragment->configEpoch ||
assembly->frameLength != fragment->frameLength ||
assembly->fragmentCount != fragment->fragmentCount ||
assembly->blockCount != fragment->blockCount)
@@ -946,6 +969,7 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video,
fragment->payload, fragment->payloadLength))
return false;
networkVideoSetFragmentSeen(assembly, fragment->fragmentIndex);
++assembly->receivedFragments;
assembly->fragmentOffsets[fragment->fragmentIndex] =
fragment->frameOffset;
assembly->fragmentLengths[fragment->fragmentIndex] =
@@ -960,6 +984,7 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video,
networkVideoSetCoverage(assembly,
fragment->frameOffset, fragment->payloadLength);
networkVideoSetFragmentSeen(assembly, fragment->fragmentIndex);
++assembly->receivedFragments;
assembly->fragmentOffsets[fragment->fragmentIndex] = fragment->frameOffset;
assembly->fragmentLengths[fragment->fragmentIndex] = fragment->payloadLength;
assembly->fragmentBlocks[fragment->fragmentIndex] = fragment->blockIndex;
@@ -987,12 +1012,15 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video,
.blockCount = assembly->blockCount,
.generation = assembly->generation,
.frameID = assembly->frameID,
.baseFrameID = assembly->baseFrameID,
.captureTimestampNs = assembly->scheduleValid ?
assembly->schedule.captureTimestampNs : now,
.presentationTimestampNs = assembly->scheduleValid ?
assembly->schedule.presentationTimestampNs : now,
.receiveTimeNs = now - assembly->firstReceiveNs,
.flags = LG_NET_VIDEO_FRAME_KEYFRAME,
.flags = assembly->update ?
0 : LG_NET_VIDEO_FRAME_KEYFRAME,
.update = assembly->update,
.complete = true,
.scheduleValid = assembly->scheduleValid,
.schedule = assembly->schedule,
@@ -1055,7 +1083,8 @@ static void networkVideoDecode(NetworkVideo * video,
NetworkVideoDecodedFrame decoded;
const NetworkVideoDecodeResult result = networkVideoDecoderDecode(
video->decoder, &encoded->config, encoded->data,
encoded->length, encoded->blockCount, &decoded);
encoded->length, encoded->blockCount, encoded->update,
encoded->complete, encoded->baseFrameID, encoded->frameID, &decoded);
free(encoded->ownedData);
if (result != NETWORK_VIDEO_DECODE_OK)
{
@@ -1116,15 +1145,31 @@ static void networkVideoQueueDecode(NetworkVideo * video,
return;
}
struct NetworkVideoEncoded replaced = {0};
bool queued = false;
struct NetworkVideoEncoded replaced = {0};
bool baseMismatch;
bool queued = false;
bool replaceable;
LG_LOCK(video->stateLock);
baseMismatch = encoded->update && video->decodePending &&
video->pendingDecode.update &&
video->pendingDecode.baseFrameID != encoded->baseFrameID;
replaceable = !video->decodePending ||
!encoded->update ||
(video->pendingDecode.update &&
video->pendingDecode.baseFrameID == encoded->baseFrameID);
if (baseMismatch)
{
video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames);
networkVideoRequestKeyframeNL(video,
LG_NET_VIDEO_CONTROL_CORRUPT, encoded->frameID);
}
if (!video->destroying && !video->decodeStopping && video->connected &&
video->wireConfigured && video->rendererAttached &&
video->decoderReady &&
video->decodeGeneration == encoded->generation &&
video->config.configEpoch == encoded->config.configEpoch &&
encoded->frameID > video->lastDecodedFrameID &&
(!encoded->update || encoded->complete) && replaceable &&
(!video->decodePending ||
encoded->frameID > video->pendingDecode.frameID))
{
@@ -1366,10 +1411,12 @@ static bool networkVideoHandleFrame(NetworkVideo * video,
.blockCount = frame.blockCount,
.generation = video->decodeGeneration,
.frameID = frame.frameID,
.baseFrameID = 0,
.captureTimestampNs = frame.captureTimestampNs,
.presentationTimestampNs = frame.presentationTimestampNs,
.receiveTimeNs = nanotime() - receiveStart,
.flags = frame.flags,
.update = false,
.complete = true,
};
encoded.scheduleValid = networkVideoFindScheduleNL(video,
@@ -1394,6 +1441,105 @@ static bool networkVideoHandleFrame(NetworkVideo * video,
return true;
}
static bool networkVideoHandleFrameUpdate(NetworkVideo * video,
const LGNetPacketView * packet)
{
LGNetVideoFrameUpdate update;
if (lgNetVideoFrameUpdateDecode(&update,
packet->payload, packet->payloadLength) != LG_NET_PARSE_OK)
return false;
if (update.flags & LG_NET_VIDEO_FRAME_UPDATE_HAS_CHECKSUM)
{
uint32_t checksum;
if (!lgNetCRC32C(update.data, update.encodedLength, &checksum) ||
checksum != update.checksum)
{
LG_LOCK(video->stateLock);
video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames);
networkVideoRequestKeyframeNL(video,
LG_NET_VIDEO_CONTROL_CORRUPT, update.frameID);
LG_UNLOCK(video->stateLock);
return true;
}
}
const uint64_t receiveStart = nanotime();
struct NetworkVideoAssembly retiredAssembly = {0};
struct NetworkVideoEncoded encoded;
bool baseMismatch = false;
LG_LOCK(video->stateLock);
if (!video->wireConfigured || update.streamID != video->config.streamID ||
update.configEpoch != video->config.configEpoch ||
update.codec != video->config.codec ||
update.encodedLength > NETWORK_VIDEO_MAX_FRAME_UPDATE_LENGTH ||
update.encodedLength > video->config.maxFrameLength ||
update.deadlineMs > video->config.maxFrameLatencyMs ||
(packet->envelope.componentEpoch &&
packet->envelope.componentEpoch != update.configEpoch))
{
LG_UNLOCK(video->stateLock);
return true;
}
if (update.frameID <= video->lastDecodedFrameID ||
update.frameID < video->highestSeenFrameID ||
(update.frameID == video->highestSeenFrameID &&
(!video->assembly.active ||
video->assembly.frameID != update.frameID)))
{
LG_UNLOCK(video->stateLock);
return true;
}
if (video->assembly.active &&
video->assembly.frameID <= update.frameID)
{
video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames);
baseMismatch = video->assembly.update &&
video->assembly.baseFrameID != update.baseFrameID;
retiredAssembly = networkVideoTakeAssemblyNL(video);
if (baseMismatch)
networkVideoRequestKeyframeNL(video,
LG_NET_VIDEO_CONTROL_CORRUPT, update.frameID);
}
if (!baseMismatch && update.frameID > video->highestSeenFrameID)
video->highestSeenFrameID = update.frameID;
encoded = (struct NetworkVideoEncoded)
{
.config = video->config,
.data = update.data,
.length = update.encodedLength,
.blockCount = update.blockCount,
.generation = video->decodeGeneration,
.frameID = update.frameID,
.baseFrameID = update.baseFrameID,
.captureTimestampNs = update.captureTimestampNs,
.presentationTimestampNs = update.presentationTimestampNs,
.receiveTimeNs = nanotime() - receiveStart,
.update = true,
.complete = true,
};
if (!baseMismatch)
encoded.scheduleValid = networkVideoFindScheduleNL(video,
update.configEpoch, update.frameID, &encoded.schedule);
LG_UNLOCK(video->stateLock);
networkVideoClearAssembly(&retiredAssembly);
if (baseMismatch)
return true;
encoded.ownedData = malloc(encoded.length);
if (!encoded.ownedData)
{
networkVideoDecodeFailure(video, &encoded, NETWORK_VIDEO_DECODE_ERROR);
return true;
}
memcpy(encoded.ownedData, encoded.data, encoded.length);
encoded.data = encoded.ownedData;
encoded.receiveTimeNs = nanotime() - receiveStart;
networkVideoQueueDecode(video, &encoded);
return true;
}
static bool networkVideoHandleFragment(NetworkVideo * video,
const LGNetPacketView * packet)
{
@@ -1441,7 +1587,7 @@ static bool networkVideoHandleFragment(NetworkVideo * video,
}
if (fragment.frameID > video->highestSeenFrameID)
video->highestSeenFrameID = fragment.frameID;
valid = networkVideoAddFragmentNL(video, &fragment, now,
valid = networkVideoAddFragmentNL(video, &fragment, false, 0, now,
encoded, &encodedCount, retired, &retiredCount);
if (!valid)
{
@@ -1459,6 +1605,97 @@ static bool networkVideoHandleFragment(NetworkVideo * video,
return true;
}
static bool networkVideoHandleFrameUpdateFragment(NetworkVideo * video,
const LGNetPacketView * packet)
{
LGNetVideoFrameUpdateFragment update;
if (lgNetVideoFrameUpdateFragmentDecode(&update,
packet->payload, packet->payloadLength) != LG_NET_PARSE_OK)
return false;
if (update.flags & LG_NET_VIDEO_FRAGMENT_CHECKSUM)
{
uint32_t checksum;
if (!lgNetCRC32C(update.payload,
update.payloadLength, &checksum) || checksum != update.checksum)
{
LG_LOCK(video->stateLock);
video->lostFragments = networkVideoSaturatingIncrement(
video->lostFragments);
networkVideoRequestKeyframeNL(video,
LG_NET_VIDEO_CONTROL_CORRUPT, update.frameID);
LG_UNLOCK(video->stateLock);
return true;
}
}
const LGNetVideoFragment fragment =
{
.streamID = update.streamID,
.flags = update.flags,
.recoveryType = update.recoveryType,
.configEpoch = update.configEpoch,
.frameID = update.frameID,
.frameOffset = update.frameOffset,
.frameLength = update.frameLength,
.blockIndex = update.blockIndex,
.blockOffset = update.blockOffset,
.blockLength = update.blockLength,
.blockCount = update.blockCount,
.fragmentIndex = update.fragmentIndex,
.fragmentCount = update.fragmentCount,
.recoveryGroup = update.recoveryGroup,
.recoveryIndex = update.recoveryIndex,
.recoveryCount = update.recoveryCount,
.payloadLength = update.payloadLength,
.checksum = update.checksum,
.payload = update.payload,
};
struct NetworkVideoAssembly retired[2] = {0};
struct NetworkVideoEncoded encoded[2] = {0};
size_t retiredCount = 0;
size_t encodedCount = 0;
bool valid;
const uint64_t now = nanotime();
LG_LOCK(video->stateLock);
if (!video->wireConfigured ||
update.streamID != video->config.streamID ||
update.configEpoch != video->config.configEpoch ||
update.frameLength > NETWORK_VIDEO_MAX_FRAME_UPDATE_LENGTH ||
update.frameLength > video->config.maxFrameLength ||
update.payloadLength > video->config.maxFragmentLength ||
(packet->envelope.componentEpoch &&
packet->envelope.componentEpoch != update.configEpoch) ||
update.frameID <= video->lastDecodedFrameID ||
update.frameID < video->highestSeenFrameID ||
(update.frameID == video->highestSeenFrameID &&
(!video->assembly.active ||
video->assembly.frameID != update.frameID)))
{
LG_UNLOCK(video->stateLock);
return true;
}
if (update.frameID > video->highestSeenFrameID)
video->highestSeenFrameID = update.frameID;
valid = networkVideoAddFragmentNL(video, &fragment, true,
update.baseFrameID, now,
encoded, &encodedCount, retired, &retiredCount);
if (!valid)
{
video->lostFragments = networkVideoSaturatingIncrement(
video->lostFragments);
if (video->assembly.active)
retired[retiredCount++] = networkVideoTakeAssemblyNL(video);
networkVideoRequestKeyframeNL(video,
LG_NET_VIDEO_CONTROL_CORRUPT, update.frameID);
}
LG_UNLOCK(video->stateLock);
networkVideoClearAssemblies(retired, retiredCount);
for (size_t index = 0; index < encodedCount; ++index)
networkVideoQueueDecode(video, &encoded[index]);
return true;
}
static bool networkVideoHandleSchedule(NetworkVideo * video,
const LGNetPacketView * packet)
{
@@ -1601,6 +1838,14 @@ static bool networkVideoHandlePacket(NetworkVideo * video,
return networkVideoUnsolicited(packet) &&
networkVideoHandleFragment(video, packet);
case LG_NET_VIDEO_MESSAGE_FRAME_UPDATE:
return !datagram && networkVideoUnsolicited(packet) &&
networkVideoHandleFrameUpdate(video, packet);
case LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT:
return !datagram && networkVideoUnsolicited(packet) &&
networkVideoHandleFrameUpdateFragment(video, packet);
case LG_NET_VIDEO_MESSAGE_SCHEDULE:
return !datagram && !bulk && networkVideoUnsolicited(packet) &&
networkVideoHandleSchedule(video, packet);

1
client/transports/Network/video/decoder.h
View File

@@ -63,6 +63,7 @@ NetworkVideoDecodeResult networkVideoDecoderConfigure(
NetworkVideoDecodeResult networkVideoDecoderDecode(
NetworkVideoDecoder * decoder, const LGNetVideoStreamConfig * config,
const void * data, size_t size, uint32_t blockCount,
bool update, bool complete, uint64_t baseFrameID, uint64_t frameID,
NetworkVideoDecodedFrame * frame);
void networkVideoDecoderRelease(NetworkVideoDecodedFrame * frame);

58
client/transports/Network/video/decoder_egl.c
View File

@@ -125,6 +125,7 @@ struct NetworkVideoUniforms
GLint outputLayer;
GLint blockOffset;
GLint blockStride;
GLint preserveMissing;
}
dequant;
@@ -181,6 +182,7 @@ struct NetworkVideoDecoder
uint32_t alignedWidth;
uint32_t alignedHeight;
uint32_t blockCapacity;
uint64_t referenceFrameID;
unsigned uploadIndex;
GLint maxTextureSize;
GLint maxArrayLayers;
@@ -188,6 +190,7 @@ struct NetworkVideoDecoder
GLint64 maxStorageBlockSize;
bool attached;
bool configured;
bool referenceValid;
};
struct NetworkVideoExternalFrame
@@ -468,13 +471,15 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder)
free(decoder->offsets);
memset(decoder->uploads, 0, sizeof(decoder->uploads));
decoder->texturePool = NULL;
decoder->offsets = NULL;
decoder->alignedWidth = 0;
decoder->alignedHeight = 0;
decoder->blockCapacity = 0;
decoder->uploadIndex = 0;
decoder->configured = false;
decoder->texturePool = NULL;
decoder->offsets = NULL;
decoder->alignedWidth = 0;
decoder->alignedHeight = 0;
decoder->blockCapacity = 0;
decoder->referenceFrameID = 0;
decoder->uploadIndex = 0;
decoder->configured = false;
decoder->referenceValid = false;
memset(decoder->coefficientTextures, 0,
sizeof(decoder->coefficientTextures));
memset(decoder->planeTextures, 0, sizeof(decoder->planeTextures));
@@ -498,14 +503,16 @@ static void networkVideoDeleteProgramsNL(NetworkVideoDecoder * decoder)
static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder)
{
decoder->uniforms.dequant.resolution = glGetUniformLocation(
decoder->uniforms.dequant.resolution = glGetUniformLocation(
decoder->dequantProgram, "resolution");
decoder->uniforms.dequant.outputLayer = glGetUniformLocation(
decoder->uniforms.dequant.outputLayer = glGetUniformLocation(
decoder->dequantProgram, "outputLayer");
decoder->uniforms.dequant.blockOffset = glGetUniformLocation(
decoder->uniforms.dequant.blockOffset = glGetUniformLocation(
decoder->dequantProgram, "blockOffset");
decoder->uniforms.dequant.blockStride = glGetUniformLocation(
decoder->uniforms.dequant.blockStride = glGetUniformLocation(
decoder->dequantProgram, "blockStride");
decoder->uniforms.dequant.preserveMissing = glGetUniformLocation(
decoder->dequantProgram, "preserveMissing");
decoder->uniforms.idwt.coefficients = glGetUniformLocation(
decoder->idwtProgram, "coefficients");
@@ -544,6 +551,7 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder)
uniforms->dequant.outputLayer < 0 ||
uniforms->dequant.blockOffset < 0 ||
uniforms->dequant.blockStride < 0 ||
uniforms->dequant.preserveMissing < 0 ||
uniforms->idwt.coefficients < 0 ||
uniforms->idwt.resolution < 0 ||
uniforms->idwt.invResolution < 0 ||
@@ -861,7 +869,9 @@ void networkVideoDecoderDetach(NetworkVideoDecoder * decoder)
decoder->maxWorkGroupCount[0] = 0;
decoder->maxWorkGroupCount[1] = 0;
decoder->maxStorageBlockSize = 0;
decoder->referenceFrameID = 0;
decoder->attached = false;
decoder->referenceValid = false;
LG_UNLOCK(decoder->lock);
}
@@ -891,6 +901,8 @@ void networkVideoDecoderReset(NetworkVideoDecoder * decoder)
networkVideoRestoreCurrent(decoder, &previous);
}
}
decoder->referenceFrameID = 0;
decoder->referenceValid = false;
LG_UNLOCK(decoder->lock);
}
@@ -1102,11 +1114,12 @@ static struct NetworkVideoUpload * networkVideoUploadNL(
}
static void networkVideoDequantNL(NetworkVideoDecoder * decoder,
const struct NetworkVideoUpload * upload)
const struct NetworkVideoUpload * upload, bool preserveMissing)
{
glUseProgram(decoder->dequantProgram);
glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, upload->offsetBuffer);
glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, upload->payloadBuffer);
glUniform1i(decoder->uniforms.dequant.preserveMissing, preserveMissing);
for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level)
{
@@ -1284,6 +1297,7 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL(
NetworkVideoDecodeResult networkVideoDecoderDecode(
NetworkVideoDecoder * decoder, const LGNetVideoStreamConfig * config,
const void * data, size_t size, uint32_t blockCount,
bool update, bool complete, uint64_t baseFrameID, uint64_t frameID,
NetworkVideoDecodedFrame * result)
{
if (!decoder || !config || !data || !size || !result)
@@ -1298,11 +1312,19 @@ NetworkVideoDecodeResult networkVideoDecoderDecode(
return NETWORK_VIDEO_DECODE_UNSUPPORTED;
}
if ((!update && baseFrameID) || (update && !complete) ||
(update && (!decoder->referenceValid ||
decoder->referenceFrameID != baseFrameID || frameID <= baseFrameID)))
{
LG_UNLOCK(decoder->lock);
return NETWORK_VIDEO_DECODE_CORRUPT;
}
NetworkPyroWaveLayout layout;
const NetworkPyroWaveResult parse = networkPyroWaveParse(
config, data, size, blockCount, decoder->offsets,
decoder->blockCapacity, &layout);
if (parse != NETWORK_PYROWAVE_OK)
if (parse != NETWORK_PYROWAVE_OK || layout.delta != update)
{
LG_UNLOCK(decoder->lock);
return parse == NETWORK_PYROWAVE_UNSUPPORTED ?
@@ -1329,7 +1351,7 @@ NetworkVideoDecodeResult networkVideoDecoderDecode(
}
const uint64_t decodeStart = nanotime();
networkVideoDequantNL(decoder, upload);
networkVideoDequantNL(decoder, upload, update);
upload->readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
networkVideoIDWTNL(decoder);
struct NetworkVideoExternalFrame * external =
@@ -1339,12 +1361,20 @@ NetworkVideoDecodeResult networkVideoDecoderDecode(
networkVideoRestoreCurrent(decoder, &previous);
if (!success)
{
decoder->referenceFrameID = 0;
decoder->referenceValid = false;
if (external)
networkVideoExternalRelease(&external->base);
LG_UNLOCK(decoder->lock);
return NETWORK_VIDEO_DECODE_ERROR;
}
if (!update)
{
decoder->referenceFrameID = complete ? frameID : 0;
decoder->referenceValid = complete;
}
uint32_t formatVersion = (uint32_t)config->configEpoch;
if (!formatVersion)
formatVersion = 1;

29
client/transports/Network/video/pyrowave.c
View File

@@ -252,6 +252,7 @@ NetworkPyroWaveResult networkPyroWaveParse(
uint8_t frameSequence = 0;
bool haveSequence = false;
bool haveHeader = false;
bool haveClear = false;
bool valid = true;
while (valid && size - cursor >= NETWORK_PYROWAVE_HEADER_SIZE)
@@ -273,11 +274,15 @@ NetworkPyroWaveResult networkPyroWaveParse(
if (extended)
{
const uint32_t code = (second >> 24) & 3U;
const uint32_t code =
(second >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) &
LG_NET_VIDEO_PYROWAVE_MODE_MASK;
const uint32_t width = (first & UINT32_C(0x00003FFF)) + 1U;
const uint32_t height = ((first >> 14) & UINT32_C(0x00003FFF)) + 1U;
const uint32_t blocks = second & UINT32_C(0x00FFFFFF);
if (code || width != config->width || height != config->height ||
if ((code != LG_NET_VIDEO_PYROWAVE_MODE_FULL &&
code != LG_NET_VIDEO_PYROWAVE_MODE_DELTA) ||
width != config->width || height != config->height ||
!blocks || blocks > capacity ||
!networkPyroWaveSequenceProfile(config, second) ||
(haveHeader &&
@@ -299,26 +304,38 @@ NetworkPyroWaveResult networkPyroWaveParse(
const uint32_t payloadWords = (first >> 16) & UINT32_C(0x00000FFF);
const uint32_t blockIndex = second >> 8;
const size_t packetSize = (size_t)payloadWords * sizeof(uint32_t);
if (!ballot || payloadWords < 2U || packetSize > size - cursor ||
const bool clear = ballot == 0;
if (payloadWords < 2U || packetSize > size - cursor ||
blockIndex >= capacity || offsets[blockIndex] != UINT32_MAX ||
!networkPyroWavePacketValid(
bytes + cursor, packetSize, (uint16_t)ballot))
(clear ? payloadWords != LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_WORDS :
!networkPyroWavePacketValid(
bytes + cursor, packetSize, (uint16_t)ballot)))
{
valid = false;
break;
}
offsets[blockIndex] = (uint32_t)(cursor / sizeof(uint32_t));
if (clear)
haveClear = true;
++uniqueBlocks;
cursor += packetSize;
}
valid = valid && cursor == size && haveHeader && haveSequence &&
declaredBlocks == uniqueBlocks && expectedBlocks == uniqueBlocks;
declaredBlocks == uniqueBlocks && expectedBlocks == uniqueBlocks &&
(!haveClear ||
((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) &
LG_NET_VIDEO_PYROWAVE_MODE_MASK) ==
LG_NET_VIDEO_PYROWAVE_MODE_DELTA);
if (!valid)
return NETWORK_PYROWAVE_INVALID;
layout->sequence = frameSequence;
layout->chromaLeft = (headerSecond >> 31) != 0;
layout->delta =
((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) &
LG_NET_VIDEO_PYROWAVE_MODE_MASK) ==
LG_NET_VIDEO_PYROWAVE_MODE_DELTA;
return NETWORK_PYROWAVE_OK;
}

1
client/transports/Network/video/pyrowave.h
View File

@@ -38,6 +38,7 @@ typedef struct NetworkPyroWaveLayout
{
uint8_t sequence;
bool chromaLeft;
bool delta;
}
NetworkPyroWaveLayout;

14
client/transports/Network/video/shaders.h
View File

@@ -58,6 +58,7 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_0[] =
"uniform int outputLayer;\n"
"uniform int blockOffset;\n"
"uniform int blockStride;\n"
"uniform bool preserveMissing;\n"
"shared uint signOffset;\n"
"shared uint planeOffset[16];\n"
"shared uint significantPrefix[128];\n"
@@ -166,10 +167,13 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_1[] =
" }\n"
" barrier();\n"
" uint bitCursor = signOffset + significantPrefix[localIndex];\n"
" uint word = payload[bitCursor >> 5u];\n"
" uint upper = payload[(bitCursor >> 5u) + 1u];\n"
" uint shift = bitCursor & 31u;\n"
" if (shift != 0u) word = (word >> shift) | (upper << (32u-shift));\n"
" uint word = 0u;\n"
" if (significant != 0u) {\n"
" word = payload[bitCursor >> 5u];\n"
" uint upper = payload[(bitCursor >> 5u) + 1u];\n"
" uint shift = bitCursor & 31u;\n"
" if (shift != 0u) word = (word >> shift) | (upper << (32u-shift));\n"
" }\n"
" uint signIndex = 0u;\n"
" for (int x=0; x<4; ++x) for (int y=0; y<2; ++y) {\n"
" if (values[y][x] != 0.0) {\n"
@@ -177,7 +181,7 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_1[] =
" ++signIndex;\n"
" }\n"
" }\n"
" storeValues(coord,values);\n"
" if (!preserveMissing || blockPresent != 0u) storeValues(coord,values);\n"
"}\n";
static const char NETWORK_VIDEO_IDWT_SHADER_0[] =

549
idd/LGIdd/transport/network/CNetworkVideo.cpp
View File

@@ -95,6 +95,13 @@ namespace
constexpr uint32_t PYROWAVE_FRAGMENT_MIN =
LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE + PYROWAVE_FRAGMENT_DATA_MIN;
constexpr uint32_t PYROWAVE_PACKET_META_WORDS = 2;
constexpr uint32_t PYROWAVE_SEQUENCE_MASK = 7;
constexpr uint32_t PYROWAVE_HEADER_CODE_MASK =
LG_NET_VIDEO_PYROWAVE_MODE_MASK <<
LG_NET_VIDEO_PYROWAVE_MODE_SHIFT;
constexpr uint32_t PYROWAVE_HEADER_CODE_DELTA =
LG_NET_VIDEO_PYROWAVE_MODE_DELTA <<
LG_NET_VIDEO_PYROWAVE_MODE_SHIFT;
constexpr uint32_t PYROWAVE_QUANT_WORDS_8X8 = 40;
constexpr uint32_t PYROWAVE_STATE_BYTES = 16;
constexpr uint32_t PYROWAVE_QUANT_LEVELS = 19;
@@ -227,6 +234,22 @@ namespace
return value ? value : 1;
}
uint32_t LoadLE32(const uint8_t * data)
{
return static_cast<uint32_t>(data[0]) |
static_cast<uint32_t>(data[1]) << 8U |
static_cast<uint32_t>(data[2]) << 16U |
static_cast<uint32_t>(data[3]) << 24U;
}
void StoreLE32(uint8_t * data, uint32_t value)
{
data[0] = static_cast<uint8_t>(value);
data[1] = static_cast<uint8_t>(value >> 8U);
data[2] = static_cast<uint8_t>(value >> 16U);
data[3] = static_cast<uint8_t>(value >> 24U);
}
LGNetEnvelopeFlags DeliveryFlag(NetworkVideoDelivery delivery)
{
switch (delivery)
@@ -243,6 +266,12 @@ namespace
return 0;
}
bool FragmentMessage(uint16_t messageType)
{
return messageType == LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT ||
messageType == LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT;
}
unsigned DeliveryIndex(NetworkVideoDelivery delivery)
{
switch (delivery)
@@ -1384,6 +1413,7 @@ void FinalizeMain()
{
uint32_t offset = 0;
uint32_t length = 0;
uint32_t index = 0;
};
struct EncodedFrame
@@ -1393,11 +1423,46 @@ void FinalizeMain()
uint64_t frameID = 0;
uint64_t captureTimeNs = 0;
uint32_t deadlineMs = 0;
uint32_t blockCapacity = 0;
std::vector<uint8_t> data;
std::vector<EncodedBlock> blocks;
std::shared_ptr<const std::vector<uint8_t>> wholePayload;
};
struct EncodedPacket
{
const uint8_t * data = nullptr;
uint32_t length = 0;
};
EncodedPacket Packet(const EncodedFrame& frame,
const EncodedBlock& block)
{
const uint32_t prefix = block.offset ? 0U : 8U;
if (block.length <= prefix || block.offset > frame.data.size() ||
block.length > frame.data.size() - block.offset)
return {};
return
{
frame.data.data() + block.offset + prefix,
block.length - prefix,
};
}
bool SamePacket(const EncodedPacket& left, const EncodedPacket& right)
{
if (!left.data || !right.data || left.length != right.length ||
left.length < 8U)
return left.data == right.data && left.length == right.length;
constexpr uint32_t SEQUENCE_BITS = PYROWAVE_SEQUENCE_MASK << 28U;
if ((LoadLE32(left.data) & ~SEQUENCE_BITS) !=
(LoadLE32(right.data) & ~SEQUENCE_BITS))
return false;
return std::equal(left.data + sizeof(uint32_t),
left.data + left.length, right.data + sizeof(uint32_t));
}
struct ShaderParameters
{
uint32_t dim0 [4] = {};
@@ -2204,14 +2269,6 @@ void FinalizeMain()
slot.packed.Get(), 0, slot.frameLimit);
}
static uint32_t LoadLE32(const uint8_t * data)
{
return static_cast<uint32_t>(data[0]) |
static_cast<uint32_t>(data[1]) << 8U |
static_cast<uint32_t>(data[2]) << 16U |
static_cast<uint32_t>(data[3]) << 24U;
}
bool Compact(const Slot& slot, const uint8_t * readback,
EncodedFrame& output) const
{
@@ -2279,7 +2336,7 @@ void FinalizeMain()
std::copy_n(packet, length,
output.data.begin() + outputOffset);
output.blocks.push_back({ outputOffset, length });
output.blocks.push_back({ outputOffset, length, blockIndex });
outputOffset += length;
}
if (output.blocks.size() != packetCount ||
@@ -2293,6 +2350,7 @@ void FinalizeMain()
output.blocks[0].offset = 0;
output.blocks[0].length += 8U;
output.blockCapacity = m_blockCapacity;
return true;
}
@@ -2651,29 +2709,33 @@ struct CNetworkVideo::Impl
{
struct PeerState
{
NetworkVideoPeer peer = {};
NetworkVideoPeerInfo info = {};
LGNetVideoSubscribe subscription = {};
LGNetVideoFeedback feedback = {};
LGNetVideoState lastStatusState = {};
uint64_t serial = 0;
uint64_t componentEpoch = 0;
uint64_t statusSequence = 0;
uint64_t statusEpoch = 0;
uint64_t lastFrameID = 0;
uint64_t lastCompleteID = 0;
uint64_t lastCaptureTimeNs = 0;
uint64_t sendReadyGeneration[3] = {};
uint32_t lastStatusDetail = 0;
uint32_t lastStatusFlags = 0;
uint32_t droppedFrames = 0;
uint8_t blockedDeliveries = 0;
bool statusSent = false;
bool subscribed = false;
bool configured = false;
bool congested = false;
bool keyframePending = false;
bool sendFailed = false;
NetworkVideoPeer peer = {};
NetworkVideoPeerInfo info = {};
LGNetVideoSubscribe subscription = {};
LGNetVideoFeedback feedback = {};
LGNetVideoState lastStatusState = {};
uint64_t serial = 0;
uint64_t componentEpoch = 0;
uint64_t statusSequence = 0;
uint64_t statusEpoch = 0;
uint64_t lastFrameID = 0;
uint64_t lastCompleteID = 0;
uint64_t lastCaptureTimeNs = 0;
uint64_t referenceFrameID = 0;
uint64_t sendReadyGeneration[3] = {};
uint32_t lastStatusDetail = 0;
uint32_t lastStatusFlags = 0;
uint32_t droppedFrames = 0;
uint8_t blockedDeliveries = 0;
std::shared_ptr<const EncodedFrame> referenceFrame;
std::vector<uint8_t> updateBlocks;
bool statusSent = false;
bool subscribed = false;
bool configured = false;
bool congested = false;
bool keyframePending = false;
bool referencePending = false;
bool sendFailed = false;
};
struct PendingFrame
@@ -2696,6 +2758,7 @@ struct CNetworkVideo::Impl
uint64_t componentEpoch = 0;
uint64_t requestID = 0;
uint64_t frameID = 0;
uint64_t baseFrameID = 0;
std::shared_ptr<const std::vector<uint8_t>> payload;
std::shared_ptr<const EncodedFrame> encoded;
uint32_t fragmentIndex = 0;
@@ -2847,6 +2910,22 @@ struct CNetworkVideo::Impl
return false;
}
static void ResetPeerReferenceNL(PeerState& peer)
{
peer.referenceFrame.reset();
peer.updateBlocks.clear();
peer.referenceFrameID = 0;
peer.referencePending = false;
}
static bool PeerNeedsReferenceNL(
const PeerState& peer, const EncodedFrame& frame)
{
return peer.keyframePending || !peer.referenceFrame ||
peer.referenceFrame->configEpoch != frame.configEpoch ||
peer.referenceFrame->blockCapacity != frame.blockCapacity;
}
uint32_t FrameLimitNL(uint64_t captureTimeNs)
{
uint64_t period = VIDEO_RATE_DEFAULT_PERIOD_NS;
@@ -2973,6 +3052,131 @@ struct CNetworkVideo::Impl
encoded.wholePayload->size() : 0, bytes);
}
static std::shared_ptr<const EncodedFrame> BuildUpdateFrame(
const EncodedFrame& reference, const EncodedFrame& current,
const std::vector<uint8_t>& previousBlocks,
std::vector<uint8_t>& updateBlocks)
{
updateBlocks.clear();
if (!reference.blockCapacity ||
reference.blockCapacity != current.blockCapacity ||
reference.configEpoch != current.configEpoch ||
reference.data.size() < 8U || current.data.size() < 8U)
return {};
try
{
updateBlocks = previousBlocks;
updateBlocks.resize(current.blockCapacity, 0);
size_t referencePosition = 0;
size_t currentPosition = 0;
for (uint32_t index = 0; index < current.blockCapacity; ++index)
{
while (referencePosition < reference.blocks.size() &&
reference.blocks[referencePosition].index < index)
++referencePosition;
while (currentPosition < current.blocks.size() &&
current.blocks[currentPosition].index < index)
++currentPosition;
const EncodedBlock * referenceBlock =
referencePosition < reference.blocks.size() &&
reference.blocks[referencePosition].index == index ?
&reference.blocks[referencePosition] : nullptr;
const EncodedBlock * currentBlock =
currentPosition < current.blocks.size() &&
current.blocks[currentPosition].index == index ?
&current.blocks[currentPosition] : nullptr;
if (!SamePacket(referenceBlock ? Packet(reference, *referenceBlock) :
EncodedPacket {},
currentBlock ? Packet(current, *currentBlock) :
EncodedPacket {}))
updateBlocks[index] = 1;
}
uint32_t operationCount = 0;
for (uint8_t changed : updateBlocks)
operationCount += changed != 0;
if (!operationCount || operationCount > UINT32_C(0x00ffffff))
return {};
std::shared_ptr<EncodedFrame> update =
std::make_shared<EncodedFrame>();
update->configEpoch = current.configEpoch;
update->sourceSerial = current.sourceSerial;
update->frameID = current.frameID;
update->captureTimeNs = current.captureTimeNs;
update->deadlineMs = current.deadlineMs;
update->blockCapacity = current.blockCapacity;
update->blocks.reserve(operationCount);
update->data.reserve(current.data.size());
update->data.resize(8U);
const uint32_t firstWord = LoadLE32(current.data.data());
const uint32_t secondWord =
(LoadLE32(current.data.data() + sizeof(uint32_t)) &
~(PYROWAVE_HEADER_CODE_MASK | UINT32_C(0x00ffffff))) |
PYROWAVE_HEADER_CODE_DELTA | operationCount;
StoreLE32(update->data.data(), firstWord);
StoreLE32(update->data.data() + sizeof(uint32_t), secondWord);
const uint32_t sequence =
(firstWord >> 28U) & PYROWAVE_SEQUENCE_MASK;
currentPosition = 0;
for (uint32_t index = 0; index < current.blockCapacity; ++index)
{
while (currentPosition < current.blocks.size() &&
current.blocks[currentPosition].index < index)
++currentPosition;
if (!updateBlocks[index])
continue;
const EncodedBlock * block =
currentPosition < current.blocks.size() &&
current.blocks[currentPosition].index == index ?
&current.blocks[currentPosition] : nullptr;
const uint32_t offset =
static_cast<uint32_t>(update->data.size());
if (block)
{
const EncodedPacket packet = Packet(current, *block);
if (!packet.data || packet.length < 8U ||
packet.length > LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH -
update->data.size())
return {};
update->data.insert(update->data.end(),
packet.data, packet.data + packet.length);
update->blocks.push_back({ offset, packet.length, index });
}
else
{
if (update->data.size() > LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH -
LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_SIZE)
return {};
update->data.resize(update->data.size() +
LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_SIZE);
uint8_t * packet = update->data.data() + offset;
StoreLE32(packet,
(LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_WORDS << 16U) |
(sequence << 28U));
StoreLE32(packet + sizeof(uint32_t), index << 8U);
update->blocks.push_back(
{ offset, LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_SIZE, index });
}
}
if (update->blocks.size() != operationCount ||
update->data.size() > LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH)
return {};
update->blocks[0].offset = 0;
update->blocks[0].length += 8U;
return update;
}
catch (...)
{
updateBlocks.clear();
return {};
}
}
bool EncodedQueuedNL(const EncodedFrame * encoded,
uint64_t excludingSerial = 0) const
{
@@ -3058,7 +3262,9 @@ struct CNetworkVideo::Impl
peer.componentEpoch = m_configEpoch;
peer.configured = true;
peer.lastCaptureTimeNs = 0;
peer.keyframePending = true;
peer.sendFailed = false;
ResetPeerReferenceNL(peer);
return QueueStatusNL(peer, LG_NET_VIDEO_STATE_CONFIGURED, 0);
}
@@ -3111,8 +3317,11 @@ struct CNetworkVideo::Impl
m_frames.clear();
m_latestFrame.reset();
for (PeerState& peer : m_peers)
{
ResetPeerReferenceNL(peer);
if (peer.subscribed)
QueueStatusNL(peer, LG_NET_VIDEO_STATE_ERROR, detail);
}
}
void DropPeerFramesNL(PeerState& peer)
@@ -3123,7 +3332,7 @@ struct CNetworkVideo::Impl
uint64_t activeFrame = 0;
for (const SendItem& item : m_sends)
if (item.frameID && item.started &&
item.messageType == LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT &&
FragmentMessage(item.messageType) &&
SamePeer(item.peer, peer.peer))
{
activeFrame = item.frameID;
@@ -3216,6 +3425,47 @@ struct CNetworkVideo::Impl
return payload;
}
static std::shared_ptr<const std::vector<uint8_t>> BuildUpdatePayload(
const EncodedFrame& encoded, uint64_t baseFrameID,
uint32_t deadlineMs, uint64_t frameID,
uint64_t presentationBaseNs)
{
LGNetVideoFrameUpdate update = {};
update.streamID = STREAM_ID;
update.codec = LG_NET_VIDEO_CODEC_PYROWAVE;
update.configEpoch = encoded.configEpoch;
update.frameID = frameID;
update.baseFrameID = baseFrameID;
update.captureTimestampNs = encoded.captureTimeNs;
update.presentationTimestampNs = presentationBaseNs +
static_cast<uint64_t>(deadlineMs) * UINT64_C(1000000);
update.encodedLength =
static_cast<uint32_t>(encoded.data.size());
update.blockCount =
static_cast<uint32_t>(encoded.blocks.size());
update.fragmentCount = 1;
update.deadlineMs = deadlineMs;
update.data = encoded.data.data();
const size_t size = lgNetVideoFrameUpdateSize(&update);
if (!size || size > UINT32_MAX)
return {};
std::shared_ptr<std::vector<uint8_t>> payload;
try
{
payload = std::make_shared<std::vector<uint8_t>>(size);
}
catch (...)
{
return {};
}
if (!lgNetVideoFrameUpdateEncode(
payload->data(), payload->size(), &update))
return {};
return payload;
}
bool QueueWholeFrameNL(PeerState& peer, const EncodedFrame& encoded,
NetworkVideoDelivery delivery, uint64_t frameID,
uint64_t presentationBaseNs)
@@ -3236,16 +3486,33 @@ struct CNetworkVideo::Impl
std::move(payload));
}
bool QueueWholeUpdateNL(PeerState& peer, const EncodedFrame& encoded,
NetworkVideoDelivery delivery, uint64_t baseFrameID, uint64_t frameID,
uint64_t presentationBaseNs)
{
const uint32_t deadlineMs = (std::min)(
encoded.deadlineMs, peer.subscription.maxFrameLatencyMs);
std::shared_ptr<const std::vector<uint8_t>> payload =
BuildUpdatePayload(encoded, baseFrameID,
deadlineMs, frameID, presentationBaseNs);
if (!payload)
return false;
return QueueNL(peer.peer, delivery,
LG_NET_VIDEO_MESSAGE_FRAME_UPDATE, LG_NET_ENVELOPE_FINAL,
encoded.configEpoch, 0, frameID, std::move(payload));
}
static bool FragmentPlan(const EncodedFrame& encoded,
uint32_t maxPayload, uint32_t firstBlockIndex,
uint32_t firstFragmentIndex,
uint32_t& payloadLimit, uint32_t& fragmentCount)
uint32_t& payloadLimit, uint32_t& fragmentCount,
uint32_t headerSize = LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE)
{
if (maxPayload <= LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE ||
if (maxPayload <= headerSize ||
firstBlockIndex > encoded.blocks.size())
return false;
payloadLimit = (std::min)(
maxPayload - LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE,
maxPayload - headerSize,
static_cast<uint32_t>(LG_NET_VIDEO_MAX_FRAGMENT_LENGTH));
if (!payloadLimit)
return false;
@@ -3292,16 +3559,22 @@ struct CNetworkVideo::Impl
bool QueueFragmentsNL(PeerState& peer,
std::shared_ptr<const EncodedFrame> encoded,
NetworkVideoDelivery delivery, uint64_t frameID, uint32_t maxPayload,
uint32_t firstBlockIndex = 0,
uint32_t firstFragmentIndex = 0)
uint32_t firstBlockIndex = 0,
uint32_t firstFragmentIndex = 0,
uint16_t messageType = LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT,
uint64_t baseFrameID = 0)
{
if (!encoded)
const bool update =
messageType == LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT;
if (!encoded || (update != (baseFrameID != 0)))
return false;
uint32_t payloadLimit;
uint32_t fragmentCount;
if (!FragmentPlan(*encoded, maxPayload, firstBlockIndex,
firstFragmentIndex,
payloadLimit, fragmentCount))
payloadLimit, fragmentCount,
update ? LG_NET_VIDEO_FRAME_UPDATE_FRAGMENT_HEADER_WIRE_SIZE :
LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE))
return false;
if (firstBlockIndex == encoded->blocks.size())
return true;
@@ -3318,10 +3591,11 @@ struct CNetworkVideo::Impl
item.serial = Next64(m_nextSendSerial);
item.peer = peer.peer;
item.delivery = delivery;
item.messageType = LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT;
item.messageType = messageType;
item.flags = LG_NET_ENVELOPE_FINAL;
item.componentEpoch = encoded->configEpoch;
item.frameID = frameID;
item.baseFrameID = baseFrameID;
item.encoded = std::move(encoded);
item.fragmentIndex = firstFragmentIndex;
item.fragmentCount = fragmentCount;
@@ -3406,6 +3680,51 @@ struct CNetworkVideo::Impl
const uint32_t length = (std::min)(
item.payloadLimit, block.length - item.blockOffset);
if (item.messageType == LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT)
{
LGNetVideoFrameUpdateFragment fragment = {};
fragment.streamID = STREAM_ID;
fragment.configEpoch = item.encoded->configEpoch;
fragment.frameID = item.frameID;
fragment.baseFrameID = item.baseFrameID;
fragment.frameOffset = block.offset + item.blockOffset;
fragment.frameLength =
static_cast<uint32_t>(item.encoded->data.size());
fragment.blockIndex = item.blockIndex;
fragment.blockOffset = item.blockOffset;
fragment.blockLength = block.length;
fragment.blockCount =
static_cast<uint32_t>(item.encoded->blocks.size());
fragment.fragmentIndex = item.fragmentIndex;
fragment.fragmentCount = item.fragmentCount;
fragment.payloadLength = length;
fragment.payload =
item.encoded->data.data() + fragment.frameOffset;
if (!item.blockOffset)
fragment.flags |= LG_NET_VIDEO_FRAGMENT_BLOCK_START;
if (item.blockOffset + length == block.length)
fragment.flags |= LG_NET_VIDEO_FRAGMENT_BLOCK_END;
if (static_cast<size_t>(fragment.frameOffset) + length ==
item.encoded->data.size())
fragment.flags |= LG_NET_VIDEO_FRAGMENT_FRAME_END;
const size_t size = lgNetVideoFrameUpdateFragmentSize(&fragment);
if (!size)
return false;
try
{
payload.resize(size);
}
catch (...)
{
return false;
}
return lgNetVideoFrameUpdateFragmentEncode(
payload.data(), payload.size(), &fragment);
}
if (item.messageType != LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT)
return false;
LGNetVideoFragment fragment = {};
fragment.streamID = STREAM_ID;
fragment.configEpoch = item.encoded->configEpoch;
@@ -3465,7 +3784,8 @@ struct CNetworkVideo::Impl
bool QueueFrameNL(PeerState& peer,
const std::shared_ptr<const EncodedFrame>& frame,
uint64_t frameID, uint64_t presentationBaseNs)
uint64_t frameID, uint64_t presentationBaseNs,
bool establishReference = false)
{
if (!frame || !frameID || !presentationBaseNs || !peer.configured ||
peer.componentEpoch != frame->configEpoch ||
@@ -3473,8 +3793,19 @@ struct CNetworkVideo::Impl
peer.subscription.maxFrameLength)
return false;
std::vector<uint8_t> cleanBlocks;
if (establishReference)
try
{
cleanBlocks.assign(frame->blockCapacity, 0);
}
catch (...)
{
return false;
}
DropPeerFramesNL(peer);
const bool datagrams =
const bool datagrams = !establishReference &&
(peer.subscription.flags & LG_NET_VIDEO_SUBSCRIBE_ALLOW_DATAGRAMS) &&
DatagramFragmentsSupported(peer.info);
const bool bulk = BulkFragmentsSupported(peer.info);
@@ -3535,12 +3866,108 @@ struct CNetworkVideo::Impl
}
peer.lastFrameID = frameID;
peer.lastCaptureTimeNs = frame->captureTimeNs;
peer.keyframePending = false;
if (establishReference)
{
peer.referenceFrame = frame;
peer.referenceFrameID = frameID;
peer.referencePending = true;
peer.updateBlocks = std::move(cleanBlocks);
}
peer.keyframePending = establishReference;
return true;
}
bool QueueUpdateNL(PeerState& peer,
const std::shared_ptr<const EncodedFrame>& update,
uint64_t baseFrameID, uint64_t frameID,
uint64_t presentationBaseNs)
{
if (!update || !baseFrameID || baseFrameID >= frameID ||
!presentationBaseNs || !peer.configured ||
peer.referencePending || peer.referenceFrameID != baseFrameID ||
peer.componentEpoch != update->configEpoch ||
update->data.empty() || update->data.size() >
peer.subscription.maxFrameLength)
return false;
DropPeerFramesNL(peer);
const bool bulk = BulkFragmentsSupported(peer.info);
NetworkVideoDelivery delivery = bulk ?
NetworkVideoDelivery::BULK : NetworkVideoDelivery::RELIABLE;
const uint32_t fullSize =
static_cast<uint32_t>(update->data.size()) +
LG_NET_VIDEO_FRAME_UPDATE_HEADER_WIRE_SIZE;
const uint32_t maxPayload = DeliveryPayload(peer.info, delivery);
bool result;
if (fullSize <= maxPayload)
result = QueueWholeUpdateNL(peer, *update, delivery,
baseFrameID, frameID, presentationBaseNs);
else
result = QueueScheduleNL(peer, *update, frameID,
presentationBaseNs) &&
QueueFragmentsNL(peer, update, delivery, frameID, maxPayload,
0, 0, LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT, baseFrameID);
if (!result)
{
DropPeerFramesNL(peer);
++peer.droppedFrames;
peer.congested = true;
return false;
}
peer.lastFrameID = frameID;
peer.lastCaptureTimeNs = update->captureTimeNs;
return true;
}
bool QueueCurrentFrameNL(PeerState& peer,
const std::shared_ptr<const EncodedFrame>& current,
uint64_t frameID, uint64_t presentationBaseNs)
{
if (!current)
return false;
if (peer.referencePending)
return true;
if (PeerNeedsReferenceNL(peer, *current))
return QueueFrameNL(
peer, current, frameID, presentationBaseNs, true);
std::vector<uint8_t> updateBlocks;
const std::shared_ptr<const EncodedFrame> update = BuildUpdateFrame(
*peer.referenceFrame, *current, peer.updateBlocks,
updateBlocks);
const bool compared = updateBlocks.size() == current->blockCapacity;
const bool changed = std::any_of(updateBlocks.begin(),
updateBlocks.end(), [](uint8_t value) { return value != 0; });
if (!update)
{
if (compared && !changed)
{
peer.lastCaptureTimeNs = current->captureTimeNs;
return true;
}
return QueueFrameNL(
peer, current, frameID, presentationBaseNs, true);
}
const uint64_t fullWireSize = current->data.size() +
LG_NET_VIDEO_FRAME_HEADER_WIRE_SIZE;
const uint64_t updateWireSize = update->data.size() +
LG_NET_VIDEO_FRAME_UPDATE_HEADER_WIRE_SIZE;
if (updateWireSize >= fullWireSize)
return QueueFrameNL(
peer, current, frameID, presentationBaseNs, true);
if (!QueueUpdateNL(peer, update, peer.referenceFrameID,
frameID, presentationBaseNs))
return false;
peer.updateBlocks = std::move(updateBlocks);
return true;
}
bool ReplayLatestFrameNL(PeerState& peer)
{
if (peer.referencePending)
return true;
if (!m_latestFrame || !m_latestSourceSerial ||
m_latestFrame->configEpoch != m_configEpoch ||
m_latestFrame->sourceSerial != m_latestSourceSerial ||
@@ -3548,7 +3975,7 @@ struct CNetworkVideo::Impl
return false;
const uint64_t frameID = Next64(m_nextFrameID);
if (!QueueFrameNL(peer, m_latestFrame, frameID,
if (!QueueCurrentFrameNL(peer, m_latestFrame, frameID,
CFrameScheduler::Nanotime()))
return false;
@@ -3562,7 +3989,10 @@ struct CNetworkVideo::Impl
bool queued = false;
CSRWExclusiveLock lock(m_lock);
for (PeerState& peer : m_peers)
if (peer.subscribed && peer.configured && peer.keyframePending)
if (peer.subscribed && peer.configured &&
(peer.keyframePending ||
(!peer.referencePending && peer.referenceFrame && m_latestFrame &&
m_latestFrame->captureTimeNs > peer.lastCaptureTimeNs)))
queued |= ReplayLatestFrameNL(peer);
return queued;
}
@@ -3604,6 +4034,8 @@ struct CNetworkVideo::Impl
return;
for (const PeerState& peer : m_peers)
if (peer.subscribed && peer.configured &&
!peer.referencePending &&
PeerNeedsReferenceNL(peer, frame) &&
PeerFrameDueNL(peer, frame) &&
PeerUsesWholeFrameNL(peer, frame))
{
@@ -3647,7 +4079,7 @@ struct CNetworkVideo::Impl
if (peer.subscribed && peer.configured &&
PeerFrameDueNL(peer, *encoded))
{
if (!QueueFrameNL(peer, encoded,
if (!QueueCurrentFrameNL(peer, encoded,
frameID, encoded->captureTimeNs))
QueueStatusNL(peer, LG_NET_VIDEO_STATE_STALLED, 2);
else
@@ -3796,11 +4228,10 @@ struct CNetworkVideo::Impl
}
return;
}
if (item.messageType == LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT &&
item.frameID)
if (FragmentMessage(item.messageType) && item.frameID)
for (SendItem& queued : m_sends)
if (queued.frameID == item.frameID &&
queued.messageType == LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT &&
FragmentMessage(queued.messageType) &&
SamePeer(queued.peer, item.peer))
queued.started = true;
madeProgress = true;
@@ -3962,6 +4393,7 @@ void CNetworkVideo::Stop()
{
peer.configured = false;
peer.componentEpoch = 0;
m_impl->ResetPeerReferenceNL(peer);
}
}
m_impl->m_encoder.Reset();
@@ -4083,6 +4515,7 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket(
const bool hadSubscribers = m_impl->HasSubscribersNL();
m_impl->DropPeerSendsNL(*peer);
m_impl->ResetPeerReferenceNL(*peer);
peer->subscription = subscribe;
peer->subscribed = true;
peer->configured = false;
@@ -4128,6 +4561,7 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket(
peer->configured = false;
peer->componentEpoch = 0;
peer->keyframePending = false;
m_impl->ResetPeerReferenceNL(*peer);
if (!m_impl->HasSubscribersNL())
m_impl->m_lastBudgetCaptureTimeNs = 0;
break;
@@ -4151,6 +4585,8 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket(
return NetworkVideoPacketResult::IGNORED;
if (!peer->subscribed)
return NetworkVideoPacketResult::INVALID;
m_impl->DropPeerSendsNL(*peer);
m_impl->ResetPeerReferenceNL(*peer);
peer->keyframePending = true;
m_impl->ReplayLatestFrameNL(*peer);
break;
@@ -4173,16 +4609,29 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket(
return NetworkVideoPacketResult::IGNORED;
if (!peer->subscribed)
return NetworkVideoPacketResult::INVALID;
bool referenceAccepted = false;
peer->feedback = feedback;
peer->lastCompleteID = feedback.lastCompleteFrameID;
peer->congested =
feedback.flags & LG_NET_VIDEO_FEEDBACK_CONGESTED;
if (peer->referencePending && peer->referenceFrameID &&
feedback.lastCompleteFrameID >= peer->referenceFrameID &&
feedback.lastDecodedFrameID >= peer->referenceFrameID)
{
peer->referencePending = false;
peer->keyframePending = false;
referenceAccepted = true;
}
if (feedback.flags & (LG_NET_VIDEO_FEEDBACK_REQUEST_KEYFRAME |
LG_NET_VIDEO_FEEDBACK_DECODER_STALLED))
{
m_impl->DropPeerSendsNL(*peer);
m_impl->ResetPeerReferenceNL(*peer);
peer->keyframePending = true;
m_impl->ReplayLatestFrameNL(*peer);
}
else if (referenceAccepted)
m_impl->QueueStatusNL(*peer, LG_NET_VIDEO_STATE_STREAMING, 0);
break;
}

2
repos/LGProtocol

Submodule repos/LGProtocol updated: a2a41197d0...a8f5a21480