diff --git a/client/transports/Network/video.c b/client/transports/Network/video.c index 22c3ec99..f5b87664 100644 --- a/client/transports/Network/video.c +++ b/client/transports/Network/video.c @@ -45,6 +45,8 @@ #define NETWORK_VIDEO_MAX_HEIGHT 8192U #define NETWORK_VIDEO_MAX_FRAME_LENGTH \ LG_NET_VIDEO_MAX_FRAME_LENGTH +#define NETWORK_VIDEO_MAX_FRAME_UPDATE_LENGTH \ + LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH #define NETWORK_VIDEO_MAX_FRAGMENTS LG_NET_VIDEO_MAX_FRAGMENTS #define NETWORK_VIDEO_MAX_BLOCKS LG_NET_VIDEO_MAX_BLOCKS #define NETWORK_VIDEO_MAX_FRAME_LATENCY_MS 50U @@ -73,15 +75,18 @@ struct NetworkVideoAssembly { bool active; bool sparseAllowed; + bool update; uint64_t generation; uint64_t configEpoch; uint64_t frameID; + uint64_t baseFrameID; uint64_t firstReceiveNs; uint64_t deadlineNs; uint32_t frameLength; uint32_t fragmentCount; uint32_t blockCount; uint32_t receivedBytes; + uint32_t receivedFragments; uint8_t * data; uint64_t * coverage; @@ -127,10 +132,12 @@ struct NetworkVideoEncoded uint32_t blockCount; uint64_t generation; uint64_t frameID; + uint64_t baseFrameID; uint64_t captureTimestampNs; uint64_t presentationTimestampNs; uint64_t receiveTimeNs; LGNetVideoFrameFlags flags; + bool update; bool complete; bool scheduleValid; LGNetVideoSchedule schedule; @@ -589,11 +596,11 @@ static void networkVideoSendFeedbackNL(NetworkVideo * video, uint64_t now) if (!lgNetVideoFeedbackEncode(payload, sizeof(payload), &feedback)) return; - const LG_TransportStatus status = video->datagramsAvailable ? - networkVideoSendDatagramNL(video, LG_NET_VIDEO_MESSAGE_FEEDBACK, - video->config.configEpoch, payload, sizeof(payload)) : - networkVideoSendReliableNL(video, LG_NET_VIDEO_MESSAGE_FEEDBACK, 0, - video->config.configEpoch, 0, payload, sizeof(payload)); + /* Feedback acknowledges the reference frame that future cumulative + * updates depend on, so it must not be lossy. */ + const LG_TransportStatus status = networkVideoSendReliableNL(video, + LG_NET_VIDEO_MESSAGE_FEEDBACK, 0, video->config.configEpoch, 0, + payload, sizeof(payload)); if (status == LG_TRANSPORT_OK) { video->feedbackFlags = 0; @@ -708,25 +715,30 @@ static bool networkVideoFindScheduleNL(NetworkVideo * video, } static bool networkVideoAllocateAssemblyNL(NetworkVideo * video, - const LGNetVideoFragment * fragment, uint64_t now, + const LGNetVideoFragment * fragment, bool update, uint64_t baseFrameID, + uint64_t now, struct NetworkVideoAssembly * failed) { + const uint32_t maxFrameLength = update ? + NETWORK_VIDEO_MAX_FRAME_UPDATE_LENGTH : NETWORK_VIDEO_MAX_FRAME_LENGTH; *failed = (struct NetworkVideoAssembly){0}; - if (fragment->frameLength > NETWORK_VIDEO_MAX_FRAME_LENGTH || + if (fragment->frameLength > maxFrameLength || fragment->frameLength > video->config.maxFrameLength || fragment->fragmentCount > NETWORK_VIDEO_MAX_FRAGMENTS || fragment->blockCount > NETWORK_VIDEO_MAX_BLOCKS) return false; const bool sparseAllowed = - (video->config.flags & LG_NET_VIDEO_STREAM_DATAGRAMS) != 0; + !update && (video->config.flags & LG_NET_VIDEO_STREAM_DATAGRAMS) != 0; struct NetworkVideoAssembly assembly = { .active = true, .sparseAllowed = sparseAllowed, + .update = update, .generation = video->decodeGeneration, .configEpoch = fragment->configEpoch, .frameID = fragment->frameID, + .baseFrameID = baseFrameID, .firstReceiveNs = now, .deadlineNs = sparseAllowed ? now + (uint64_t)video->config.maxFrameLatencyMs * UINT64_C(1000000) : @@ -775,7 +787,9 @@ static bool networkVideoBlockComplete( static bool networkVideoAssemblyComplete( const struct NetworkVideoAssembly * assembly) { - if (assembly->receivedBytes != assembly->frameLength) + if (assembly->receivedBytes != assembly->frameLength || + (assembly->update && + assembly->receivedFragments != assembly->fragmentCount)) return false; for (uint32_t index = 0; index < assembly->blockCount; ++index) if (!networkVideoBlockComplete(&assembly->blocks[index])) @@ -790,7 +804,7 @@ static bool networkVideoCompactAssemblyNL(NetworkVideo * video, /* Transport block zero contains the sequence header and first codec block. * Complete later blocks remain independently decodable; absent codec block * indices are zero-filled by the decoder. */ - if (!assembly->active || !assembly->sparseAllowed || + if (!assembly->active || assembly->update || !assembly->sparseAllowed || networkVideoAssemblyComplete(assembly) || !assembly->blockCount || assembly->blockCount > NETWORK_VIDEO_PYROWAVE_COUNT_MASK) return false; @@ -861,12 +875,14 @@ static bool networkVideoCompactAssemblyNL(NetworkVideo * video, .blockCount = compactBlocks, .generation = assembly->generation, .frameID = assembly->frameID, + .baseFrameID = 0, .captureTimestampNs = assembly->scheduleValid ? assembly->schedule.captureTimestampNs : now, .presentationTimestampNs = assembly->scheduleValid ? assembly->schedule.presentationTimestampNs : now, .receiveTimeNs = now - assembly->firstReceiveNs, .flags = LG_NET_VIDEO_FRAME_KEYFRAME, + .update = false, .complete = false, .scheduleValid = assembly->scheduleValid, .schedule = assembly->schedule, @@ -875,7 +891,8 @@ static bool networkVideoCompactAssemblyNL(NetworkVideo * video, } static bool networkVideoAddFragmentNL(NetworkVideo * video, - const LGNetVideoFragment * fragment, uint64_t now, + const LGNetVideoFragment * fragment, bool update, uint64_t baseFrameID, + uint64_t now, struct NetworkVideoEncoded encoded[2], size_t * encodedCount, struct NetworkVideoAssembly retired[2], size_t * retiredCount) { @@ -884,14 +901,18 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video, { if (assembly->active) { + if (assembly->update && update && + assembly->baseFrameID != baseFrameID) + return false; video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); - if (*encodedCount < 2U && + if (!assembly->update && !update && *encodedCount < 2U && networkVideoCompactAssemblyNL(video, assembly, now, &encoded[*encodedCount])) ++*encodedCount; retired[(*retiredCount)++] = networkVideoTakeAssemblyNL(video); } - if (!networkVideoAllocateAssemblyNL(video, fragment, now, + if (!networkVideoAllocateAssemblyNL(video, fragment, + update, baseFrameID, now, &retired[*retiredCount])) { ++*retiredCount; @@ -902,7 +923,9 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video, else if (fragment->frameID < assembly->frameID) return true; - if (assembly->configEpoch != fragment->configEpoch || + if (assembly->update != update || + assembly->baseFrameID != baseFrameID || + assembly->configEpoch != fragment->configEpoch || assembly->frameLength != fragment->frameLength || assembly->fragmentCount != fragment->fragmentCount || assembly->blockCount != fragment->blockCount) @@ -946,6 +969,7 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video, fragment->payload, fragment->payloadLength)) return false; networkVideoSetFragmentSeen(assembly, fragment->fragmentIndex); + ++assembly->receivedFragments; assembly->fragmentOffsets[fragment->fragmentIndex] = fragment->frameOffset; assembly->fragmentLengths[fragment->fragmentIndex] = @@ -960,6 +984,7 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video, networkVideoSetCoverage(assembly, fragment->frameOffset, fragment->payloadLength); networkVideoSetFragmentSeen(assembly, fragment->fragmentIndex); + ++assembly->receivedFragments; assembly->fragmentOffsets[fragment->fragmentIndex] = fragment->frameOffset; assembly->fragmentLengths[fragment->fragmentIndex] = fragment->payloadLength; assembly->fragmentBlocks[fragment->fragmentIndex] = fragment->blockIndex; @@ -987,12 +1012,15 @@ static bool networkVideoAddFragmentNL(NetworkVideo * video, .blockCount = assembly->blockCount, .generation = assembly->generation, .frameID = assembly->frameID, + .baseFrameID = assembly->baseFrameID, .captureTimestampNs = assembly->scheduleValid ? assembly->schedule.captureTimestampNs : now, .presentationTimestampNs = assembly->scheduleValid ? assembly->schedule.presentationTimestampNs : now, .receiveTimeNs = now - assembly->firstReceiveNs, - .flags = LG_NET_VIDEO_FRAME_KEYFRAME, + .flags = assembly->update ? + 0 : LG_NET_VIDEO_FRAME_KEYFRAME, + .update = assembly->update, .complete = true, .scheduleValid = assembly->scheduleValid, .schedule = assembly->schedule, @@ -1055,7 +1083,8 @@ static void networkVideoDecode(NetworkVideo * video, NetworkVideoDecodedFrame decoded; const NetworkVideoDecodeResult result = networkVideoDecoderDecode( video->decoder, &encoded->config, encoded->data, - encoded->length, encoded->blockCount, &decoded); + encoded->length, encoded->blockCount, encoded->update, + encoded->complete, encoded->baseFrameID, encoded->frameID, &decoded); free(encoded->ownedData); if (result != NETWORK_VIDEO_DECODE_OK) { @@ -1116,15 +1145,31 @@ static void networkVideoQueueDecode(NetworkVideo * video, return; } - struct NetworkVideoEncoded replaced = {0}; - bool queued = false; + struct NetworkVideoEncoded replaced = {0}; + bool baseMismatch; + bool queued = false; + bool replaceable; LG_LOCK(video->stateLock); + baseMismatch = encoded->update && video->decodePending && + video->pendingDecode.update && + video->pendingDecode.baseFrameID != encoded->baseFrameID; + replaceable = !video->decodePending || + !encoded->update || + (video->pendingDecode.update && + video->pendingDecode.baseFrameID == encoded->baseFrameID); + if (baseMismatch) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_CORRUPT, encoded->frameID); + } if (!video->destroying && !video->decodeStopping && video->connected && video->wireConfigured && video->rendererAttached && video->decoderReady && video->decodeGeneration == encoded->generation && video->config.configEpoch == encoded->config.configEpoch && encoded->frameID > video->lastDecodedFrameID && + (!encoded->update || encoded->complete) && replaceable && (!video->decodePending || encoded->frameID > video->pendingDecode.frameID)) { @@ -1366,10 +1411,12 @@ static bool networkVideoHandleFrame(NetworkVideo * video, .blockCount = frame.blockCount, .generation = video->decodeGeneration, .frameID = frame.frameID, + .baseFrameID = 0, .captureTimestampNs = frame.captureTimestampNs, .presentationTimestampNs = frame.presentationTimestampNs, .receiveTimeNs = nanotime() - receiveStart, .flags = frame.flags, + .update = false, .complete = true, }; encoded.scheduleValid = networkVideoFindScheduleNL(video, @@ -1394,6 +1441,105 @@ static bool networkVideoHandleFrame(NetworkVideo * video, return true; } +static bool networkVideoHandleFrameUpdate(NetworkVideo * video, + const LGNetPacketView * packet) +{ + LGNetVideoFrameUpdate update; + if (lgNetVideoFrameUpdateDecode(&update, + packet->payload, packet->payloadLength) != LG_NET_PARSE_OK) + return false; + + if (update.flags & LG_NET_VIDEO_FRAME_UPDATE_HAS_CHECKSUM) + { + uint32_t checksum; + if (!lgNetCRC32C(update.data, update.encodedLength, &checksum) || + checksum != update.checksum) + { + LG_LOCK(video->stateLock); + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_CORRUPT, update.frameID); + LG_UNLOCK(video->stateLock); + return true; + } + } + + const uint64_t receiveStart = nanotime(); + struct NetworkVideoAssembly retiredAssembly = {0}; + struct NetworkVideoEncoded encoded; + bool baseMismatch = false; + LG_LOCK(video->stateLock); + if (!video->wireConfigured || update.streamID != video->config.streamID || + update.configEpoch != video->config.configEpoch || + update.codec != video->config.codec || + update.encodedLength > NETWORK_VIDEO_MAX_FRAME_UPDATE_LENGTH || + update.encodedLength > video->config.maxFrameLength || + update.deadlineMs > video->config.maxFrameLatencyMs || + (packet->envelope.componentEpoch && + packet->envelope.componentEpoch != update.configEpoch)) + { + LG_UNLOCK(video->stateLock); + return true; + } + if (update.frameID <= video->lastDecodedFrameID || + update.frameID < video->highestSeenFrameID || + (update.frameID == video->highestSeenFrameID && + (!video->assembly.active || + video->assembly.frameID != update.frameID))) + { + LG_UNLOCK(video->stateLock); + return true; + } + if (video->assembly.active && + video->assembly.frameID <= update.frameID) + { + video->lostFrames = networkVideoSaturatingIncrement(video->lostFrames); + baseMismatch = video->assembly.update && + video->assembly.baseFrameID != update.baseFrameID; + retiredAssembly = networkVideoTakeAssemblyNL(video); + if (baseMismatch) + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_CORRUPT, update.frameID); + } + if (!baseMismatch && update.frameID > video->highestSeenFrameID) + video->highestSeenFrameID = update.frameID; + encoded = (struct NetworkVideoEncoded) + { + .config = video->config, + .data = update.data, + .length = update.encodedLength, + .blockCount = update.blockCount, + .generation = video->decodeGeneration, + .frameID = update.frameID, + .baseFrameID = update.baseFrameID, + .captureTimestampNs = update.captureTimestampNs, + .presentationTimestampNs = update.presentationTimestampNs, + .receiveTimeNs = nanotime() - receiveStart, + .update = true, + .complete = true, + }; + if (!baseMismatch) + encoded.scheduleValid = networkVideoFindScheduleNL(video, + update.configEpoch, update.frameID, &encoded.schedule); + LG_UNLOCK(video->stateLock); + + networkVideoClearAssembly(&retiredAssembly); + if (baseMismatch) + return true; + + encoded.ownedData = malloc(encoded.length); + if (!encoded.ownedData) + { + networkVideoDecodeFailure(video, &encoded, NETWORK_VIDEO_DECODE_ERROR); + return true; + } + memcpy(encoded.ownedData, encoded.data, encoded.length); + encoded.data = encoded.ownedData; + encoded.receiveTimeNs = nanotime() - receiveStart; + networkVideoQueueDecode(video, &encoded); + return true; +} + static bool networkVideoHandleFragment(NetworkVideo * video, const LGNetPacketView * packet) { @@ -1441,7 +1587,7 @@ static bool networkVideoHandleFragment(NetworkVideo * video, } if (fragment.frameID > video->highestSeenFrameID) video->highestSeenFrameID = fragment.frameID; - valid = networkVideoAddFragmentNL(video, &fragment, now, + valid = networkVideoAddFragmentNL(video, &fragment, false, 0, now, encoded, &encodedCount, retired, &retiredCount); if (!valid) { @@ -1459,6 +1605,97 @@ static bool networkVideoHandleFragment(NetworkVideo * video, return true; } +static bool networkVideoHandleFrameUpdateFragment(NetworkVideo * video, + const LGNetPacketView * packet) +{ + LGNetVideoFrameUpdateFragment update; + if (lgNetVideoFrameUpdateFragmentDecode(&update, + packet->payload, packet->payloadLength) != LG_NET_PARSE_OK) + return false; + + if (update.flags & LG_NET_VIDEO_FRAGMENT_CHECKSUM) + { + uint32_t checksum; + if (!lgNetCRC32C(update.payload, + update.payloadLength, &checksum) || checksum != update.checksum) + { + LG_LOCK(video->stateLock); + video->lostFragments = networkVideoSaturatingIncrement( + video->lostFragments); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_CORRUPT, update.frameID); + LG_UNLOCK(video->stateLock); + return true; + } + } + + const LGNetVideoFragment fragment = + { + .streamID = update.streamID, + .flags = update.flags, + .recoveryType = update.recoveryType, + .configEpoch = update.configEpoch, + .frameID = update.frameID, + .frameOffset = update.frameOffset, + .frameLength = update.frameLength, + .blockIndex = update.blockIndex, + .blockOffset = update.blockOffset, + .blockLength = update.blockLength, + .blockCount = update.blockCount, + .fragmentIndex = update.fragmentIndex, + .fragmentCount = update.fragmentCount, + .recoveryGroup = update.recoveryGroup, + .recoveryIndex = update.recoveryIndex, + .recoveryCount = update.recoveryCount, + .payloadLength = update.payloadLength, + .checksum = update.checksum, + .payload = update.payload, + }; + struct NetworkVideoAssembly retired[2] = {0}; + struct NetworkVideoEncoded encoded[2] = {0}; + size_t retiredCount = 0; + size_t encodedCount = 0; + bool valid; + const uint64_t now = nanotime(); + LG_LOCK(video->stateLock); + if (!video->wireConfigured || + update.streamID != video->config.streamID || + update.configEpoch != video->config.configEpoch || + update.frameLength > NETWORK_VIDEO_MAX_FRAME_UPDATE_LENGTH || + update.frameLength > video->config.maxFrameLength || + update.payloadLength > video->config.maxFragmentLength || + (packet->envelope.componentEpoch && + packet->envelope.componentEpoch != update.configEpoch) || + update.frameID <= video->lastDecodedFrameID || + update.frameID < video->highestSeenFrameID || + (update.frameID == video->highestSeenFrameID && + (!video->assembly.active || + video->assembly.frameID != update.frameID))) + { + LG_UNLOCK(video->stateLock); + return true; + } + if (update.frameID > video->highestSeenFrameID) + video->highestSeenFrameID = update.frameID; + valid = networkVideoAddFragmentNL(video, &fragment, true, + update.baseFrameID, now, + encoded, &encodedCount, retired, &retiredCount); + if (!valid) + { + video->lostFragments = networkVideoSaturatingIncrement( + video->lostFragments); + if (video->assembly.active) + retired[retiredCount++] = networkVideoTakeAssemblyNL(video); + networkVideoRequestKeyframeNL(video, + LG_NET_VIDEO_CONTROL_CORRUPT, update.frameID); + } + LG_UNLOCK(video->stateLock); + networkVideoClearAssemblies(retired, retiredCount); + for (size_t index = 0; index < encodedCount; ++index) + networkVideoQueueDecode(video, &encoded[index]); + return true; +} + static bool networkVideoHandleSchedule(NetworkVideo * video, const LGNetPacketView * packet) { @@ -1601,6 +1838,14 @@ static bool networkVideoHandlePacket(NetworkVideo * video, return networkVideoUnsolicited(packet) && networkVideoHandleFragment(video, packet); + case LG_NET_VIDEO_MESSAGE_FRAME_UPDATE: + return !datagram && networkVideoUnsolicited(packet) && + networkVideoHandleFrameUpdate(video, packet); + + case LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT: + return !datagram && networkVideoUnsolicited(packet) && + networkVideoHandleFrameUpdateFragment(video, packet); + case LG_NET_VIDEO_MESSAGE_SCHEDULE: return !datagram && !bulk && networkVideoUnsolicited(packet) && networkVideoHandleSchedule(video, packet); diff --git a/client/transports/Network/video/decoder.h b/client/transports/Network/video/decoder.h index 4792f237..9e82a708 100644 --- a/client/transports/Network/video/decoder.h +++ b/client/transports/Network/video/decoder.h @@ -63,6 +63,7 @@ NetworkVideoDecodeResult networkVideoDecoderConfigure( NetworkVideoDecodeResult networkVideoDecoderDecode( NetworkVideoDecoder * decoder, const LGNetVideoStreamConfig * config, const void * data, size_t size, uint32_t blockCount, + bool update, bool complete, uint64_t baseFrameID, uint64_t frameID, NetworkVideoDecodedFrame * frame); void networkVideoDecoderRelease(NetworkVideoDecodedFrame * frame); diff --git a/client/transports/Network/video/decoder_egl.c b/client/transports/Network/video/decoder_egl.c index 61dd2df6..3dfe1678 100644 --- a/client/transports/Network/video/decoder_egl.c +++ b/client/transports/Network/video/decoder_egl.c @@ -125,6 +125,7 @@ struct NetworkVideoUniforms GLint outputLayer; GLint blockOffset; GLint blockStride; + GLint preserveMissing; } dequant; @@ -181,6 +182,7 @@ struct NetworkVideoDecoder uint32_t alignedWidth; uint32_t alignedHeight; uint32_t blockCapacity; + uint64_t referenceFrameID; unsigned uploadIndex; GLint maxTextureSize; GLint maxArrayLayers; @@ -188,6 +190,7 @@ struct NetworkVideoDecoder GLint64 maxStorageBlockSize; bool attached; bool configured; + bool referenceValid; }; struct NetworkVideoExternalFrame @@ -468,13 +471,15 @@ static void networkVideoDeleteConfigurationNL(NetworkVideoDecoder * decoder) free(decoder->offsets); memset(decoder->uploads, 0, sizeof(decoder->uploads)); - decoder->texturePool = NULL; - decoder->offsets = NULL; - decoder->alignedWidth = 0; - decoder->alignedHeight = 0; - decoder->blockCapacity = 0; - decoder->uploadIndex = 0; - decoder->configured = false; + decoder->texturePool = NULL; + decoder->offsets = NULL; + decoder->alignedWidth = 0; + decoder->alignedHeight = 0; + decoder->blockCapacity = 0; + decoder->referenceFrameID = 0; + decoder->uploadIndex = 0; + decoder->configured = false; + decoder->referenceValid = false; memset(decoder->coefficientTextures, 0, sizeof(decoder->coefficientTextures)); memset(decoder->planeTextures, 0, sizeof(decoder->planeTextures)); @@ -498,14 +503,16 @@ static void networkVideoDeleteProgramsNL(NetworkVideoDecoder * decoder) static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) { - decoder->uniforms.dequant.resolution = glGetUniformLocation( + decoder->uniforms.dequant.resolution = glGetUniformLocation( decoder->dequantProgram, "resolution"); - decoder->uniforms.dequant.outputLayer = glGetUniformLocation( + decoder->uniforms.dequant.outputLayer = glGetUniformLocation( decoder->dequantProgram, "outputLayer"); - decoder->uniforms.dequant.blockOffset = glGetUniformLocation( + decoder->uniforms.dequant.blockOffset = glGetUniformLocation( decoder->dequantProgram, "blockOffset"); - decoder->uniforms.dequant.blockStride = glGetUniformLocation( + decoder->uniforms.dequant.blockStride = glGetUniformLocation( decoder->dequantProgram, "blockStride"); + decoder->uniforms.dequant.preserveMissing = glGetUniformLocation( + decoder->dequantProgram, "preserveMissing"); decoder->uniforms.idwt.coefficients = glGetUniformLocation( decoder->idwtProgram, "coefficients"); @@ -544,6 +551,7 @@ static bool networkVideoCacheUniformsNL(NetworkVideoDecoder * decoder) uniforms->dequant.outputLayer < 0 || uniforms->dequant.blockOffset < 0 || uniforms->dequant.blockStride < 0 || + uniforms->dequant.preserveMissing < 0 || uniforms->idwt.coefficients < 0 || uniforms->idwt.resolution < 0 || uniforms->idwt.invResolution < 0 || @@ -861,7 +869,9 @@ void networkVideoDecoderDetach(NetworkVideoDecoder * decoder) decoder->maxWorkGroupCount[0] = 0; decoder->maxWorkGroupCount[1] = 0; decoder->maxStorageBlockSize = 0; + decoder->referenceFrameID = 0; decoder->attached = false; + decoder->referenceValid = false; LG_UNLOCK(decoder->lock); } @@ -891,6 +901,8 @@ void networkVideoDecoderReset(NetworkVideoDecoder * decoder) networkVideoRestoreCurrent(decoder, &previous); } } + decoder->referenceFrameID = 0; + decoder->referenceValid = false; LG_UNLOCK(decoder->lock); } @@ -1102,11 +1114,12 @@ static struct NetworkVideoUpload * networkVideoUploadNL( } static void networkVideoDequantNL(NetworkVideoDecoder * decoder, - const struct NetworkVideoUpload * upload) + const struct NetworkVideoUpload * upload, bool preserveMissing) { glUseProgram(decoder->dequantProgram); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, upload->offsetBuffer); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, upload->payloadBuffer); + glUniform1i(decoder->uniforms.dequant.preserveMissing, preserveMissing); for (unsigned level = 0; level < NETWORK_VIDEO_LEVELS; ++level) { @@ -1284,6 +1297,7 @@ static struct NetworkVideoExternalFrame * networkVideoConvertNL( NetworkVideoDecodeResult networkVideoDecoderDecode( NetworkVideoDecoder * decoder, const LGNetVideoStreamConfig * config, const void * data, size_t size, uint32_t blockCount, + bool update, bool complete, uint64_t baseFrameID, uint64_t frameID, NetworkVideoDecodedFrame * result) { if (!decoder || !config || !data || !size || !result) @@ -1298,11 +1312,19 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( return NETWORK_VIDEO_DECODE_UNSUPPORTED; } + if ((!update && baseFrameID) || (update && !complete) || + (update && (!decoder->referenceValid || + decoder->referenceFrameID != baseFrameID || frameID <= baseFrameID))) + { + LG_UNLOCK(decoder->lock); + return NETWORK_VIDEO_DECODE_CORRUPT; + } + NetworkPyroWaveLayout layout; const NetworkPyroWaveResult parse = networkPyroWaveParse( config, data, size, blockCount, decoder->offsets, decoder->blockCapacity, &layout); - if (parse != NETWORK_PYROWAVE_OK) + if (parse != NETWORK_PYROWAVE_OK || layout.delta != update) { LG_UNLOCK(decoder->lock); return parse == NETWORK_PYROWAVE_UNSUPPORTED ? @@ -1329,7 +1351,7 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( } const uint64_t decodeStart = nanotime(); - networkVideoDequantNL(decoder, upload); + networkVideoDequantNL(decoder, upload, update); upload->readySync = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0); networkVideoIDWTNL(decoder); struct NetworkVideoExternalFrame * external = @@ -1339,12 +1361,20 @@ NetworkVideoDecodeResult networkVideoDecoderDecode( networkVideoRestoreCurrent(decoder, &previous); if (!success) { + decoder->referenceFrameID = 0; + decoder->referenceValid = false; if (external) networkVideoExternalRelease(&external->base); LG_UNLOCK(decoder->lock); return NETWORK_VIDEO_DECODE_ERROR; } + if (!update) + { + decoder->referenceFrameID = complete ? frameID : 0; + decoder->referenceValid = complete; + } + uint32_t formatVersion = (uint32_t)config->configEpoch; if (!formatVersion) formatVersion = 1; diff --git a/client/transports/Network/video/pyrowave.c b/client/transports/Network/video/pyrowave.c index 3033ed4e..47254cbc 100644 --- a/client/transports/Network/video/pyrowave.c +++ b/client/transports/Network/video/pyrowave.c @@ -252,6 +252,7 @@ NetworkPyroWaveResult networkPyroWaveParse( uint8_t frameSequence = 0; bool haveSequence = false; bool haveHeader = false; + bool haveClear = false; bool valid = true; while (valid && size - cursor >= NETWORK_PYROWAVE_HEADER_SIZE) @@ -273,11 +274,15 @@ NetworkPyroWaveResult networkPyroWaveParse( if (extended) { - const uint32_t code = (second >> 24) & 3U; + const uint32_t code = + (second >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) & + LG_NET_VIDEO_PYROWAVE_MODE_MASK; const uint32_t width = (first & UINT32_C(0x00003FFF)) + 1U; const uint32_t height = ((first >> 14) & UINT32_C(0x00003FFF)) + 1U; const uint32_t blocks = second & UINT32_C(0x00FFFFFF); - if (code || width != config->width || height != config->height || + if ((code != LG_NET_VIDEO_PYROWAVE_MODE_FULL && + code != LG_NET_VIDEO_PYROWAVE_MODE_DELTA) || + width != config->width || height != config->height || !blocks || blocks > capacity || !networkPyroWaveSequenceProfile(config, second) || (haveHeader && @@ -299,26 +304,38 @@ NetworkPyroWaveResult networkPyroWaveParse( const uint32_t payloadWords = (first >> 16) & UINT32_C(0x00000FFF); const uint32_t blockIndex = second >> 8; const size_t packetSize = (size_t)payloadWords * sizeof(uint32_t); - if (!ballot || payloadWords < 2U || packetSize > size - cursor || + const bool clear = ballot == 0; + if (payloadWords < 2U || packetSize > size - cursor || blockIndex >= capacity || offsets[blockIndex] != UINT32_MAX || - !networkPyroWavePacketValid( - bytes + cursor, packetSize, (uint16_t)ballot)) + (clear ? payloadWords != LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_WORDS : + !networkPyroWavePacketValid( + bytes + cursor, packetSize, (uint16_t)ballot))) { valid = false; break; } offsets[blockIndex] = (uint32_t)(cursor / sizeof(uint32_t)); + if (clear) + haveClear = true; ++uniqueBlocks; cursor += packetSize; } valid = valid && cursor == size && haveHeader && haveSequence && - declaredBlocks == uniqueBlocks && expectedBlocks == uniqueBlocks; + declaredBlocks == uniqueBlocks && expectedBlocks == uniqueBlocks && + (!haveClear || + ((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) & + LG_NET_VIDEO_PYROWAVE_MODE_MASK) == + LG_NET_VIDEO_PYROWAVE_MODE_DELTA); if (!valid) return NETWORK_PYROWAVE_INVALID; layout->sequence = frameSequence; layout->chromaLeft = (headerSecond >> 31) != 0; + layout->delta = + ((headerSecond >> LG_NET_VIDEO_PYROWAVE_MODE_SHIFT) & + LG_NET_VIDEO_PYROWAVE_MODE_MASK) == + LG_NET_VIDEO_PYROWAVE_MODE_DELTA; return NETWORK_PYROWAVE_OK; } diff --git a/client/transports/Network/video/pyrowave.h b/client/transports/Network/video/pyrowave.h index ce02300e..e024a1bc 100644 --- a/client/transports/Network/video/pyrowave.h +++ b/client/transports/Network/video/pyrowave.h @@ -38,6 +38,7 @@ typedef struct NetworkPyroWaveLayout { uint8_t sequence; bool chromaLeft; + bool delta; } NetworkPyroWaveLayout; diff --git a/client/transports/Network/video/shaders.h b/client/transports/Network/video/shaders.h index 5512d86b..80ba9f2f 100644 --- a/client/transports/Network/video/shaders.h +++ b/client/transports/Network/video/shaders.h @@ -58,6 +58,7 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_0[] = "uniform int outputLayer;\n" "uniform int blockOffset;\n" "uniform int blockStride;\n" + "uniform bool preserveMissing;\n" "shared uint signOffset;\n" "shared uint planeOffset[16];\n" "shared uint significantPrefix[128];\n" @@ -166,10 +167,13 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_1[] = " }\n" " barrier();\n" " uint bitCursor = signOffset + significantPrefix[localIndex];\n" - " uint word = payload[bitCursor >> 5u];\n" - " uint upper = payload[(bitCursor >> 5u) + 1u];\n" - " uint shift = bitCursor & 31u;\n" - " if (shift != 0u) word = (word >> shift) | (upper << (32u-shift));\n" + " uint word = 0u;\n" + " if (significant != 0u) {\n" + " word = payload[bitCursor >> 5u];\n" + " uint upper = payload[(bitCursor >> 5u) + 1u];\n" + " uint shift = bitCursor & 31u;\n" + " if (shift != 0u) word = (word >> shift) | (upper << (32u-shift));\n" + " }\n" " uint signIndex = 0u;\n" " for (int x=0; x<4; ++x) for (int y=0; y<2; ++y) {\n" " if (values[y][x] != 0.0) {\n" @@ -177,7 +181,7 @@ static const char NETWORK_VIDEO_DEQUANT_SHADER_1[] = " ++signIndex;\n" " }\n" " }\n" - " storeValues(coord,values);\n" + " if (!preserveMissing || blockPresent != 0u) storeValues(coord,values);\n" "}\n"; static const char NETWORK_VIDEO_IDWT_SHADER_0[] = diff --git a/idd/LGIdd/transport/network/CNetworkVideo.cpp b/idd/LGIdd/transport/network/CNetworkVideo.cpp index 9c49ec7a..2cc78d8c 100644 --- a/idd/LGIdd/transport/network/CNetworkVideo.cpp +++ b/idd/LGIdd/transport/network/CNetworkVideo.cpp @@ -95,6 +95,13 @@ namespace constexpr uint32_t PYROWAVE_FRAGMENT_MIN = LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE + PYROWAVE_FRAGMENT_DATA_MIN; constexpr uint32_t PYROWAVE_PACKET_META_WORDS = 2; + constexpr uint32_t PYROWAVE_SEQUENCE_MASK = 7; + constexpr uint32_t PYROWAVE_HEADER_CODE_MASK = + LG_NET_VIDEO_PYROWAVE_MODE_MASK << + LG_NET_VIDEO_PYROWAVE_MODE_SHIFT; + constexpr uint32_t PYROWAVE_HEADER_CODE_DELTA = + LG_NET_VIDEO_PYROWAVE_MODE_DELTA << + LG_NET_VIDEO_PYROWAVE_MODE_SHIFT; constexpr uint32_t PYROWAVE_QUANT_WORDS_8X8 = 40; constexpr uint32_t PYROWAVE_STATE_BYTES = 16; constexpr uint32_t PYROWAVE_QUANT_LEVELS = 19; @@ -227,6 +234,22 @@ namespace return value ? value : 1; } + uint32_t LoadLE32(const uint8_t * data) + { + return static_cast(data[0]) | + static_cast(data[1]) << 8U | + static_cast(data[2]) << 16U | + static_cast(data[3]) << 24U; + } + + void StoreLE32(uint8_t * data, uint32_t value) + { + data[0] = static_cast(value); + data[1] = static_cast(value >> 8U); + data[2] = static_cast(value >> 16U); + data[3] = static_cast(value >> 24U); + } + LGNetEnvelopeFlags DeliveryFlag(NetworkVideoDelivery delivery) { switch (delivery) @@ -243,6 +266,12 @@ namespace return 0; } + bool FragmentMessage(uint16_t messageType) + { + return messageType == LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT || + messageType == LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT; + } + unsigned DeliveryIndex(NetworkVideoDelivery delivery) { switch (delivery) @@ -1384,6 +1413,7 @@ void FinalizeMain() { uint32_t offset = 0; uint32_t length = 0; + uint32_t index = 0; }; struct EncodedFrame @@ -1393,11 +1423,46 @@ void FinalizeMain() uint64_t frameID = 0; uint64_t captureTimeNs = 0; uint32_t deadlineMs = 0; + uint32_t blockCapacity = 0; std::vector data; std::vector blocks; std::shared_ptr> wholePayload; }; + struct EncodedPacket + { + const uint8_t * data = nullptr; + uint32_t length = 0; + }; + + EncodedPacket Packet(const EncodedFrame& frame, + const EncodedBlock& block) + { + const uint32_t prefix = block.offset ? 0U : 8U; + if (block.length <= prefix || block.offset > frame.data.size() || + block.length > frame.data.size() - block.offset) + return {}; + return + { + frame.data.data() + block.offset + prefix, + block.length - prefix, + }; + } + + bool SamePacket(const EncodedPacket& left, const EncodedPacket& right) + { + if (!left.data || !right.data || left.length != right.length || + left.length < 8U) + return left.data == right.data && left.length == right.length; + + constexpr uint32_t SEQUENCE_BITS = PYROWAVE_SEQUENCE_MASK << 28U; + if ((LoadLE32(left.data) & ~SEQUENCE_BITS) != + (LoadLE32(right.data) & ~SEQUENCE_BITS)) + return false; + return std::equal(left.data + sizeof(uint32_t), + left.data + left.length, right.data + sizeof(uint32_t)); + } + struct ShaderParameters { uint32_t dim0 [4] = {}; @@ -2204,14 +2269,6 @@ void FinalizeMain() slot.packed.Get(), 0, slot.frameLimit); } - static uint32_t LoadLE32(const uint8_t * data) - { - return static_cast(data[0]) | - static_cast(data[1]) << 8U | - static_cast(data[2]) << 16U | - static_cast(data[3]) << 24U; - } - bool Compact(const Slot& slot, const uint8_t * readback, EncodedFrame& output) const { @@ -2279,7 +2336,7 @@ void FinalizeMain() std::copy_n(packet, length, output.data.begin() + outputOffset); - output.blocks.push_back({ outputOffset, length }); + output.blocks.push_back({ outputOffset, length, blockIndex }); outputOffset += length; } if (output.blocks.size() != packetCount || @@ -2293,6 +2350,7 @@ void FinalizeMain() output.blocks[0].offset = 0; output.blocks[0].length += 8U; + output.blockCapacity = m_blockCapacity; return true; } @@ -2651,29 +2709,33 @@ struct CNetworkVideo::Impl { struct PeerState { - NetworkVideoPeer peer = {}; - NetworkVideoPeerInfo info = {}; - LGNetVideoSubscribe subscription = {}; - LGNetVideoFeedback feedback = {}; - LGNetVideoState lastStatusState = {}; - uint64_t serial = 0; - uint64_t componentEpoch = 0; - uint64_t statusSequence = 0; - uint64_t statusEpoch = 0; - uint64_t lastFrameID = 0; - uint64_t lastCompleteID = 0; - uint64_t lastCaptureTimeNs = 0; - uint64_t sendReadyGeneration[3] = {}; - uint32_t lastStatusDetail = 0; - uint32_t lastStatusFlags = 0; - uint32_t droppedFrames = 0; - uint8_t blockedDeliveries = 0; - bool statusSent = false; - bool subscribed = false; - bool configured = false; - bool congested = false; - bool keyframePending = false; - bool sendFailed = false; + NetworkVideoPeer peer = {}; + NetworkVideoPeerInfo info = {}; + LGNetVideoSubscribe subscription = {}; + LGNetVideoFeedback feedback = {}; + LGNetVideoState lastStatusState = {}; + uint64_t serial = 0; + uint64_t componentEpoch = 0; + uint64_t statusSequence = 0; + uint64_t statusEpoch = 0; + uint64_t lastFrameID = 0; + uint64_t lastCompleteID = 0; + uint64_t lastCaptureTimeNs = 0; + uint64_t referenceFrameID = 0; + uint64_t sendReadyGeneration[3] = {}; + uint32_t lastStatusDetail = 0; + uint32_t lastStatusFlags = 0; + uint32_t droppedFrames = 0; + uint8_t blockedDeliveries = 0; + std::shared_ptr referenceFrame; + std::vector updateBlocks; + bool statusSent = false; + bool subscribed = false; + bool configured = false; + bool congested = false; + bool keyframePending = false; + bool referencePending = false; + bool sendFailed = false; }; struct PendingFrame @@ -2696,6 +2758,7 @@ struct CNetworkVideo::Impl uint64_t componentEpoch = 0; uint64_t requestID = 0; uint64_t frameID = 0; + uint64_t baseFrameID = 0; std::shared_ptr> payload; std::shared_ptr encoded; uint32_t fragmentIndex = 0; @@ -2847,6 +2910,22 @@ struct CNetworkVideo::Impl return false; } + static void ResetPeerReferenceNL(PeerState& peer) + { + peer.referenceFrame.reset(); + peer.updateBlocks.clear(); + peer.referenceFrameID = 0; + peer.referencePending = false; + } + + static bool PeerNeedsReferenceNL( + const PeerState& peer, const EncodedFrame& frame) + { + return peer.keyframePending || !peer.referenceFrame || + peer.referenceFrame->configEpoch != frame.configEpoch || + peer.referenceFrame->blockCapacity != frame.blockCapacity; + } + uint32_t FrameLimitNL(uint64_t captureTimeNs) { uint64_t period = VIDEO_RATE_DEFAULT_PERIOD_NS; @@ -2973,6 +3052,131 @@ struct CNetworkVideo::Impl encoded.wholePayload->size() : 0, bytes); } + static std::shared_ptr BuildUpdateFrame( + const EncodedFrame& reference, const EncodedFrame& current, + const std::vector& previousBlocks, + std::vector& updateBlocks) + { + updateBlocks.clear(); + if (!reference.blockCapacity || + reference.blockCapacity != current.blockCapacity || + reference.configEpoch != current.configEpoch || + reference.data.size() < 8U || current.data.size() < 8U) + return {}; + + try + { + updateBlocks = previousBlocks; + updateBlocks.resize(current.blockCapacity, 0); + size_t referencePosition = 0; + size_t currentPosition = 0; + for (uint32_t index = 0; index < current.blockCapacity; ++index) + { + while (referencePosition < reference.blocks.size() && + reference.blocks[referencePosition].index < index) + ++referencePosition; + while (currentPosition < current.blocks.size() && + current.blocks[currentPosition].index < index) + ++currentPosition; + const EncodedBlock * referenceBlock = + referencePosition < reference.blocks.size() && + reference.blocks[referencePosition].index == index ? + &reference.blocks[referencePosition] : nullptr; + const EncodedBlock * currentBlock = + currentPosition < current.blocks.size() && + current.blocks[currentPosition].index == index ? + ¤t.blocks[currentPosition] : nullptr; + if (!SamePacket(referenceBlock ? Packet(reference, *referenceBlock) : + EncodedPacket {}, + currentBlock ? Packet(current, *currentBlock) : + EncodedPacket {})) + updateBlocks[index] = 1; + } + + uint32_t operationCount = 0; + for (uint8_t changed : updateBlocks) + operationCount += changed != 0; + if (!operationCount || operationCount > UINT32_C(0x00ffffff)) + return {}; + + std::shared_ptr update = + std::make_shared(); + update->configEpoch = current.configEpoch; + update->sourceSerial = current.sourceSerial; + update->frameID = current.frameID; + update->captureTimeNs = current.captureTimeNs; + update->deadlineMs = current.deadlineMs; + update->blockCapacity = current.blockCapacity; + update->blocks.reserve(operationCount); + update->data.reserve(current.data.size()); + update->data.resize(8U); + + const uint32_t firstWord = LoadLE32(current.data.data()); + const uint32_t secondWord = + (LoadLE32(current.data.data() + sizeof(uint32_t)) & + ~(PYROWAVE_HEADER_CODE_MASK | UINT32_C(0x00ffffff))) | + PYROWAVE_HEADER_CODE_DELTA | operationCount; + StoreLE32(update->data.data(), firstWord); + StoreLE32(update->data.data() + sizeof(uint32_t), secondWord); + + const uint32_t sequence = + (firstWord >> 28U) & PYROWAVE_SEQUENCE_MASK; + currentPosition = 0; + for (uint32_t index = 0; index < current.blockCapacity; ++index) + { + while (currentPosition < current.blocks.size() && + current.blocks[currentPosition].index < index) + ++currentPosition; + if (!updateBlocks[index]) + continue; + const EncodedBlock * block = + currentPosition < current.blocks.size() && + current.blocks[currentPosition].index == index ? + ¤t.blocks[currentPosition] : nullptr; + const uint32_t offset = + static_cast(update->data.size()); + if (block) + { + const EncodedPacket packet = Packet(current, *block); + if (!packet.data || packet.length < 8U || + packet.length > LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH - + update->data.size()) + return {}; + update->data.insert(update->data.end(), + packet.data, packet.data + packet.length); + update->blocks.push_back({ offset, packet.length, index }); + } + else + { + if (update->data.size() > LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH - + LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_SIZE) + return {}; + update->data.resize(update->data.size() + + LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_SIZE); + uint8_t * packet = update->data.data() + offset; + StoreLE32(packet, + (LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_WORDS << 16U) | + (sequence << 28U)); + StoreLE32(packet + sizeof(uint32_t), index << 8U); + update->blocks.push_back( + { offset, LG_NET_VIDEO_PYROWAVE_CLEAR_PACKET_SIZE, index }); + } + } + if (update->blocks.size() != operationCount || + update->data.size() > LG_NET_VIDEO_MAX_FRAME_UPDATE_LENGTH) + return {}; + + update->blocks[0].offset = 0; + update->blocks[0].length += 8U; + return update; + } + catch (...) + { + updateBlocks.clear(); + return {}; + } + } + bool EncodedQueuedNL(const EncodedFrame * encoded, uint64_t excludingSerial = 0) const { @@ -3058,7 +3262,9 @@ struct CNetworkVideo::Impl peer.componentEpoch = m_configEpoch; peer.configured = true; peer.lastCaptureTimeNs = 0; + peer.keyframePending = true; peer.sendFailed = false; + ResetPeerReferenceNL(peer); return QueueStatusNL(peer, LG_NET_VIDEO_STATE_CONFIGURED, 0); } @@ -3111,8 +3317,11 @@ struct CNetworkVideo::Impl m_frames.clear(); m_latestFrame.reset(); for (PeerState& peer : m_peers) + { + ResetPeerReferenceNL(peer); if (peer.subscribed) QueueStatusNL(peer, LG_NET_VIDEO_STATE_ERROR, detail); + } } void DropPeerFramesNL(PeerState& peer) @@ -3123,7 +3332,7 @@ struct CNetworkVideo::Impl uint64_t activeFrame = 0; for (const SendItem& item : m_sends) if (item.frameID && item.started && - item.messageType == LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT && + FragmentMessage(item.messageType) && SamePeer(item.peer, peer.peer)) { activeFrame = item.frameID; @@ -3216,6 +3425,47 @@ struct CNetworkVideo::Impl return payload; } + static std::shared_ptr> BuildUpdatePayload( + const EncodedFrame& encoded, uint64_t baseFrameID, + uint32_t deadlineMs, uint64_t frameID, + uint64_t presentationBaseNs) + { + LGNetVideoFrameUpdate update = {}; + update.streamID = STREAM_ID; + update.codec = LG_NET_VIDEO_CODEC_PYROWAVE; + update.configEpoch = encoded.configEpoch; + update.frameID = frameID; + update.baseFrameID = baseFrameID; + update.captureTimestampNs = encoded.captureTimeNs; + update.presentationTimestampNs = presentationBaseNs + + static_cast(deadlineMs) * UINT64_C(1000000); + update.encodedLength = + static_cast(encoded.data.size()); + update.blockCount = + static_cast(encoded.blocks.size()); + update.fragmentCount = 1; + update.deadlineMs = deadlineMs; + update.data = encoded.data.data(); + + const size_t size = lgNetVideoFrameUpdateSize(&update); + if (!size || size > UINT32_MAX) + return {}; + + std::shared_ptr> payload; + try + { + payload = std::make_shared>(size); + } + catch (...) + { + return {}; + } + if (!lgNetVideoFrameUpdateEncode( + payload->data(), payload->size(), &update)) + return {}; + return payload; + } + bool QueueWholeFrameNL(PeerState& peer, const EncodedFrame& encoded, NetworkVideoDelivery delivery, uint64_t frameID, uint64_t presentationBaseNs) @@ -3236,16 +3486,33 @@ struct CNetworkVideo::Impl std::move(payload)); } + bool QueueWholeUpdateNL(PeerState& peer, const EncodedFrame& encoded, + NetworkVideoDelivery delivery, uint64_t baseFrameID, uint64_t frameID, + uint64_t presentationBaseNs) + { + const uint32_t deadlineMs = (std::min)( + encoded.deadlineMs, peer.subscription.maxFrameLatencyMs); + std::shared_ptr> payload = + BuildUpdatePayload(encoded, baseFrameID, + deadlineMs, frameID, presentationBaseNs); + if (!payload) + return false; + return QueueNL(peer.peer, delivery, + LG_NET_VIDEO_MESSAGE_FRAME_UPDATE, LG_NET_ENVELOPE_FINAL, + encoded.configEpoch, 0, frameID, std::move(payload)); + } + static bool FragmentPlan(const EncodedFrame& encoded, uint32_t maxPayload, uint32_t firstBlockIndex, uint32_t firstFragmentIndex, - uint32_t& payloadLimit, uint32_t& fragmentCount) + uint32_t& payloadLimit, uint32_t& fragmentCount, + uint32_t headerSize = LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE) { - if (maxPayload <= LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE || + if (maxPayload <= headerSize || firstBlockIndex > encoded.blocks.size()) return false; payloadLimit = (std::min)( - maxPayload - LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE, + maxPayload - headerSize, static_cast(LG_NET_VIDEO_MAX_FRAGMENT_LENGTH)); if (!payloadLimit) return false; @@ -3292,16 +3559,22 @@ struct CNetworkVideo::Impl bool QueueFragmentsNL(PeerState& peer, std::shared_ptr encoded, NetworkVideoDelivery delivery, uint64_t frameID, uint32_t maxPayload, - uint32_t firstBlockIndex = 0, - uint32_t firstFragmentIndex = 0) + uint32_t firstBlockIndex = 0, + uint32_t firstFragmentIndex = 0, + uint16_t messageType = LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT, + uint64_t baseFrameID = 0) { - if (!encoded) + const bool update = + messageType == LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT; + if (!encoded || (update != (baseFrameID != 0))) return false; uint32_t payloadLimit; uint32_t fragmentCount; if (!FragmentPlan(*encoded, maxPayload, firstBlockIndex, firstFragmentIndex, - payloadLimit, fragmentCount)) + payloadLimit, fragmentCount, + update ? LG_NET_VIDEO_FRAME_UPDATE_FRAGMENT_HEADER_WIRE_SIZE : + LG_NET_VIDEO_FRAGMENT_HEADER_WIRE_SIZE)) return false; if (firstBlockIndex == encoded->blocks.size()) return true; @@ -3318,10 +3591,11 @@ struct CNetworkVideo::Impl item.serial = Next64(m_nextSendSerial); item.peer = peer.peer; item.delivery = delivery; - item.messageType = LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT; + item.messageType = messageType; item.flags = LG_NET_ENVELOPE_FINAL; item.componentEpoch = encoded->configEpoch; item.frameID = frameID; + item.baseFrameID = baseFrameID; item.encoded = std::move(encoded); item.fragmentIndex = firstFragmentIndex; item.fragmentCount = fragmentCount; @@ -3406,6 +3680,51 @@ struct CNetworkVideo::Impl const uint32_t length = (std::min)( item.payloadLimit, block.length - item.blockOffset); + if (item.messageType == LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT) + { + LGNetVideoFrameUpdateFragment fragment = {}; + fragment.streamID = STREAM_ID; + fragment.configEpoch = item.encoded->configEpoch; + fragment.frameID = item.frameID; + fragment.baseFrameID = item.baseFrameID; + fragment.frameOffset = block.offset + item.blockOffset; + fragment.frameLength = + static_cast(item.encoded->data.size()); + fragment.blockIndex = item.blockIndex; + fragment.blockOffset = item.blockOffset; + fragment.blockLength = block.length; + fragment.blockCount = + static_cast(item.encoded->blocks.size()); + fragment.fragmentIndex = item.fragmentIndex; + fragment.fragmentCount = item.fragmentCount; + fragment.payloadLength = length; + fragment.payload = + item.encoded->data.data() + fragment.frameOffset; + if (!item.blockOffset) + fragment.flags |= LG_NET_VIDEO_FRAGMENT_BLOCK_START; + if (item.blockOffset + length == block.length) + fragment.flags |= LG_NET_VIDEO_FRAGMENT_BLOCK_END; + if (static_cast(fragment.frameOffset) + length == + item.encoded->data.size()) + fragment.flags |= LG_NET_VIDEO_FRAGMENT_FRAME_END; + + const size_t size = lgNetVideoFrameUpdateFragmentSize(&fragment); + if (!size) + return false; + try + { + payload.resize(size); + } + catch (...) + { + return false; + } + return lgNetVideoFrameUpdateFragmentEncode( + payload.data(), payload.size(), &fragment); + } + if (item.messageType != LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT) + return false; + LGNetVideoFragment fragment = {}; fragment.streamID = STREAM_ID; fragment.configEpoch = item.encoded->configEpoch; @@ -3465,7 +3784,8 @@ struct CNetworkVideo::Impl bool QueueFrameNL(PeerState& peer, const std::shared_ptr& frame, - uint64_t frameID, uint64_t presentationBaseNs) + uint64_t frameID, uint64_t presentationBaseNs, + bool establishReference = false) { if (!frame || !frameID || !presentationBaseNs || !peer.configured || peer.componentEpoch != frame->configEpoch || @@ -3473,8 +3793,19 @@ struct CNetworkVideo::Impl peer.subscription.maxFrameLength) return false; + std::vector cleanBlocks; + if (establishReference) + try + { + cleanBlocks.assign(frame->blockCapacity, 0); + } + catch (...) + { + return false; + } + DropPeerFramesNL(peer); - const bool datagrams = + const bool datagrams = !establishReference && (peer.subscription.flags & LG_NET_VIDEO_SUBSCRIBE_ALLOW_DATAGRAMS) && DatagramFragmentsSupported(peer.info); const bool bulk = BulkFragmentsSupported(peer.info); @@ -3535,12 +3866,108 @@ struct CNetworkVideo::Impl } peer.lastFrameID = frameID; peer.lastCaptureTimeNs = frame->captureTimeNs; - peer.keyframePending = false; + if (establishReference) + { + peer.referenceFrame = frame; + peer.referenceFrameID = frameID; + peer.referencePending = true; + peer.updateBlocks = std::move(cleanBlocks); + } + peer.keyframePending = establishReference; + return true; + } + + bool QueueUpdateNL(PeerState& peer, + const std::shared_ptr& update, + uint64_t baseFrameID, uint64_t frameID, + uint64_t presentationBaseNs) + { + if (!update || !baseFrameID || baseFrameID >= frameID || + !presentationBaseNs || !peer.configured || + peer.referencePending || peer.referenceFrameID != baseFrameID || + peer.componentEpoch != update->configEpoch || + update->data.empty() || update->data.size() > + peer.subscription.maxFrameLength) + return false; + + DropPeerFramesNL(peer); + const bool bulk = BulkFragmentsSupported(peer.info); + NetworkVideoDelivery delivery = bulk ? + NetworkVideoDelivery::BULK : NetworkVideoDelivery::RELIABLE; + const uint32_t fullSize = + static_cast(update->data.size()) + + LG_NET_VIDEO_FRAME_UPDATE_HEADER_WIRE_SIZE; + const uint32_t maxPayload = DeliveryPayload(peer.info, delivery); + bool result; + if (fullSize <= maxPayload) + result = QueueWholeUpdateNL(peer, *update, delivery, + baseFrameID, frameID, presentationBaseNs); + else + result = QueueScheduleNL(peer, *update, frameID, + presentationBaseNs) && + QueueFragmentsNL(peer, update, delivery, frameID, maxPayload, + 0, 0, LG_NET_VIDEO_MESSAGE_FRAME_UPDATE_FRAGMENT, baseFrameID); + + if (!result) + { + DropPeerFramesNL(peer); + ++peer.droppedFrames; + peer.congested = true; + return false; + } + peer.lastFrameID = frameID; + peer.lastCaptureTimeNs = update->captureTimeNs; + return true; + } + + bool QueueCurrentFrameNL(PeerState& peer, + const std::shared_ptr& current, + uint64_t frameID, uint64_t presentationBaseNs) + { + if (!current) + return false; + if (peer.referencePending) + return true; + if (PeerNeedsReferenceNL(peer, *current)) + return QueueFrameNL( + peer, current, frameID, presentationBaseNs, true); + std::vector updateBlocks; + const std::shared_ptr update = BuildUpdateFrame( + *peer.referenceFrame, *current, peer.updateBlocks, + updateBlocks); + const bool compared = updateBlocks.size() == current->blockCapacity; + const bool changed = std::any_of(updateBlocks.begin(), + updateBlocks.end(), [](uint8_t value) { return value != 0; }); + if (!update) + { + if (compared && !changed) + { + peer.lastCaptureTimeNs = current->captureTimeNs; + return true; + } + return QueueFrameNL( + peer, current, frameID, presentationBaseNs, true); + } + + const uint64_t fullWireSize = current->data.size() + + LG_NET_VIDEO_FRAME_HEADER_WIRE_SIZE; + const uint64_t updateWireSize = update->data.size() + + LG_NET_VIDEO_FRAME_UPDATE_HEADER_WIRE_SIZE; + if (updateWireSize >= fullWireSize) + return QueueFrameNL( + peer, current, frameID, presentationBaseNs, true); + + if (!QueueUpdateNL(peer, update, peer.referenceFrameID, + frameID, presentationBaseNs)) + return false; + peer.updateBlocks = std::move(updateBlocks); return true; } bool ReplayLatestFrameNL(PeerState& peer) { + if (peer.referencePending) + return true; if (!m_latestFrame || !m_latestSourceSerial || m_latestFrame->configEpoch != m_configEpoch || m_latestFrame->sourceSerial != m_latestSourceSerial || @@ -3548,7 +3975,7 @@ struct CNetworkVideo::Impl return false; const uint64_t frameID = Next64(m_nextFrameID); - if (!QueueFrameNL(peer, m_latestFrame, frameID, + if (!QueueCurrentFrameNL(peer, m_latestFrame, frameID, CFrameScheduler::Nanotime())) return false; @@ -3562,7 +3989,10 @@ struct CNetworkVideo::Impl bool queued = false; CSRWExclusiveLock lock(m_lock); for (PeerState& peer : m_peers) - if (peer.subscribed && peer.configured && peer.keyframePending) + if (peer.subscribed && peer.configured && + (peer.keyframePending || + (!peer.referencePending && peer.referenceFrame && m_latestFrame && + m_latestFrame->captureTimeNs > peer.lastCaptureTimeNs))) queued |= ReplayLatestFrameNL(peer); return queued; } @@ -3604,6 +4034,8 @@ struct CNetworkVideo::Impl return; for (const PeerState& peer : m_peers) if (peer.subscribed && peer.configured && + !peer.referencePending && + PeerNeedsReferenceNL(peer, frame) && PeerFrameDueNL(peer, frame) && PeerUsesWholeFrameNL(peer, frame)) { @@ -3647,7 +4079,7 @@ struct CNetworkVideo::Impl if (peer.subscribed && peer.configured && PeerFrameDueNL(peer, *encoded)) { - if (!QueueFrameNL(peer, encoded, + if (!QueueCurrentFrameNL(peer, encoded, frameID, encoded->captureTimeNs)) QueueStatusNL(peer, LG_NET_VIDEO_STATE_STALLED, 2); else @@ -3796,11 +4228,10 @@ struct CNetworkVideo::Impl } return; } - if (item.messageType == LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT && - item.frameID) + if (FragmentMessage(item.messageType) && item.frameID) for (SendItem& queued : m_sends) if (queued.frameID == item.frameID && - queued.messageType == LG_NET_VIDEO_MESSAGE_FRAME_FRAGMENT && + FragmentMessage(queued.messageType) && SamePeer(queued.peer, item.peer)) queued.started = true; madeProgress = true; @@ -3962,6 +4393,7 @@ void CNetworkVideo::Stop() { peer.configured = false; peer.componentEpoch = 0; + m_impl->ResetPeerReferenceNL(peer); } } m_impl->m_encoder.Reset(); @@ -4083,6 +4515,7 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket( const bool hadSubscribers = m_impl->HasSubscribersNL(); m_impl->DropPeerSendsNL(*peer); + m_impl->ResetPeerReferenceNL(*peer); peer->subscription = subscribe; peer->subscribed = true; peer->configured = false; @@ -4128,6 +4561,7 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket( peer->configured = false; peer->componentEpoch = 0; peer->keyframePending = false; + m_impl->ResetPeerReferenceNL(*peer); if (!m_impl->HasSubscribersNL()) m_impl->m_lastBudgetCaptureTimeNs = 0; break; @@ -4151,6 +4585,8 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket( return NetworkVideoPacketResult::IGNORED; if (!peer->subscribed) return NetworkVideoPacketResult::INVALID; + m_impl->DropPeerSendsNL(*peer); + m_impl->ResetPeerReferenceNL(*peer); peer->keyframePending = true; m_impl->ReplayLatestFrameNL(*peer); break; @@ -4173,16 +4609,29 @@ NetworkVideoPacketResult CNetworkVideo::HandlePacket( return NetworkVideoPacketResult::IGNORED; if (!peer->subscribed) return NetworkVideoPacketResult::INVALID; + bool referenceAccepted = false; peer->feedback = feedback; peer->lastCompleteID = feedback.lastCompleteFrameID; peer->congested = feedback.flags & LG_NET_VIDEO_FEEDBACK_CONGESTED; + if (peer->referencePending && peer->referenceFrameID && + feedback.lastCompleteFrameID >= peer->referenceFrameID && + feedback.lastDecodedFrameID >= peer->referenceFrameID) + { + peer->referencePending = false; + peer->keyframePending = false; + referenceAccepted = true; + } if (feedback.flags & (LG_NET_VIDEO_FEEDBACK_REQUEST_KEYFRAME | LG_NET_VIDEO_FEEDBACK_DECODER_STALLED)) { + m_impl->DropPeerSendsNL(*peer); + m_impl->ResetPeerReferenceNL(*peer); peer->keyframePending = true; m_impl->ReplayLatestFrameNL(*peer); } + else if (referenceAccepted) + m_impl->QueueStatusNL(*peer, LG_NET_VIDEO_STATE_STREAMING, 0); break; } diff --git a/repos/LGProtocol b/repos/LGProtocol index a2a41197..a8f5a214 160000 --- a/repos/LGProtocol +++ b/repos/LGProtocol @@ -1 +1 @@ -Subproject commit a2a41197d00129c21fd6e7d40968837854f39103 +Subproject commit a8f5a21480472777d4f61ed55202fde03b2f0fc9