diff --git a/client/include/interface/transport.h b/client/include/interface/transport.h index ced5b1d8..db71bc29 100644 --- a/client/include/interface/transport.h +++ b/client/include/interface/transport.h @@ -135,7 +135,9 @@ typedef struct LG_TransportFrame uint64_t serial; uint64_t timestamp; uint32_t scheduleGeneration; + uint32_t scheduleEpoch; LG_TransportFrameFlags flags; + bool scheduleOwner; // Backend-owned immutable metadata, valid until releaseFrame. const LG_TransportFrameFormat * format; @@ -207,6 +209,7 @@ typedef struct LG_TransportControl uint64_t targetSlack; int64_t phaseError; uint32_t feedbackFrameSerial; + uint32_t feedbackScheduleEpoch; uint32_t lease; } frameSchedule; diff --git a/client/renderers/EGL/texture_buffer.c b/client/renderers/EGL/texture_buffer.c index 0033720a..790ca642 100644 --- a/client/renderers/EGL/texture_buffer.c +++ b/client/renderers/EGL/texture_buffer.c @@ -171,10 +171,13 @@ bool egl_texBufferStreamInit(EGL_Texture ** texture, EGL_TexType type, { case EGL_TEXTYPE_BUFFER_STREAM: case EGL_TEXTYPE_FRAMEBUFFER: - case EGL_TEXTYPE_DMABUF: this->texCount = 2; break; + case EGL_TEXTYPE_DMABUF: + this->texCount = LGMP_Q_FRAME_BUFFER_LEN; + break; + case EGL_TEXTYPE_BUFFER_MAP: this->texCount = 1; break; diff --git a/client/renderers/EGL/texture_buffer.h b/client/renderers/EGL/texture_buffer.h index ff41947d..29807ba0 100644 --- a/client/renderers/EGL/texture_buffer.h +++ b/client/renderers/EGL/texture_buffer.h @@ -22,9 +22,10 @@ #include "texture.h" #include "texture_util.h" +#include "common/LGMPConfig.h" #include "common/locking.h" -#define EGL_TEX_BUFFER_MAX 2 +#define EGL_TEX_BUFFER_MAX LGMP_Q_FRAME_BUFFER_LEN typedef struct TextureBuffer { diff --git a/client/renderers/EGL/texture_dmabuf.c b/client/renderers/EGL/texture_dmabuf.c index 0140383e..e380b668 100644 --- a/client/renderers/EGL/texture_dmabuf.c +++ b/client/renderers/EGL/texture_dmabuf.c @@ -42,7 +42,7 @@ typedef struct TexDMABUF EGLDisplay display; - struct FdImage images[2]; + struct FdImage images[EGL_TEX_BUFFER_MAX]; int lastIndex; int renderIndex; @@ -220,11 +220,28 @@ static bool egl_texDMABUFUpdate(EGL_Texture * texture, DEBUG_ASSERT(update->type == EGL_TEXTYPE_DMABUF); - struct FdImage *fdImage = - (this->images[0].fd == update->dmaFD) ? &this->images[0] : - (this->images[1].fd == update->dmaFD) ? &this->images[1] : - (this->images[0].fd == -1) ? &this->images[0] : - &this->images[1]; + struct FdImage *fdImage = NULL; + for (int i = 0; i < ARRAY_LENGTH(this->images); ++i) + if (this->images[i].fd == update->dmaFD) + { + fdImage = &this->images[i]; + break; + } + + if (!fdImage) + for (int i = 0; i < ARRAY_LENGTH(this->images); ++i) + if (this->images[i].fd == -1) + { + fdImage = &this->images[i]; + break; + } + + if (!fdImage) + { + DEBUG_ERROR("No free DMABUF image slot"); + return false; + } + EGLImage image = fdImage->image; if (unlikely(image == EGL_NO_IMAGE)) { @@ -255,7 +272,7 @@ static bool egl_texDMABUFUpdate(EGL_Texture * texture, fdImage->fd = update->dmaFD; fdImage->image = image; - int slot = (fdImage == &this->images[0]) ? 0 : 1; + const int slot = (int)(fdImage - this->images); fdImage->texIndex = slot; GLsync sync = 0; INTERLOCKED_SECTION(parent->copyLock, @@ -287,7 +304,7 @@ static bool egl_texDMABUFUpdate(EGL_Texture * texture, INTERLOCKED_SECTION(parent->copyLock, { fdImage->frameToken = update->frameToken; - this->lastIndex = (fdImage == &this->images[0]) ? 0 : 1; + this->lastIndex = (int)(fdImage - this->images); }); return true; diff --git a/client/src/frame_scheduler.c b/client/src/frame_scheduler.c index 2e1c3748..533cc9fa 100644 --- a/client/src/frame_scheduler.c +++ b/client/src/frame_scheduler.c @@ -43,6 +43,7 @@ static struct _Atomic(bool) supported; _Atomic(bool) active; bool controlPending; + bool immediatePending; _Atomic(uint32_t) generation; _Atomic(uint64_t) period; uint64_t lastSend; @@ -50,6 +51,7 @@ static struct int64_t phaseError; uint32_t feedbackFrameSerial; + uint32_t feedbackScheduleEpoch; unsigned feedbackSamples; bool feedbackDirty; @@ -89,21 +91,24 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags, int64_t phaseError; uint32_t feedbackFrameSerial; + uint32_t feedbackScheduleEpoch; LG_LOCK(l_frameScheduler.lock); - phaseError = l_frameScheduler.phaseError; - feedbackFrameSerial = l_frameScheduler.feedbackFrameSerial; + phaseError = l_frameScheduler.phaseError; + feedbackFrameSerial = l_frameScheduler.feedbackFrameSerial; + feedbackScheduleEpoch = l_frameScheduler.feedbackScheduleEpoch; LG_UNLOCK(l_frameScheduler.lock); const LG_TransportControl control = { .type = LG_TRANSPORT_CONTROL_FRAME_SCHEDULE, .frameSchedule = { - .generation = l_frameScheduler.generation, - .flags = flags, - .period = period, - .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, - .phaseError = phaseError, - .feedbackFrameSerial = feedbackFrameSerial, - .lease = FRAME_SCHEDULER_LEASE_MS, + .generation = l_frameScheduler.generation, + .flags = flags, + .period = period, + .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, + .phaseError = phaseError, + .feedbackFrameSerial = feedbackFrameSerial, + .feedbackScheduleEpoch = feedbackScheduleEpoch, + .lease = FRAME_SCHEDULER_LEASE_MS, }, }; @@ -118,8 +123,11 @@ static bool sendSchedule(LG_TransportFrameScheduleFlags flags, } l_frameScheduler.controlPending = true; + if (flags & LG_TRANSPORT_FRAME_SCHEDULE_IMMEDIATE) + l_frameScheduler.immediatePending = false; LG_LOCK(l_frameScheduler.lock); - if (l_frameScheduler.feedbackFrameSerial == feedbackFrameSerial) + if (l_frameScheduler.feedbackFrameSerial == feedbackFrameSerial && + l_frameScheduler.feedbackScheduleEpoch == feedbackScheduleEpoch) l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); return true; @@ -140,17 +148,19 @@ void frameScheduler_start(LG_TransportFeatureFlags features) { l_frameScheduler.supported = features & LG_TRANSPORT_FEATURE_FRAME_SCHEDULE; - l_frameScheduler.active = false; - l_frameScheduler.controlPending = false; - l_frameScheduler.lastSend = 0; - l_frameScheduler.lastCadence = 0; + l_frameScheduler.active = false; + l_frameScheduler.controlPending = false; + l_frameScheduler.immediatePending = true; + l_frameScheduler.lastSend = 0; + l_frameScheduler.lastCadence = 0; ++l_frameScheduler.generation; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } @@ -159,9 +169,10 @@ void frameScheduler_stop(void) if (l_frameScheduler.supported && l_frameScheduler.active) sendSchedule(LG_TRANSPORT_FRAME_SCHEDULE_RELEASE, 0); - l_frameScheduler.supported = false; - l_frameScheduler.active = false; - l_frameScheduler.controlPending = false; + l_frameScheduler.supported = false; + l_frameScheduler.active = false; + l_frameScheduler.controlPending = false; + l_frameScheduler.immediatePending = false; } void frameScheduler_update(void) @@ -179,13 +190,15 @@ void frameScheduler_update(void) FRAME_SCHEDULER_CADENCE_GRACE_NS && sendSchedule(LG_TRANSPORT_FRAME_SCHEDULE_RELEASE, 0)) { - l_frameScheduler.active = false; - l_frameScheduler.period = 0; + l_frameScheduler.active = false; + l_frameScheduler.period = 0; + l_frameScheduler.immediatePending = true; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } return; @@ -204,18 +217,21 @@ void frameScheduler_update(void) { l_frameScheduler.period = period; ++l_frameScheduler.generation; + l_frameScheduler.immediatePending = true; LG_LOCK(l_frameScheduler.lock); - l_frameScheduler.phaseError = 0; - l_frameScheduler.feedbackFrameSerial = 0; - l_frameScheduler.feedbackSamples = 0; - l_frameScheduler.feedbackDirty = false; + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackFrameSerial = 0; + l_frameScheduler.feedbackScheduleEpoch = 0; + l_frameScheduler.feedbackSamples = 0; + l_frameScheduler.feedbackDirty = false; LG_UNLOCK(l_frameScheduler.lock); } else l_frameScheduler.period = (l_frameScheduler.period * 7 + period) / 8; - if (l_frameScheduler.active && !reset) + if (l_frameScheduler.active && !reset && + !l_frameScheduler.immediatePending) { LG_LOCK(l_frameScheduler.lock); const bool feedbackDirty = l_frameScheduler.feedbackDirty; @@ -230,6 +246,8 @@ void frameScheduler_update(void) LG_TRANSPORT_FRAME_SCHEDULE_ACTIVE; if (reset) flags |= LG_TRANSPORT_FRAME_SCHEDULE_RESET; + if (l_frameScheduler.immediatePending) + flags |= LG_TRANSPORT_FRAME_SCHEDULE_IMMEDIATE; if (sendSchedule(flags, l_frameScheduler.period)) { @@ -239,9 +257,9 @@ void frameScheduler_update(void) } void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, - uint64_t measuredPhase) + uint32_t scheduleEpoch, uint64_t measuredPhase) { - if (!generation) + if (!generation || !scheduleEpoch) return; LG_LOCK(l_frameScheduler.lock); @@ -259,6 +277,13 @@ void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, return; } + if (l_frameScheduler.feedbackScheduleEpoch && + l_frameScheduler.feedbackScheduleEpoch != scheduleEpoch) + { + l_frameScheduler.phaseError = 0; + l_frameScheduler.feedbackSamples = 0; + } + int64_t error = measuredPhase > FRAME_SCHEDULER_TARGET_SLACK_NS ? (int64_t)(measuredPhase - FRAME_SCHEDULER_TARGET_SLACK_NS) : -(int64_t)(FRAME_SCHEDULER_TARGET_SLACK_NS - measuredPhase); @@ -276,7 +301,8 @@ void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, if (l_frameScheduler.feedbackSamples < 32) ++l_frameScheduler.feedbackSamples; - l_frameScheduler.feedbackFrameSerial = (uint32_t)frameSerial; - l_frameScheduler.feedbackDirty = true; + l_frameScheduler.feedbackFrameSerial = (uint32_t)frameSerial; + l_frameScheduler.feedbackScheduleEpoch = scheduleEpoch; + l_frameScheduler.feedbackDirty = true; LG_UNLOCK(l_frameScheduler.lock); } diff --git a/client/src/frame_scheduler.h b/client/src/frame_scheduler.h index ae7a3981..168a2821 100644 --- a/client/src/frame_scheduler.h +++ b/client/src/frame_scheduler.h @@ -31,6 +31,6 @@ void frameScheduler_start(LG_TransportFeatureFlags features); void frameScheduler_stop(void); void frameScheduler_update(void); void frameScheduler_feedback(uint64_t frameSerial, uint32_t generation, - uint64_t measuredPhase); + uint32_t scheduleEpoch, uint64_t measuredPhase); #endif diff --git a/client/src/main.c b/client/src/main.c index e7572a22..bee74a32 100644 --- a/client/src/main.c +++ b/client/src/main.c @@ -214,8 +214,10 @@ struct FrameTimingRecord LG_RendererFrameToken token; uint64_t frameSerial; uint32_t scheduleGeneration; + uint32_t scheduleEpoch; unsigned readyMask; bool producerValid; + bool scheduleOwner; uint64_t captureTime; uint64_t postProcessTime; @@ -326,8 +328,9 @@ static void frameTimingCancel(LG_RendererFrameToken token) } static void frameTimingQueue(LG_RendererFrameToken token, uint64_t frameSerial, - uint32_t scheduleGeneration, uint64_t importTime, - uint64_t importWaitTime, uint64_t dispatchStart, uint64_t queueStart) + uint32_t scheduleGeneration, uint32_t scheduleEpoch, bool scheduleOwner, + uint64_t importTime, uint64_t importWaitTime, uint64_t dispatchStart, + uint64_t queueStart) { INTERLOCKED_SECTION(l_frameTiming.lock, { struct FrameTimingRecord * record = frameTimingRecord(token); @@ -342,6 +345,8 @@ static void frameTimingQueue(LG_RendererFrameToken token, uint64_t frameSerial, record->queueStart = queueStart; record->frameSerial = frameSerial; record->scheduleGeneration = scheduleGeneration; + record->scheduleEpoch = scheduleEpoch; + record->scheduleOwner = scheduleOwner; if (record->timestamp < queueStart) record->timestamp = queueStart; } @@ -390,6 +395,8 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, uint64_t feedbackFrameSerial = 0; uint64_t feedbackQueueStart = 0; uint32_t feedbackGeneration = 0; + uint32_t feedbackEpoch = 0; + bool feedbackOwner = false; LG_LOCK(l_frameTiming.lock); if (l_frameTiming.retireToken <= timing->frameToken) @@ -411,6 +418,8 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, feedbackFrameSerial = record->frameSerial; feedbackGeneration = record->scheduleGeneration; + feedbackEpoch = record->scheduleEpoch; + feedbackOwner = record->scheduleOwner; feedbackQueueStart = record->queueStart; if (unlikely( @@ -434,11 +443,12 @@ static void frameTimingFinishRender(const LG_RendererFrameTiming * timing, } LG_UNLOCK(l_frameTiming.lock); - if (g_state.jitRender && feedbackFrameSerial && feedbackGeneration && - feedbackQueueStart && prepareStart >= feedbackQueueStart) + if (g_state.jitRender && feedbackOwner && feedbackFrameSerial && + feedbackGeneration && feedbackEpoch && feedbackQueueStart && + prepareStart >= feedbackQueueStart) { frameScheduler_feedback( - feedbackFrameSerial, feedbackGeneration, + feedbackFrameSerial, feedbackGeneration, feedbackEpoch, prepareStart - feedbackQueueStart); } } @@ -936,7 +946,8 @@ int main_frameThread(void * unused) break; } - if (frame.serial == frameSerial && g_state.formatValid) + if (frame.serial == frameSerial && g_state.formatValid && + !frame.scheduleOwner) { g_state.transportOps->releaseFrame(g_state.transport, &frame); continue; @@ -1113,8 +1124,8 @@ int main_frameThread(void * unused) memory_order_release); #endif frameTimingQueue(frameToken, frame.serial, frame.scheduleGeneration, - g_state.frameImportTime, g_state.frameImportWaitTime, - dispatchStart, queueStart); + frame.scheduleEpoch, frame.scheduleOwner, g_state.frameImportTime, + g_state.frameImportWaitTime, dispatchStart, queueStart); if (g_state.jitRender) { diff --git a/client/transports/LGMP/lgmp.c b/client/transports/LGMP/lgmp.c index e0de96f7..6914e0fc 100644 --- a/client/transports/LGMP/lgmp.c +++ b/client/transports/LGMP/lgmp.c @@ -47,24 +47,28 @@ struct DMAFrameInfo struct LG_Transport { - struct IVSHMEM shm; - PLGMPClient client; - PLGMPClientQueue frameQueue; - PLGMPClientQueue pointerQueue; - LG_Lock pointerLock; + struct IVSHMEM shm; + PLGMPClient client; + PLGMPClientQueue frameQueue; + PLGMPClientQueue ownerFrameQueue[LGMP_Q_FRAME_LEN]; + PLGMPClientQueue pendingFrameQueue; + PLGMPClientQueue pointerQueue; + LG_Lock pointerLock; unsigned cursorPollInterval; unsigned framePollInterval; bool allowDMA; bool connected; bool framePending; + bool frameScheduleSupported; const KVMFRFrame * pendingFrame; uint32_t clientID; uint32_t frameSerial; + bool frameSerialValid; bool formatValid; LG_TransportFrameFormat format; - struct DMAFrameInfo dma[LGMP_Q_FRAME_LEN]; + struct DMAFrameInfo dma[LGMP_Q_FRAME_BUFFER_LEN]; uint8_t * pointerData; size_t pointerDataSize; }; @@ -180,7 +184,7 @@ static bool lgmp_create(LG_Transport ** result) this->framePollInterval = framePoll; this->cursorPollInterval = cursorPoll; - for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + for (unsigned i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) this->dma[i].fd = -1; LG_LOCK_INIT(this->pointerLock); @@ -210,25 +214,39 @@ static bool lgmp_create(LG_Transport ** result) static void lgmp_stopFrame(struct LG_Transport * this) { - if (this->framePending && this->frameQueue) + if (this->framePending && this->pendingFrameQueue) { - const LGMP_STATUS status = lgmpClientMessageDone(this->frameQueue); + const LGMP_STATUS status = + lgmpClientMessageDone(this->pendingFrameQueue); if (status != LGMP_OK) DEBUG_WARN("Failed to release pending LGMP frame: %s", lgmpStatusString(status)); } - this->framePending = false; - this->pendingFrame = NULL; + this->framePending = false; + this->pendingFrame = NULL; + this->pendingFrameQueue = NULL; + LGMP_STATUS status = lgmpClientUnsubscribe(&this->frameQueue); if (status != LGMP_OK) { - DEBUG_WARN("Failed to unsubscribe from the LGMP frame queue: %s", + DEBUG_WARN("Failed to unsubscribe from the shared LGMP frame queue: %s", lgmpStatusString(status)); this->frameQueue = NULL; } + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + { + status = lgmpClientUnsubscribe(&this->ownerFrameQueue[i]); + if (status != LGMP_OK) + { + DEBUG_WARN("Failed to unsubscribe from owner LGMP frame queue %u: %s", + i, lgmpStatusString(status)); + this->ownerFrameQueue[i] = NULL; + } + } - this->frameSerial = 0; - this->formatValid = false; + this->frameSerial = 0; + this->frameSerialValid = false; + this->formatValid = false; } static void lgmp_stopPointer(struct LG_Transport * this) @@ -252,7 +270,7 @@ static void lgmp_closeQueues(struct LG_Transport * this) static void lgmp_closeDMA(struct LG_Transport * this) { - for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + for (unsigned i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) { if (this->dma[i].fd >= 0) close(this->dma[i].fd); @@ -370,9 +388,12 @@ static LG_TransportStatus lgmp_connect(LG_Transport * this, case LGMP_OK: if (!lgmp_parseSession(data, size, session)) return LG_TRANSPORT_INVALID_VERSION; - this->connected = true; - this->frameSerial = 0; - this->formatValid = false; + this->connected = true; + this->frameScheduleSupported = + session->features & LG_TRANSPORT_FEATURE_FRAME_SCHEDULE; + this->frameSerial = 0; + this->frameSerialValid = false; + this->formatValid = false; return LG_TRANSPORT_OK; case LGMP_ERR_INVALID_VERSION: @@ -392,8 +413,9 @@ static void lgmp_disconnect(LG_Transport * this) { lgmp_closeQueues(this); lgmp_closeDMA(this); - this->connected = false; - this->clientID = 0; + this->connected = false; + this->frameScheduleSupported = false; + this->clientID = 0; } static bool lgmp_sessionValid(LG_Transport * this) @@ -457,11 +479,119 @@ static LG_TransportStatus lgmp_process(PLGMPClientQueue queue, } } +struct LGMPFrameMessage +{ + PLGMPClientQueue queue; + LGMPMessage message; + const KVMFRFrame * frame; + bool owner; +}; + +static LG_TransportStatus lgmp_pollFrameQueue(PLGMPClientQueue queue, + bool owner, struct LGMPFrameMessage * result) +{ + const LGMP_STATUS advance = lgmpClientAdvanceToLast(queue); + switch (advance) + { + case LGMP_OK: + break; + case LGMP_ERR_QUEUE_EMPTY: + break; + case LGMP_ERR_INVALID_SESSION: + return LG_TRANSPORT_DISCONNECTED; + default: + DEBUG_ERROR("lgmpClientAdvanceToLast failed: %s", + lgmpStatusString(advance)); + return LG_TRANSPORT_ERROR; + } + + const LG_TransportStatus status = lgmp_process(queue, 0, &result->message); + if (status == LG_TRANSPORT_OK) + { + result->queue = queue; + result->owner = owner || result->message.udata != 0; + } + return status; +} + +static LG_TransportStatus lgmp_doneFrameMessage( + struct LGMPFrameMessage * message) +{ + if (!message || !message->queue) + return LG_TRANSPORT_OK; + + const LGMP_STATUS status = lgmpClientMessageDone(message->queue); + message->queue = NULL; + message->frame = NULL; + + if (status == LGMP_OK) + return LG_TRANSPORT_OK; + if (status == LGMP_ERR_INVALID_SESSION) + return LG_TRANSPORT_DISCONNECTED; + + DEBUG_WARN("Failed to release discarded LGMP frame: %s", + lgmpStatusString(status)); + return LG_TRANSPORT_ERROR; +} + +static void lgmp_mergeFrameStatus(LG_TransportStatus status, + LG_TransportStatus * result) +{ + if (status != LG_TRANSPORT_OK && + (*result == LG_TRANSPORT_OK || + status == LG_TRANSPORT_DISCONNECTED)) + *result = status; +} + +static bool lgmp_validateFrameMessage(struct LGMPFrameMessage * message) +{ + if (message->message.size < sizeof(KVMFRFrame)) + { + DEBUG_ERROR("LGMP frame payload is too small"); + return false; + } + + const KVMFRFrame * frame = (const KVMFRFrame *)message->message.mem; + const size_t frameDataSize = (size_t)frame->dataHeight * frame->pitch; + if (frame->type <= FRAME_TYPE_INVALID || frame->type >= FRAME_TYPE_MAX || + frame->offset > message->message.size - sizeof(FrameBuffer) || + frameDataSize > + message->message.size - frame->offset - sizeof(FrameBuffer)) + { + DEBUG_ERROR("LGMP frame payload contains invalid dimensions or offsets"); + return false; + } + + message->frame = frame; + return true; +} + +static bool lgmp_frameSerialNewer(uint32_t lhs, uint32_t rhs) +{ + return lhs != rhs && + (uint32_t)(lhs - rhs) < UINT32_C(0x80000000); +} + +static void lgmp_selectNewestFrameMessage( + struct LGMPFrameMessage * candidate, + struct LGMPFrameMessage ** selected) +{ + if (!candidate->frame) + return; + + if (!*selected || + lgmp_frameSerialNewer(candidate->frame->frameSerial, + (*selected)->frame->frameSerial) || + (candidate->frame->frameSerial == (*selected)->frame->frameSerial && + candidate->owner && !(*selected)->owner)) + *selected = candidate; +} + static int lgmp_getDMA(struct LG_Transport * this, const KVMFRFrame * frame, size_t dataSize) { struct DMAFrameInfo * dma = NULL; - for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + for (unsigned i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) if (this->dma[i].frame == frame) { dma = &this->dma[i]; @@ -474,7 +604,7 @@ static int lgmp_getDMA(struct LG_Transport * this, const KVMFRFrame * frame, } if (!dma) - for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + for (unsigned i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) if (!this->dma[i].frame) { dma = &this->dma[i]; @@ -502,48 +632,144 @@ static LG_TransportStatus lgmp_nextFrame(LG_Transport * this, bool useDMA, if (this->framePending) return LG_TRANSPORT_ERROR; - LG_TransportStatus status = lgmp_subscribe(this->client, LGMP_Q_FRAME, - &this->frameQueue); - if (status != LG_TRANSPORT_OK) + if (this->frameScheduleSupported) + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + { + const LG_TransportStatus status = lgmp_subscribe(this->client, + LGMP_Q_FRAME_OWNER + i, &this->ownerFrameQueue[i]); + if (status != LG_TRANSPORT_OK && status != LG_TRANSPORT_TIMEOUT) + return status; + } + + const LG_TransportStatus sharedSubscribe = lgmp_subscribe(this->client, + LGMP_Q_FRAME, &this->frameQueue); + if (sharedSubscribe != LG_TRANSPORT_OK && + sharedSubscribe != LG_TRANSPORT_TIMEOUT) + return sharedSubscribe; + + struct LGMPFrameMessage owner[LGMP_Q_FRAME_LEN] = {}; + struct LGMPFrameMessage shared = {}; + LG_TransportStatus sharedStatus = LG_TRANSPORT_TIMEOUT; + LG_TransportStatus ownerStatus[LGMP_Q_FRAME_LEN]; + + /* Select the newest frame across the shared and owner delivery lanes. */ + if (this->frameQueue) + sharedStatus = lgmp_pollFrameQueue(this->frameQueue, false, &shared); + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) { - if (status == LG_TRANSPORT_TIMEOUT) - usleep(1000); - return status; + ownerStatus[i] = LG_TRANSPORT_TIMEOUT; + if (this->ownerFrameQueue[i]) + ownerStatus[i] = lgmp_pollFrameQueue( + this->ownerFrameQueue[i], true, &owner[i]); } - LGMPMessage message; - status = lgmp_process(this->frameQueue, this->framePollInterval, &message); - if (status != LG_TRANSPORT_OK) - return status; + LG_TransportStatus failure = LG_TRANSPORT_OK; + if (sharedStatus != LG_TRANSPORT_OK && + sharedStatus != LG_TRANSPORT_TIMEOUT) + failure = sharedStatus; + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + if (ownerStatus[i] != LG_TRANSPORT_OK && + ownerStatus[i] != LG_TRANSPORT_TIMEOUT && + (failure == LG_TRANSPORT_OK || + ownerStatus[i] == LG_TRANSPORT_DISCONNECTED)) + failure = ownerStatus[i]; - if (message.size < sizeof(KVMFRFrame)) + if (failure != LG_TRANSPORT_OK) { - lgmpClientMessageDone(this->frameQueue); - DEBUG_ERROR("LGMP frame payload is too small"); - return LG_TRANSPORT_ERROR; + lgmp_mergeFrameStatus( + lgmp_doneFrameMessage(&shared), &failure); + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + lgmp_mergeFrameStatus( + lgmp_doneFrameMessage(&owner[i]), &failure); + return failure; } - const KVMFRFrame * frame = (const KVMFRFrame *)message.mem; - const size_t frameDataSize = (size_t)frame->dataHeight * frame->pitch; - if (frame->type <= FRAME_TYPE_INVALID || frame->type >= FRAME_TYPE_MAX || - frame->offset > message.size - sizeof(FrameBuffer) || - frameDataSize > message.size - frame->offset - sizeof(FrameBuffer)) + bool malformed = false; + LG_TransportStatus releaseFailure = LG_TRANSPORT_OK; + if (sharedStatus == LG_TRANSPORT_OK && + !lgmp_validateFrameMessage(&shared)) { - lgmpClientMessageDone(this->frameQueue); - DEBUG_ERROR("LGMP frame payload contains invalid dimensions or offsets"); - return LG_TRANSPORT_ERROR; + malformed = true; + releaseFailure = lgmp_doneFrameMessage(&shared); + } + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + if (ownerStatus[i] == LG_TRANSPORT_OK && + !lgmp_validateFrameMessage(&owner[i])) + { + malformed = true; + const LG_TransportStatus done = + lgmp_doneFrameMessage(&owner[i]); + lgmp_mergeFrameStatus(done, &releaseFailure); + } + + if (releaseFailure != LG_TRANSPORT_OK) + { + lgmp_mergeFrameStatus( + lgmp_doneFrameMessage(&shared), &releaseFailure); + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + lgmp_mergeFrameStatus( + lgmp_doneFrameMessage(&owner[i]), &releaseFailure); + return releaseFailure; } - if (frame->frameSerial == this->frameSerial && this->frameSerial) + struct LGMPFrameMessage * selected = NULL; + lgmp_selectNewestFrameMessage(&shared, &selected); + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + lgmp_selectNewestFrameMessage(&owner[i], &selected); + + if (!selected) { - lgmpClientMessageDone(this->frameQueue); - return LG_TRANSPORT_TIMEOUT; + if (this->framePollInterval) + usleep(this->framePollInterval); + return malformed ? LG_TRANSPORT_ERROR : LG_TRANSPORT_TIMEOUT; } - this->frameSerial = frame->frameSerial; + + if (selected != &shared) + releaseFailure = lgmp_doneFrameMessage(&shared); + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + if (selected != &owner[i]) + { + const LG_TransportStatus done = + lgmp_doneFrameMessage(&owner[i]); + lgmp_mergeFrameStatus(done, &releaseFailure); + } + + if (releaseFailure != LG_TRANSPORT_OK) + { + lgmp_mergeFrameStatus( + lgmp_doneFrameMessage(selected), &releaseFailure); + return releaseFailure; + } + + const KVMFRFrame * frame = selected->frame; + const bool equalSerial = this->frameSerialValid && + frame->frameSerial == this->frameSerial; + if (this->frameSerialValid && !equalSerial && + !lgmp_frameSerialNewer(frame->frameSerial, this->frameSerial)) + { + const LG_TransportStatus done = lgmp_doneFrameMessage(selected); + return done == LG_TRANSPORT_OK ? LG_TRANSPORT_TIMEOUT : done; + } + if (equalSerial && !selected->owner) + { + const LG_TransportStatus done = lgmp_doneFrameMessage(selected); + return done == LG_TRANSPORT_OK ? LG_TRANSPORT_TIMEOUT : done; + } + + const bool fullDamage = !this->frameSerialValid || equalSerial || + frame->frameSerial != this->frameSerial + 1; + this->frameSerial = frame->frameSerial; + this->frameSerialValid = true; memset(result, 0, sizeof(*result)); - result->serial = frame->frameSerial; - result->scheduleGeneration = message.udata; + result->serial = frame->frameSerial; + if (selected->owner) + { + const uint64_t scheduleToken = selected->message.udata; + result->scheduleGeneration = (uint32_t)(scheduleToken >> 32); + result->scheduleEpoch = (uint32_t)scheduleToken; + result->scheduleOwner = true; + } if (frame->flags & FRAME_FLAG_BLOCK_SCREENSAVER) result->flags |= LG_TRANSPORT_FRAME_BLOCK_SCREENSAVER; if (frame->flags & FRAME_FLAG_REQUEST_ACTIVATION) @@ -596,22 +822,24 @@ static LG_TransportStatus lgmp_nextFrame(LG_Transport * this, bool useDMA, result->dmaFD = lgmp_getDMA(this, frame, dataSize); if (result->dmaFD < 0) { - lgmpClientMessageDone(this->frameQueue); + const LG_TransportStatus done = lgmp_doneFrameMessage(selected); DEBUG_ERROR("Failed to obtain a DMA buffer for the frame"); - return LG_TRANSPORT_ERROR; + return done == LG_TRANSPORT_DISCONNECTED ? + done : LG_TRANSPORT_ERROR; } } - if (frame->damageRectsCount <= KVMFR_MAX_DAMAGE_RECTS) + if (!fullDamage && frame->damageRectsCount <= KVMFR_MAX_DAMAGE_RECTS) { result->damageRects = frame->damageRects; result->damageRectsCount = frame->damageRectsCount; } - else + else if (!fullDamage) DEBUG_WARN("Invalid damage rectangles, forcing a full update"); - this->framePending = true; - this->pendingFrame = frame; + this->framePending = true; + this->pendingFrame = frame; + this->pendingFrameQueue = selected->queue; return LG_TRANSPORT_OK; } @@ -652,10 +880,11 @@ static void lgmp_getFrameTiming(LG_Transport * this, static void lgmp_releaseFrame(LG_Transport * this, LG_TransportFrame * frame) { - if (this->framePending && this->frameQueue) - lgmpClientMessageDone(this->frameQueue); - this->framePending = false; - this->pendingFrame = NULL; + if (this->framePending && this->pendingFrameQueue) + lgmpClientMessageDone(this->pendingFrameQueue); + this->framePending = false; + this->pendingFrame = NULL; + this->pendingFrameQueue = NULL; memset(frame, 0, sizeof(*frame)); } @@ -721,7 +950,7 @@ static LG_TransportStatus lgmp_nextPointer(LG_Transport * this, if (status == LG_TRANSPORT_OK) { memcpy(this->pointerData, message.mem, needed); - pointerFlags = message.udata; + pointerFlags = (uint32_t)message.udata; } lgmpClientMessageDone(this->pointerQueue); } @@ -797,15 +1026,17 @@ static LG_TransportStatus lgmp_sendControl(LG_Transport * this, case LG_TRANSPORT_CONTROL_FRAME_SCHEDULE: { const KVMFRFrameSchedule message = { - .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, - .clientID = this->clientID, - .generation = control->frameSchedule.generation, - .flags = control->frameSchedule.flags, - .period = control->frameSchedule.period, - .targetSlack = control->frameSchedule.targetSlack, - .phaseError = control->frameSchedule.phaseError, - .feedbackFrameSerial = control->frameSchedule.feedbackFrameSerial, - .lease = control->frameSchedule.lease, + .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, + .clientID = this->clientID, + .generation = control->frameSchedule.generation, + .flags = control->frameSchedule.flags, + .period = control->frameSchedule.period, + .targetSlack = control->frameSchedule.targetSlack, + .phaseError = control->frameSchedule.phaseError, + .feedbackFrameSerial = control->frameSchedule.feedbackFrameSerial, + .feedbackScheduleEpoch = + control->frameSchedule.feedbackScheduleEpoch, + .lease = control->frameSchedule.lease, }; memcpy(buffer, &message, sizeof(message)); size = sizeof(message); diff --git a/common/include/common/KVMFR.h b/common/include/common/KVMFR.h index 3b92d326..8e3ad0a5 100644 --- a/common/include/common/KVMFR.h +++ b/common/include/common/KVMFR.h @@ -30,7 +30,7 @@ #include "LGMPConfig.h" #define KVMFR_MAGIC "KVMFR---" -#define KVMFR_VERSION 26 +#define KVMFR_VERSION 27 // Fallback used by producers that cannot report the source display's SDR // white level. IDD frames override this with IDDCX_METADATA2::SdrWhiteLevel. @@ -296,8 +296,9 @@ typedef struct KVMFRFrameSchedule // Positive when the frame arrived early, negative when it arrived late. int64_t phaseError; // ready-to-render phase error (ns) uint32_t feedbackFrameSerial; + uint32_t feedbackScheduleEpoch; uint32_t lease; // lease duration (ms) - uint8_t reserved[16]; + uint8_t reserved[12]; } KVMFRFrameSchedule; diff --git a/common/include/common/LGMPConfig.h b/common/include/common/LGMPConfig.h index 6039e9be..263bc183 100644 --- a/common/include/common/LGMPConfig.h +++ b/common/include/common/LGMPConfig.h @@ -23,8 +23,13 @@ #define LGMP_Q_POINTER 1 #define LGMP_Q_FRAME 2 +// Base ID for LGMP_Q_FRAME_LEN independent owner-delivery queues. +#define LGMP_Q_FRAME_OWNER 3 -#define LGMP_Q_FRAME_LEN 2 -#define LGMP_Q_POINTER_LEN 32 +// Two delivery lanes plus a spare buffer let the timing owner continue to +// alternate buffers while a secondary client holds the shared delivery. +#define LGMP_Q_FRAME_LEN 2 +#define LGMP_Q_FRAME_BUFFER_LEN 3 +#define LGMP_Q_POINTER_LEN 32 #endif diff --git a/idd/LGIdd/CFrameBufferPool.h b/idd/LGIdd/CFrameBufferPool.h index e961f369..acef28da 100644 --- a/idd/LGIdd/CFrameBufferPool.h +++ b/idd/LGIdd/CFrameBufferPool.h @@ -30,7 +30,7 @@ class CFrameBufferPool { CSwapChainProcessor * m_swapChain; - CFrameBufferResource m_buffers[LGMP_Q_FRAME_LEN]; + CFrameBufferResource m_buffers[LGMP_Q_FRAME_BUFFER_LEN]; public: void Init(CSwapChainProcessor * swapChain); @@ -39,4 +39,4 @@ class CFrameBufferPool CFrameBufferResource* CFrameBufferPool::Get( const CIndirectDeviceContext::PreparedFrameBuffer& buffer, size_t minSize); -}; \ No newline at end of file +}; diff --git a/idd/LGIdd/CFrameScheduler.cpp b/idd/LGIdd/CFrameScheduler.cpp index 82be4b46..533cdf4c 100644 --- a/idd/LGIdd/CFrameScheduler.cpp +++ b/idd/LGIdd/CFrameScheduler.cpp @@ -90,8 +90,7 @@ bool CFrameScheduler::ElectOwner(uint64_t now) if (!client.active || client.expiry <= now) { client.active = false; - fastest = nullptr; - break; + continue; } if (!fastest || client.period < fastest->period) @@ -108,6 +107,7 @@ bool CFrameScheduler::ElectOwner(uint64_t now) const uint32_t oldClientID = m_schedule.clientID; const uint32_t oldGeneration = m_schedule.generation; + const uint32_t oldEpoch = m_schedule.epoch; const uint64_t oldPeriod = m_schedule.period; const uint64_t oldSlack = m_schedule.targetSlack; if (!fastest) @@ -119,16 +119,27 @@ bool CFrameScheduler::ElectOwner(uint64_t now) { m_schedule.clientID = fastest->clientID; m_schedule.generation = fastest->generation; + m_schedule.epoch = oldEpoch; m_schedule.period = fastest->period; m_schedule.targetSlack = fastest->targetSlack; - m_scheduling = true; + m_scheduling = true; } const bool ownerChanged = oldClientID != m_schedule.clientID; if (ownerChanged || oldGeneration != m_schedule.generation) { - m_nextDeadline = m_scheduling ? now + m_schedule.period : 0; - m_forceNext = m_scheduling; + if (m_scheduling) + { + if (!++m_epoch) + ++m_epoch; + m_schedule.epoch = m_epoch; + } + m_nextDeadline = m_scheduling ? now + m_schedule.period : 0; + m_forceNext = m_scheduling; + m_republishNext = m_scheduling; + + if (fastest) + fastest->lastFeedbackFrameSerial = 0; m_lastPublishedFrameSerial = 0; m_lastPhaseError = 0; @@ -138,8 +149,8 @@ bool CFrameScheduler::ElectOwner(uint64_t now) m_lastLogPublished = m_publishedFrames; if (ownerChanged && m_scheduling) - DEBUG_INFO("Frame timing owner %u generation %u at %.3f Hz", - m_schedule.clientID, m_schedule.generation, + DEBUG_INFO("Frame timing owner %u generation %u epoch %u at %.3f Hz", + m_schedule.clientID, m_schedule.generation, m_schedule.epoch, 1000000000.0 / m_schedule.period); else if (ownerChanged && oldClientID) DEBUG_INFO("Frame timing owner released; using push delivery"); @@ -154,9 +165,11 @@ void CFrameScheduler::Reset() AcquireSRWLockExclusive(&m_lock); for (Client& client : m_clients) client = {}; - m_schedule = {}; - m_scheduling = false; - m_forceNext = false; + m_schedule = {}; + m_scheduling = false; + m_forceNext = false; + m_republishNext = false; + m_epoch = 0; m_lastArrival = 0; m_guestPeriod = 0; @@ -198,11 +211,15 @@ void CFrameScheduler::UpdateSubscribers(const uint32_t * clientIDs, } if (client) - client->subscribed = true; + { + client->subscribed = true; + client->subscriptionSeen = true; + } } for (Client& client : m_clients) - if (client.clientID && !client.subscribed) + if (client.clientID && !client.subscribed && + (client.subscriptionSeen || !client.active || client.expiry <= now)) client = {}; const bool changed = ElectOwner(now); @@ -211,8 +228,8 @@ void CFrameScheduler::UpdateSubscribers(const uint32_t * clientIDs, WakePublisher(); } -bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, - uint64_t now) +bool CFrameScheduler::UpdateSchedule(uint32_t sourceClientID, + const KVMFRFrameSchedule& schedule, uint64_t now) { static const KVMFRFrameScheduleFlags validFlags = KVMFR_FRAME_SCHEDULE_ACTIVE | @@ -220,19 +237,20 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, KVMFR_FRAME_SCHEDULE_RESET | KVMFR_FRAME_SCHEDULE_IMMEDIATE; - if (!schedule.clientID || schedule.flags & ~validFlags) + if (!sourceClientID || schedule.clientID != sourceClientID || + schedule.flags & ~validFlags) return false; - AcquireSRWLockExclusive(&m_lock); - Client * client = FindClient(schedule.clientID); - bool wake = false; - if (schedule.flags & KVMFR_FRAME_SCHEDULE_RELEASE) { - if (client && client->subscribed) + AcquireSRWLockExclusive(&m_lock); + Client * client = FindClient(schedule.clientID); + bool wake = false; + if (client) { - client->active = false; - client->expiry = 0; + client->active = false; + client->expiry = 0; + client->immediate = false; wake = ElectOwner(now); } ReleaseSRWLockExclusive(&m_lock); @@ -241,12 +259,6 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, return true; } - if (!client || !client->subscribed) - { - ReleaseSRWLockExclusive(&m_lock); - return false; - } - if (!(schedule.flags & KVMFR_FRAME_SCHEDULE_ACTIVE) || schedule.period < MIN_PERIOD_NS || schedule.period > MAX_PERIOD_NS || @@ -254,24 +266,47 @@ bool CFrameScheduler::UpdateSchedule(const KVMFRFrameSchedule& schedule, schedule.phaseError > static_cast(schedule.period) || schedule.phaseError < -static_cast(schedule.period) || schedule.lease < MIN_LEASE_MS || schedule.lease > MAX_LEASE_MS) + return false; + + AcquireSRWLockExclusive(&m_lock); + Client * client = FindClient(schedule.clientID); + if (!client) + for (Client& candidate : m_clients) + if (!candidate.clientID) + { + candidate.clientID = schedule.clientID; + client = &candidate; + break; + } + + if (!client) { ReleaseSRWLockExclusive(&m_lock); return false; } + bool wake = false; + if (client->generation != schedule.generation) + { client->lastFeedbackFrameSerial = 0; + client->immediate = false; + } client->generation = schedule.generation; client->period = schedule.period; client->targetSlack = schedule.targetSlack; client->expiry = now + static_cast(schedule.lease) * 1000000; client->active = true; if (schedule.flags & KVMFR_FRAME_SCHEDULE_IMMEDIATE) - { - m_forceNext = true; - wake = true; - } + client->immediate = true; wake |= ElectOwner(now); + if (m_scheduling && client->clientID == m_schedule.clientID && + client->generation == m_schedule.generation && client->immediate) + { + m_forceNext = true; + m_republishNext = true; + wake = true; + } wake |= ApplyFeedback(*client, schedule); ReleaseSRWLockExclusive(&m_lock); if (wake) @@ -284,6 +319,7 @@ bool CFrameScheduler::ApplyFeedback(Client& client, { if (!m_scheduling || client.clientID != m_schedule.clientID || schedule.generation != m_schedule.generation || + schedule.feedbackScheduleEpoch != m_schedule.epoch || !schedule.feedbackFrameSerial || (client.lastFeedbackFrameSerial && static_cast(schedule.feedbackFrameSerial - @@ -366,11 +402,12 @@ void CFrameScheduler::ForceFrame() } bool CFrameScheduler::GetPublishTarget(uint64_t now, uint64_t& target, - uint32_t& generation, bool& periodic) + Schedule& schedule, bool& periodic, bool& republish) { - target = now; - generation = 0; - periodic = false; + target = now; + schedule = {}; + periodic = false; + republish = false; AcquireSRWLockExclusive(&m_lock); if (!m_scheduling) { @@ -378,7 +415,8 @@ bool CFrameScheduler::GetPublishTarget(uint64_t now, uint64_t& target, return true; } - generation = m_schedule.generation; + schedule = m_schedule; + republish = m_republishNext; if (m_forceNext) { @@ -412,13 +450,20 @@ void CFrameScheduler::FrameSuperseded() ReleaseSRWLockExclusive(&m_lock); } -void CFrameScheduler::FramePublished(uint32_t generation, +void CFrameScheduler::FramePublished(const Schedule& schedule, uint32_t frameSerial, uint64_t now, bool periodic) { AcquireSRWLockExclusive(&m_lock); - if (m_scheduling && generation == m_schedule.generation) + if (m_scheduling && schedule.clientID == m_schedule.clientID && + schedule.generation == m_schedule.generation && + schedule.epoch == m_schedule.epoch) { - m_forceNext = false; + m_forceNext = false; + m_republishNext = false; + + Client * client = FindClient(m_schedule.clientID); + if (client) + client->immediate = false; m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; if (periodic) @@ -430,6 +475,38 @@ void CFrameScheduler::FramePublished(uint32_t generation, ReleaseSRWLockExclusive(&m_lock); } +void CFrameScheduler::FrameRepublished(const Schedule& schedule, + uint32_t frameSerial) +{ + AcquireSRWLockExclusive(&m_lock); + if (m_scheduling && schedule.clientID == m_schedule.clientID && + schedule.generation == m_schedule.generation && + schedule.epoch == m_schedule.epoch) + { + m_republishNext = false; + + Client * client = FindClient(m_schedule.clientID); + if (client) + client->immediate = false; + m_lastPublishedFrameSerial = frameSerial; + ++m_publishedFrames; + } + ReleaseSRWLockExclusive(&m_lock); +} + +void CFrameScheduler::FrameDelivered(const uint32_t * clientIDs, + unsigned count) +{ + AcquireSRWLockExclusive(&m_lock); + for (unsigned i = 0; i < count; ++i) + { + Client * client = FindClient(clientIDs[i]); + if (client) + client->immediate = false; + } + ReleaseSRWLockExclusive(&m_lock); +} + void CFrameScheduler::LogStatistics(uint64_t now) { AcquireSRWLockExclusive(&m_lock); diff --git a/idd/LGIdd/CFrameScheduler.h b/idd/LGIdd/CFrameScheduler.h index 1f6366c3..05d1f38b 100644 --- a/idd/LGIdd/CFrameScheduler.h +++ b/idd/LGIdd/CFrameScheduler.h @@ -36,6 +36,7 @@ public: { uint32_t clientID; uint32_t generation; + uint32_t epoch; uint64_t period; uint64_t targetSlack; }; @@ -50,7 +51,9 @@ private: uint64_t expiry; uint32_t lastFeedbackFrameSerial; bool subscribed; + bool subscriptionSeen; bool active; + bool immediate; }; mutable SRWLOCK m_lock = SRWLOCK_INIT; @@ -59,6 +62,8 @@ private: Schedule m_schedule = {}; bool m_scheduling = false; bool m_forceNext = false; + bool m_republishNext = false; + uint32_t m_epoch = 0; uint64_t m_lastArrival = 0; uint64_t m_guestPeriod = 0; @@ -91,16 +96,19 @@ public: void Reset(); void UpdateSubscribers(const uint32_t * clientIDs, unsigned count, uint64_t now); - bool UpdateSchedule(const KVMFRFrameSchedule& schedule, uint64_t now); + bool UpdateSchedule(uint32_t sourceClientID, + const KVMFRFrameSchedule& schedule, uint64_t now); bool GetSchedule(Schedule& schedule) const; HANDLE GetWakeEvent() const { return m_wakeEvent; } void ObserveFrame(uint64_t now); void ForceFrame(); bool GetPublishTarget(uint64_t now, uint64_t& target, - uint32_t& generation, bool& periodic); + Schedule& schedule, bool& periodic, bool& republish); void FrameSuperseded(); - void FramePublished(uint32_t generation, uint32_t frameSerial, + void FramePublished(const Schedule& schedule, uint32_t frameSerial, uint64_t now, bool periodic); + void FrameRepublished(const Schedule& schedule, uint32_t frameSerial); + void FrameDelivered(const uint32_t * clientIDs, unsigned count); void RecordFrameTiming(uint64_t duration); void LogStatistics(uint64_t now); }; diff --git a/idd/LGIdd/CIndirectDeviceContext.cpp b/idd/LGIdd/CIndirectDeviceContext.cpp index 555578b8..9a3731f8 100644 --- a/idd/LGIdd/CIndirectDeviceContext.cpp +++ b/idd/LGIdd/CIndirectDeviceContext.cpp @@ -44,6 +44,22 @@ static const struct LGMPQueueConfig POINTER_QUEUE_CONFIG = 1000 //subTimeout }; +static uint64_t FrameScheduleToken( + const CFrameScheduler::Schedule& schedule) +{ + return static_cast(schedule.generation) << 32 | + schedule.epoch; +} + +static bool FrameScheduleMatches( + const CFrameScheduler::Schedule& a, + const CFrameScheduler::Schedule& b) +{ + return a.clientID == b.clientID && + a.generation == b.generation && + a.epoch == b.epoch; +} + static const UINT IDDCX_VERSION_1_10 = 0x1A00; static const UINT64 FRAME_BYTES_PER_PIXEL = 4; @@ -879,7 +895,7 @@ bool CIndirectDeviceContext::GetResolutionMemoryRequirements( return false; ivshmemSize = frameMemoryStart + - frameAllocationSize * LGMP_Q_FRAME_LEN; + frameAllocationSize * LGMP_Q_FRAME_BUFFER_LEN; return true; } @@ -1026,6 +1042,23 @@ bool CIndirectDeviceContext::InitializeLGMP() return false; } + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + { + const struct LGMPQueueConfig config = + { + LGMP_Q_FRAME_OWNER + i, //queueID + LGMP_Q_FRAME_LEN, //numMessages + 1000 //subTimeout + }; + if ((status = lgmpHostQueueNew( + m_lgmp, config, &m_frameOwnerQueue[i])) != LGMP_OK) + { + DEBUG_ERROR("lgmpHostQueueCreate Failed (Frame Owner %u): %s", + i, lgmpStatusString(status)); + return false; + } + } + if ((status = lgmpHostQueueNew(m_lgmp, POINTER_QUEUE_CONFIG, &m_pointerQueue)) != LGMP_OK) { DEBUG_ERROR("lgmpHostQueueCreate Failed (Pointer): %s", lgmpStatusString(status)); @@ -1110,7 +1143,7 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize) const size_t alignmentMask = m_alignSize - 1; size_t frameAllocationSize = - (available - alignmentPadding) / LGMP_Q_FRAME_LEN; + (available - alignmentPadding) / LGMP_Q_FRAME_BUFFER_LEN; frameAllocationSize &= ~alignmentMask; if (frameAllocationSize <= m_alignSize || frameAllocationSize > UINT32_MAX) @@ -1127,7 +1160,7 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize) (unsigned int)(maxFrameSize / 1048576)); LGMP_STATUS status; - for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i) + for (int i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) { if ((status = lgmpHostMemAllocAligned(m_lgmp, (uint32_t)frameAllocationSize, @@ -1150,7 +1183,14 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize) m_maxFrameSize = maxFrameSize; m_publishedFrameIndex.store(-1, std::memory_order_release); - m_frameResendPending = false; + m_frameResendPending = false; + m_framePublishSequence = 0; + memset(m_frameLastPublishSequence, 0, + sizeof(m_frameLastPublishSequence)); + for (FrameDelivery& delivery : m_frameDelivery) + delivery = {}; + for (OwnerDelivery& delivery : m_ownerDelivery) + delivery = {}; WDF_TIMER_CONFIG config; WDF_TIMER_CONFIG_INIT_PERIODIC(&config, @@ -1198,7 +1238,14 @@ void CIndirectDeviceContext::DeInitLGMP() { m_frameScheduler.Reset(); m_publishedFrameIndex.store(-1, std::memory_order_release); - m_frameResendPending = false; + m_frameResendPending = false; + m_framePublishSequence = 0; + memset(m_frameLastPublishSequence, 0, + sizeof(m_frameLastPublishSequence)); + for (FrameDelivery& delivery : m_frameDelivery) + delivery = {}; + for (OwnerDelivery& delivery : m_ownerDelivery) + delivery = {}; return; } @@ -1212,13 +1259,20 @@ void CIndirectDeviceContext::DeInitLGMP() AcquireSRWLockExclusive(&m_framePublishLock); m_publishedFrameIndex.store(-1, std::memory_order_release); - m_frameResendPending = false; + m_frameResendPending = false; + m_framePublishSequence = 0; + memset(m_frameLastPublishSequence, 0, + sizeof(m_frameLastPublishSequence)); + for (FrameDelivery& delivery : m_frameDelivery) + delivery = {}; + for (OwnerDelivery& delivery : m_ownerDelivery) + delivery = {}; ReleaseSRWLockExclusive(&m_framePublishLock); - for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i) + for (int i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) m_frameInFlight[i].store(false, std::memory_order_release); - for (int i = 0; i < LGMP_Q_FRAME_LEN; ++i) + for (int i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) lgmpHostMemFree(&m_frameMemory[i]); for (int i = 0; i < LGMP_Q_POINTER_LEN; ++i) lgmpHostMemFree(&m_pointerMemory[i]); @@ -1264,18 +1318,37 @@ void CIndirectDeviceContext::LGMPTimer() } const uint64_t now = CFrameScheduler::Nanotime(); - uint32_t clientIDs[LGMP_MAX_CLIENTS] = {}; - unsigned clientCount = 0; - status = lgmpHostGetClientIDs(m_frameQueue, clientIDs, &clientCount); - if (status == LGMP_OK) - m_frameScheduler.UpdateSubscribers(clientIDs, clientCount, now); - else - DEBUG_WARN("Failed to query LGMP frame subscribers: %s", - lgmpStatusString(status)); + uint32_t clientIDs[LGMP_MAX_CLIENTS] = {}; + unsigned clientCount = 0; + LGMP_STATUS subscriberStatus = lgmpHostGetClientIDs( + m_frameQueue, clientIDs, &clientCount); + for (unsigned queueIndex = 0; + subscriberStatus == LGMP_OK && queueIndex < LGMP_Q_FRAME_LEN; + ++queueIndex) + { + uint32_t queueClientIDs[LGMP_MAX_CLIENTS] = {}; + unsigned queueClientCount = 0; + subscriberStatus = lgmpHostGetClientIDs( + m_frameOwnerQueue[queueIndex], queueClientIDs, &queueClientCount); + + unsigned commonCount = 0; + for (unsigned i = 0; + subscriberStatus == LGMP_OK && i < clientCount; + ++i) + for (unsigned candidate = 0; candidate < queueClientCount; ++candidate) + if (clientIDs[i] == queueClientIDs[candidate]) + { + clientIDs[commonCount++] = clientIDs[i]; + break; + } + clientCount = commonCount; + } uint8_t data[LGMP_MSGS_SIZE]; size_t size; - while ((status = lgmpHostReadData(m_pointerQueue, &data, &size)) == LGMP_OK) + uint32_t sourceClientID; + while ((status = lgmpHostReadDataWithSource( + m_pointerQueue, &data, &size, &sourceClientID)) == LGMP_OK) { KVMFRMessage * msg = (KVMFRMessage *)data; switch (msg->type) @@ -1296,10 +1369,28 @@ void CIndirectDeviceContext::LGMPTimer() case KVMFR_MESSAGE_FRAME_SCHEDULE: { - if (size != sizeof(KVMFRFrameSchedule) || - !m_frameScheduler.UpdateSchedule( - *reinterpret_cast(msg), now)) + const KVMFRFrameSchedule * frameSchedule = + reinterpret_cast(msg); + const bool valid = size == sizeof(*frameSchedule) && + m_frameScheduler.UpdateSchedule( + sourceClientID, *frameSchedule, now); + if (!valid) DEBUG_WARN("Ignoring invalid KVMFR frame schedule"); + else if ((frameSchedule->flags & + (KVMFR_FRAME_SCHEDULE_ACTIVE | + KVMFR_FRAME_SCHEDULE_IMMEDIATE)) == + (KVMFR_FRAME_SCHEDULE_ACTIVE | + KVMFR_FRAME_SCHEDULE_IMMEDIATE)) + { + CFrameScheduler::Schedule owner = {}; + if (!m_frameScheduler.GetSchedule(owner) || + owner.clientID != sourceClientID) + { + AcquireSRWLockExclusive(&m_framePublishLock); + m_frameResendPending = true; + ReleaseSRWLockExclusive(&m_framePublishLock); + } + } break; } } @@ -1307,28 +1398,22 @@ void CIndirectDeviceContext::LGMPTimer() lgmpHostAckData(m_pointerQueue); } + if (subscriberStatus == LGMP_OK) + m_frameScheduler.UpdateSubscribers(clientIDs, clientCount, now); + else + DEBUG_WARN("Failed to query LGMP frame subscribers: %s", + lgmpStatusString(subscriberStatus)); + m_frameScheduler.LogStatistics(now); - AcquireSRWLockExclusive(&m_framePublishLock); if (lgmpHostQueueNewSubs(m_frameQueue)) - m_frameResendPending = true; - - if (m_frameResendPending && m_monitor && - lgmpHostQueuePending(m_frameQueue) == 0) { - const LONG frameIndex = - m_publishedFrameIndex.load(std::memory_order_acquire); - if (frameIndex >= 0) - { - status = lgmpHostQueuePost( - m_frameQueue, 0, m_frameMemory[frameIndex]); - if (status == LGMP_OK) - m_frameResendPending = false; - else if (status != LGMP_ERR_QUEUE_FULL) - DEBUG_ERROR("Failed to resend frame: %s", lgmpStatusString(status)); - } + AcquireSRWLockExclusive(&m_framePublishLock); + m_frameResendPending = true; + ReleaseSRWLockExclusive(&m_framePublishLock); } - ReleaseSRWLockExclusive(&m_framePublishLock); + + ProcessFrameDeliveries(); if (lgmpHostQueueNewSubs(m_pointerQueue)) { @@ -1337,18 +1422,347 @@ void CIndirectDeviceContext::LGMPTimer() } } -bool CIndirectDeviceContext::FrameBufferAvailable() const +bool CIndirectDeviceContext::PostSharedFrame(unsigned frameIndex, + uint32_t excludeClientID) { - if (!m_lgmp || !m_frameQueue || + if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN || + lgmpHostQueuePending(m_frameQueue) != 0) + return false; + + uint32_t clientIDs[LGMP_MAX_CLIENTS] = {}; + unsigned clientCount = 0; + LGMP_STATUS status = + lgmpHostGetClientIDs(m_frameQueue, clientIDs, &clientCount); + if (status != LGMP_OK) + { + DEBUG_ERROR("Failed to query shared frame subscribers: %s", + lgmpStatusString(status)); + return false; + } + + unsigned targetCount = 0; + for (unsigned i = 0; i < clientCount; ++i) + { + bool excluded = clientIDs[i] == excludeClientID; + for (const OwnerDelivery& delivery : m_ownerDelivery) + if (delivery.active && delivery.clientID == clientIDs[i]) + { + excluded = true; + break; + } + + if (!excluded) + clientIDs[targetCount++] = clientIDs[i]; + } + + if (!targetCount) + { + m_frameDelivery[frameIndex].sharedOwnerToken = 0; + m_frameDelivery[frameIndex].sharedOwnerClientID = 0; + m_frameDelivery[frameIndex].sharedOwnerPending = false; + m_frameDelivery[frameIndex].sharedPending = false; + m_frameDelivery[frameIndex].sharedDelivered = true; + m_frameResendPending = false; + return true; + } + + unsigned recipientCount = 0; + status = lgmpHostQueuePostForClients( + m_frameQueue, 0, m_frameMemory[frameIndex], + clientIDs, targetCount, &recipientCount); + if (status != LGMP_OK) + { + if (status != LGMP_ERR_QUEUE_FULL) + DEBUG_ERROR("Failed to publish shared frame: %s", + lgmpStatusString(status)); + return false; + } + + if (!recipientCount) + { + m_frameDelivery[frameIndex].sharedOwnerToken = 0; + m_frameDelivery[frameIndex].sharedOwnerClientID = 0; + m_frameDelivery[frameIndex].sharedOwnerPending = false; + m_frameDelivery[frameIndex].sharedPending = false; + m_frameDelivery[frameIndex].sharedDelivered = true; + m_frameResendPending = false; + return true; + } + + m_frameDelivery[frameIndex].sharedOwnerToken = 0; + m_frameDelivery[frameIndex].sharedOwnerClientID = 0; + m_frameDelivery[frameIndex].sharedOwnerPending = false; + m_frameDelivery[frameIndex].sharedPending = true; + m_frameDelivery[frameIndex].sharedDelivered = true; + m_frameResendPending = false; + m_frameScheduler.FrameDelivered(clientIDs, targetCount); + return true; +} + +bool CIndirectDeviceContext::PostSharedOwnerFrame(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule) +{ + if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN || !schedule.clientID || lgmpHostQueuePending(m_frameQueue) >= LGMP_Q_FRAME_LEN) return false; + unsigned recipientCount = 0; + const LGMP_STATUS status = lgmpHostQueuePostForClients( + m_frameQueue, FrameScheduleToken(schedule), m_frameMemory[frameIndex], + &schedule.clientID, 1, &recipientCount); + if (status != LGMP_OK || !recipientCount) + { + if (status != LGMP_OK && status != LGMP_ERR_QUEUE_FULL) + DEBUG_ERROR("Failed to publish shared owner frame: %s", + lgmpStatusString(status)); + return false; + } + + FrameDelivery& delivery = m_frameDelivery[frameIndex]; + delivery.sharedOwnerToken = FrameScheduleToken(schedule); + delivery.sharedOwnerClientID = schedule.clientID; + delivery.sharedOwnerPending = true; + delivery.sharedPending = true; + if (!delivery.sharedDelivered) + m_frameResendPending = true; + return true; +} + +void CIndirectDeviceContext::ProcessFrameDeliveries() +{ + if (!m_frameQueue) + return; + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + if (!m_frameOwnerQueue[i]) + return; + + AcquireSRWLockExclusive(&m_framePublishLock); + + for (unsigned i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) + { + if (m_frameDelivery[i].sharedOwnerPending && + !lgmpHostQueueMessagePending( + m_frameQueue, m_frameMemory[i], + m_frameDelivery[i].sharedOwnerToken)) + m_frameDelivery[i].sharedOwnerPending = false; + + if (m_frameDelivery[i].sharedPending && + !lgmpHostQueuePayloadPending(m_frameQueue, m_frameMemory[i])) + m_frameDelivery[i].sharedPending = false; + } + + CFrameScheduler::Schedule schedule = {}; + const bool scheduled = m_frameScheduler.GetSchedule(schedule); + const uint64_t scheduleToken = scheduled ? + FrameScheduleToken(schedule) : 0; const LONG publishedFrameIndex = m_publishedFrameIndex.load(std::memory_order_acquire); - const unsigned frameIndex = - static_cast(publishedFrameIndex + 1) % LGMP_Q_FRAME_LEN; - return !m_frameInFlight[frameIndex].load(std::memory_order_acquire); + int newestAcked = -1; + uint32_t newestAckedClient = 0; + for (unsigned queueIndex = 0; + queueIndex < LGMP_Q_FRAME_LEN; + ++queueIndex) + { + OwnerDelivery& owner = m_ownerDelivery[queueIndex]; + if (!owner.active || + lgmpHostQueuePayloadPending( + m_frameOwnerQueue[queueIndex], + m_frameMemory[owner.frameIndex])) + continue; + + const unsigned frameIndex = owner.frameIndex; + const bool latestDelivery = + static_cast(frameIndex) == publishedFrameIndex && + m_frameLastPublishSequence[frameIndex] == m_framePublishSequence; + const bool authoritative = (!scheduled && latestDelivery) || + (scheduled && owner.clientID == schedule.clientID && + owner.token == scheduleToken); + if (authoritative && + (newestAcked < 0 || + m_frameLastPublishSequence[frameIndex] > + m_frameLastPublishSequence[newestAcked])) + { + newestAcked = static_cast(frameIndex); + newestAckedClient = owner.clientID; + } + else if (!authoritative && !latestDelivery) + m_frameResendPending = true; + + m_frameDelivery[frameIndex].ownerQueueMask &= + ~(1U << queueIndex); + owner = {}; + } + + if (newestAcked >= 0) + { + FrameDelivery& delivery = m_frameDelivery[newestAcked]; + const bool needsShared = !delivery.sharedDelivered; + + const bool latest = newestAcked == publishedFrameIndex && + m_frameLastPublishSequence[newestAcked] == m_framePublishSequence; + if (needsShared && + (!latest || + m_frameInFlight[newestAcked].load(std::memory_order_acquire) || + lgmpHostQueuePending(m_frameQueue) != 0)) + { + m_frameResendPending = true; + } + else if (needsShared && + !PostSharedFrame( + static_cast(newestAcked), newestAckedClient)) + { + m_frameResendPending = true; + } + } + + if (m_frameResendPending && + lgmpHostQueuePending(m_frameQueue) == 0) + { + const LONG frameIndex = + m_publishedFrameIndex.load(std::memory_order_acquire); + if (frameIndex >= 0 && + !m_frameDelivery[frameIndex].sharedPending && + !m_frameInFlight[frameIndex].load(std::memory_order_acquire)) + { + FrameDelivery& delivery = m_frameDelivery[frameIndex]; + bool ownerDelivered = !scheduled; + if (scheduled) + { + ownerDelivered = + delivery.sharedOwnerClientID == schedule.clientID && + delivery.sharedOwnerToken == scheduleToken; + for (const OwnerDelivery& owner : m_ownerDelivery) + if (owner.active && + owner.clientID == schedule.clientID && + owner.token == scheduleToken && + owner.frameIndex == static_cast(frameIndex)) + { + ownerDelivered = true; + break; + } + } + + const bool reserveSharedOwner = scheduled && + delivery.sharedOwnerClientID == schedule.clientID && + delivery.sharedOwnerToken == scheduleToken && + FindAvailableOwnerQueue(static_cast(frameIndex)) < 0; + if (ownerDelivered && !reserveSharedOwner) + { + const uint32_t excludeClientID = scheduled ? + schedule.clientID : delivery.sharedOwnerClientID; + PostSharedFrame( + static_cast(frameIndex), excludeClientID); + } + } + } + ReleaseSRWLockExclusive(&m_framePublishLock); +} + +int CIndirectDeviceContext::FindAvailableOwnerQueue( + unsigned preferredIndex) const +{ + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + { + const unsigned queueIndex = + (preferredIndex + i) % LGMP_Q_FRAME_LEN; + if (!m_ownerDelivery[queueIndex].active && + m_frameOwnerQueue[queueIndex] && + lgmpHostQueuePending(m_frameOwnerQueue[queueIndex]) == 0) + return static_cast(queueIndex); + } + + return -1; +} + +int CIndirectDeviceContext::FindOwnerDelivery(uint32_t clientID) const +{ + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + if (m_ownerDelivery[i].active && + m_ownerDelivery[i].clientID == clientID) + return static_cast(i); + + return -1; +} + +int CIndirectDeviceContext::FindSharedOwnerDelivery( + uint32_t clientID) const +{ + for (unsigned i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) + if (m_frameDelivery[i].sharedOwnerPending && + m_frameDelivery[i].sharedOwnerClientID == clientID) + return static_cast(i); + + return -1; +} + +bool CIndirectDeviceContext::HasOwnerDelivery(uint32_t clientID) const +{ + return FindOwnerDelivery(clientID) >= 0 || + FindSharedOwnerDelivery(clientID) >= 0; +} + +int CIndirectDeviceContext::FindAvailableFrameBuffer() const +{ + const LONG publishedFrameIndex = + m_publishedFrameIndex.load(std::memory_order_acquire); + int available = -1; + uint64_t newestPublish = 0; + for (unsigned frameIndex = 0; + frameIndex < LGMP_Q_FRAME_BUFFER_LEN; + ++frameIndex) + { + if (static_cast(frameIndex) == publishedFrameIndex || + m_frameInFlight[frameIndex].load(std::memory_order_acquire) || + m_frameDelivery[frameIndex].ownerQueueMask || + m_frameDelivery[frameIndex].sharedPending || + lgmpHostQueuePayloadPending( + m_frameQueue, m_frameMemory[frameIndex])) + continue; + + bool ownerPending = false; + for (unsigned queueIndex = 0; + !ownerPending && queueIndex < LGMP_Q_FRAME_LEN; + ++queueIndex) + ownerPending = lgmpHostQueuePayloadPending( + m_frameOwnerQueue[queueIndex], m_frameMemory[frameIndex]); + + if (ownerPending) + continue; + + if (available < 0 || + m_frameLastPublishSequence[frameIndex] > newestPublish) + { + available = static_cast(frameIndex); + newestPublish = m_frameLastPublishSequence[frameIndex]; + } + } + + return available; +} + +bool CIndirectDeviceContext::FrameBufferAvailable( + const CFrameScheduler::Schedule& schedule) +{ + if (!m_lgmp || !m_frameQueue) + return false; + for (unsigned i = 0; i < LGMP_Q_FRAME_LEN; ++i) + if (!m_frameOwnerQueue[i]) + return false; + + AcquireSRWLockShared(&m_framePublishLock); + bool deliveryAvailable; + if (schedule.clientID) + deliveryAvailable = !HasOwnerDelivery(schedule.clientID) && + (FindAvailableOwnerQueue(0) >= 0 || + lgmpHostQueuePending(m_frameQueue) < LGMP_Q_FRAME_LEN); + else + deliveryAvailable = lgmpHostQueuePending(m_frameQueue) == 0; + + const bool available = deliveryAvailable && + FindAvailableFrameBuffer() >= 0; + ReleaseSRWLockShared(&m_framePublishLock); + return available; } void CIndirectDeviceContext::ProcessFrameQueue() @@ -1362,6 +1776,9 @@ void CIndirectDeviceContext::ProcessFrameQueue() if (status != LGMP_OK && status != LGMP_ERR_CORRUPTED) DEBUG_ERROR("lgmpHostProcess Failed: %s", lgmpStatusString(status)); + + if (status == LGMP_OK) + ProcessFrameDeliveries(); } CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrameBuffer( @@ -1375,24 +1792,28 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame const unsigned dataHeight = dstFormat.dataHeight ? dstFormat.dataHeight : dstFormat.desc.Height; - if (!FrameBufferAvailable()) - return result; - if (dstFormat.format == FRAME_TYPE_INVALID) { DEBUG_ERROR("Unsupported frame format, skipping frame"); return result; } - const LONG publishedFrameIndex = - m_publishedFrameIndex.load(std::memory_order_acquire); - const unsigned frameIndex = - static_cast(publishedFrameIndex + 1) % LGMP_Q_FRAME_LEN; - + AcquireSRWLockExclusive(&m_framePublishLock); + const int availableFrameIndex = FindAvailableFrameBuffer(); bool expected = false; - if (!m_frameInFlight[frameIndex].compare_exchange_strong( - expected, true, std::memory_order_acq_rel)) + const bool acquired = availableFrameIndex >= 0 && + m_frameInFlight[availableFrameIndex].compare_exchange_strong( + expected, true, std::memory_order_acq_rel); + if (acquired) + m_frameDelivery[availableFrameIndex] = {}; + const bool fullCopy = acquired && + (!m_frameLastPublishSequence[availableFrameIndex] || + m_framePublishSequence > + m_frameLastPublishSequence[availableFrameIndex] + 1); + ReleaseSRWLockExclusive(&m_framePublishLock); + if (!acquired) return result; + const unsigned frameIndex = static_cast(availableFrameIndex); if (m_width != dataWidth || m_height != dataHeight || @@ -1518,45 +1939,198 @@ CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrame result.frameIndex = frameIndex; result.mem = fb->data; + result.fullCopy = fullCopy; return result; } bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, - uint32_t scheduleGeneration) + const CFrameScheduler::Schedule& schedule) { - if (!m_frameQueue || frameIndex >= LGMP_Q_FRAME_LEN) + if (!m_frameQueue || frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return false; AcquireSRWLockExclusive(&m_framePublishLock); - const LGMP_STATUS status = - lgmpHostQueuePost( - m_frameQueue, scheduleGeneration, m_frameMemory[frameIndex]); - if (status == LGMP_OK) + CFrameScheduler::Schedule currentSchedule = {}; + const bool scheduling = + m_frameScheduler.GetSchedule(currentSchedule); + if (scheduling != (schedule.clientID != 0) || + (scheduling && !FrameScheduleMatches(schedule, currentSchedule))) { + ReleaseSRWLockExclusive(&m_framePublishLock); + return false; + } + + LGMP_STATUS status = LGMP_OK; + bool published = false; + if (schedule.clientID) + { + if (HasOwnerDelivery(schedule.clientID)) + status = LGMP_ERR_QUEUE_FULL; + else + { + const int ownerQueueIndex = FindAvailableOwnerQueue(frameIndex); + if (ownerQueueIndex < 0) + published = PostSharedOwnerFrame(frameIndex, schedule); + else + { + unsigned recipientCount = 0; + status = lgmpHostQueuePostForClients( + m_frameOwnerQueue[ownerQueueIndex], FrameScheduleToken(schedule), + m_frameMemory[frameIndex], + &schedule.clientID, 1, &recipientCount); + if (status == LGMP_OK && recipientCount) + { + const unsigned queueIndex = + static_cast(ownerQueueIndex); + OwnerDelivery& owner = m_ownerDelivery[queueIndex]; + owner.token = FrameScheduleToken(schedule); + owner.clientID = schedule.clientID; + owner.frameIndex = frameIndex; + owner.active = true; + + FrameDelivery& delivery = m_frameDelivery[frameIndex]; + delivery.ownerQueueMask |= 1U << queueIndex; + delivery.sharedDelivered = false; + published = true; + + if (!PostSharedFrame(frameIndex, schedule.clientID)) + m_frameResendPending = true; + } + } + } + } + else + published = PostSharedFrame(frameIndex, 0); + + if (published) + { + m_frameLastPublishSequence[frameIndex] = ++m_framePublishSequence; m_publishedFrameIndex.store( static_cast(frameIndex), std::memory_order_release); - m_frameResendPending = false; } ReleaseSRWLockExclusive(&m_framePublishLock); - if (status != LGMP_OK) + if (!published) { - DEBUG_ERROR("Failed to publish frame: %s", lgmpStatusString(status)); + if (status != LGMP_OK && status != LGMP_ERR_QUEUE_FULL) + DEBUG_ERROR("Failed to publish frame: %s", + lgmpStatusString(status)); return false; } return true; } -void CIndirectDeviceContext::CommitFrameBuffer(unsigned frameIndex, - uint32_t scheduleGeneration, bool periodic) +bool CIndirectDeviceContext::RepublishFrameBuffer( + const CFrameScheduler::Schedule& schedule) { - if (frameIndex >= LGMP_Q_FRAME_LEN) + if (!schedule.clientID) + return false; + + AcquireSRWLockExclusive(&m_framePublishLock); + CFrameScheduler::Schedule currentSchedule = {}; + if (!m_frameScheduler.GetSchedule(currentSchedule) || + !FrameScheduleMatches(schedule, currentSchedule)) + { + ReleaseSRWLockExclusive(&m_framePublishLock); + return false; + } + + const LONG frameIndex = + m_publishedFrameIndex.load(std::memory_order_acquire); + if (frameIndex < 0 || + m_frameInFlight[frameIndex].load(std::memory_order_acquire)) + { + ReleaseSRWLockExclusive(&m_framePublishLock); + return false; + } + + const uint64_t scheduleToken = FrameScheduleToken(schedule); + const int existingSharedDelivery = + FindSharedOwnerDelivery(schedule.clientID); + if (existingSharedDelivery >= 0) + { + const FrameDelivery& delivery = + m_frameDelivery[existingSharedDelivery]; + const bool delivered = existingSharedDelivery == frameIndex && + delivery.sharedOwnerToken == scheduleToken; + const uint32_t frameSerial = m_frame[frameIndex]->frameSerial; + ReleaseSRWLockExclusive(&m_framePublishLock); + if (delivered) + m_frameScheduler.FrameRepublished(schedule, frameSerial); + return delivered; + } + + const int existingDelivery = + FindOwnerDelivery(schedule.clientID); + if (existingDelivery >= 0) + { + const OwnerDelivery& owner = m_ownerDelivery[existingDelivery]; + const bool delivered = owner.frameIndex == + static_cast(frameIndex) && + owner.token == scheduleToken; + const uint32_t frameSerial = m_frame[frameIndex]->frameSerial; + ReleaseSRWLockExclusive(&m_framePublishLock); + if (delivered) + m_frameScheduler.FrameRepublished(schedule, frameSerial); + return delivered; + } + + const int ownerQueueIndex = + FindAvailableOwnerQueue(static_cast(frameIndex)); + if (ownerQueueIndex < 0) + { + const bool published = PostSharedOwnerFrame( + static_cast(frameIndex), schedule); + const uint32_t frameSerial = m_frame[frameIndex]->frameSerial; + ReleaseSRWLockExclusive(&m_framePublishLock); + if (published) + m_frameScheduler.FrameRepublished(schedule, frameSerial); + return published; + } + + const uint32_t frameSerial = m_frame[frameIndex]->frameSerial; + + unsigned recipientCount = 0; + const LGMP_STATUS status = lgmpHostQueuePostForClients( + m_frameOwnerQueue[ownerQueueIndex], scheduleToken, + m_frameMemory[frameIndex], + &schedule.clientID, 1, &recipientCount); + if (status == LGMP_OK && recipientCount) + { + const unsigned queueIndex = static_cast(ownerQueueIndex); + OwnerDelivery& owner = m_ownerDelivery[queueIndex]; + owner.token = scheduleToken; + owner.clientID = schedule.clientID; + owner.frameIndex = static_cast(frameIndex); + owner.active = true; + + FrameDelivery& delivery = m_frameDelivery[frameIndex]; + delivery.ownerQueueMask |= 1U << queueIndex; + } + ReleaseSRWLockExclusive(&m_framePublishLock); + + if (status != LGMP_OK || !recipientCount) + { + if (status != LGMP_OK && status != LGMP_ERR_QUEUE_FULL) + DEBUG_ERROR("Failed to republish frame: %s", + lgmpStatusString(status)); + return false; + } + + m_frameScheduler.FrameRepublished(schedule, frameSerial); + return true; +} + +void CIndirectDeviceContext::CommitFrameBuffer(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule, bool periodic) +{ + if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return; m_frameScheduler.FramePublished( - scheduleGeneration, m_frame[frameIndex]->frameSerial, + schedule, m_frame[frameIndex]->frameSerial, CFrameScheduler::Nanotime(), periodic); } @@ -1571,10 +2145,11 @@ void CIndirectDeviceContext::ForceFrame() } bool CIndirectDeviceContext::GetPublishTarget(uint64_t now, - uint64_t& target, uint32_t& generation, bool& periodic) + uint64_t& target, CFrameScheduler::Schedule& schedule, bool& periodic, + bool& republish) { return m_frameScheduler.GetPublishTarget( - now, target, generation, periodic); + now, target, schedule, periodic, republish); } void CIndirectDeviceContext::FrameSuperseded() @@ -1589,17 +2164,20 @@ void CIndirectDeviceContext::RecordFrameTiming(uint64_t duration) void CIndirectDeviceContext::AbortFrameBuffer(unsigned frameIndex) { - if (frameIndex >= LGMP_Q_FRAME_LEN) + if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return; + AcquireSRWLockExclusive(&m_framePublishLock); m_frameBuffer[frameIndex]->wp = 0; - InterlockedExchange((volatile LONG *)&m_frame[frameIndex]->timingValid, 0); + InterlockedExchange( + (volatile LONG *)&m_frame[frameIndex]->timingValid, 0); m_frameInFlight[frameIndex].store(false, std::memory_order_release); + ReleaseSRWLockExclusive(&m_framePublishLock); } void CIndirectDeviceContext::FailFrameBuffer(unsigned frameIndex) { - if (frameIndex >= LGMP_Q_FRAME_LEN) + if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return; InterlockedExchange((volatile LONG *)&m_frame[frameIndex]->timingValid, 0); @@ -1609,7 +2187,7 @@ void CIndirectDeviceContext::FailFrameBuffer(unsigned frameIndex) void CIndirectDeviceContext::CompleteFrameBuffer(unsigned frameIndex) { - if (frameIndex < LGMP_Q_FRAME_LEN) + if (frameIndex < LGMP_Q_FRAME_BUFFER_LEN) m_frameInFlight[frameIndex].store(false, std::memory_order_release); } @@ -1617,7 +2195,7 @@ void CIndirectDeviceContext::SetFrameTiming(unsigned frameIndex, uint64_t captureTime, uint64_t postProcessTime, uint64_t copyTime, uint64_t readyTime) { - if (frameIndex >= LGMP_Q_FRAME_LEN) + if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return; KVMFRFrame * frame = m_frame[frameIndex]; diff --git a/idd/LGIdd/CIndirectDeviceContext.h b/idd/LGIdd/CIndirectDeviceContext.h index d5133753..a56d14e9 100644 --- a/idd/LGIdd/CIndirectDeviceContext.h +++ b/idd/LGIdd/CIndirectDeviceContext.h @@ -89,6 +89,7 @@ private: PLGMPHost m_lgmp = nullptr; WDFTIMER m_lgmpTimer = nullptr; PLGMPHostQueue m_frameQueue = nullptr; + PLGMPHostQueue m_frameOwnerQueue[LGMP_Q_FRAME_LEN] = {}; SRWLOCK m_lgmpProcessLock = SRWLOCK_INIT; CFrameScheduler m_frameScheduler; @@ -108,14 +109,37 @@ private: size_t m_frameMemoryOffset = 0; size_t m_maxFrameSize = 0; std::atomic m_publishedFrameIndex = -1; - std::atomic m_frameInFlight[LGMP_Q_FRAME_LEN] = {}; - SRWLOCK m_framePublishLock = SRWLOCK_INIT; - bool m_frameResendPending = false; - uint32_t m_formatVer = 0; - uint32_t m_frameSerial = 0; - PLGMPMemory m_frameMemory[LGMP_Q_FRAME_LEN] = {}; - KVMFRFrame * m_frame [LGMP_Q_FRAME_LEN] = {}; - FrameBuffer * m_frameBuffer[LGMP_Q_FRAME_LEN] = {}; + std::atomic m_frameInFlight[LGMP_Q_FRAME_BUFFER_LEN] = {}; + SRWLOCK m_framePublishLock = SRWLOCK_INIT; + bool m_frameResendPending = false; + uint64_t m_framePublishSequence = 0; + uint64_t m_frameLastPublishSequence[LGMP_Q_FRAME_BUFFER_LEN] = {}; + + struct FrameDelivery + { + uint64_t sharedOwnerToken = 0; + unsigned ownerQueueMask = 0; + uint32_t sharedOwnerClientID = 0; + bool sharedOwnerPending = false; + bool sharedPending = false; + bool sharedDelivered = false; + }; + + struct OwnerDelivery + { + uint64_t token = 0; + uint32_t clientID = 0; + unsigned frameIndex = 0; + bool active = false; + }; + + FrameDelivery m_frameDelivery[LGMP_Q_FRAME_BUFFER_LEN] = {}; + OwnerDelivery m_ownerDelivery[LGMP_Q_FRAME_LEN] = {}; + uint32_t m_formatVer = 0; + uint32_t m_frameSerial = 0; + PLGMPMemory m_frameMemory[LGMP_Q_FRAME_BUFFER_LEN] = {}; + KVMFRFrame * m_frame [LGMP_Q_FRAME_BUFFER_LEN] = {}; + FrameBuffer * m_frameBuffer[LGMP_Q_FRAME_BUFFER_LEN] = {}; unsigned m_width = 0; unsigned m_height = 0; @@ -155,6 +179,15 @@ private: bool InitializeLGMP(); void DeInitLGMP(); void LGMPTimer(); + void ProcessFrameDeliveries(); + int FindAvailableFrameBuffer() const; + int FindAvailableOwnerQueue(unsigned preferredIndex) const; + int FindOwnerDelivery(uint32_t clientID) const; + int FindSharedOwnerDelivery(uint32_t clientID) const; + bool HasOwnerDelivery(uint32_t clientID) const; + bool PostSharedFrame(unsigned frameIndex, uint32_t excludeClientID); + bool PostSharedOwnerFrame(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule); void ResendCursor(); void SendColorTransform(); void InitializeEdid(); @@ -222,14 +255,21 @@ public: { unsigned frameIndex; uint8_t* mem; + bool fullCopy; }; - bool FrameBufferAvailable() const; + bool FrameBufferAvailable(const CFrameScheduler::Schedule& schedule); + bool HasPublishedFrame() const + { + return m_publishedFrameIndex.load(std::memory_order_acquire) >= 0; + } void ProcessFrameQueue(); PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects); - bool PublishFrameBuffer(unsigned frameIndex, uint32_t scheduleGeneration); - void CommitFrameBuffer(unsigned frameIndex, uint32_t scheduleGeneration, - bool periodic); + bool PublishFrameBuffer(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule); + bool RepublishFrameBuffer(const CFrameScheduler::Schedule& schedule); + void CommitFrameBuffer(unsigned frameIndex, + const CFrameScheduler::Schedule& schedule, bool periodic); void AbortFrameBuffer(unsigned frameIndex); void FailFrameBuffer(unsigned frameIndex); void CompleteFrameBuffer(unsigned frameIndex); @@ -241,7 +281,7 @@ public: void ObserveFrame(uint64_t now); void ForceFrame(); bool GetPublishTarget(uint64_t now, uint64_t& target, - uint32_t& generation, bool& periodic); + CFrameScheduler::Schedule& schedule, bool& periodic, bool& republish); void FrameSuperseded(); HANDLE GetFrameScheduleEvent() const { diff --git a/idd/LGIdd/CSwapChainProcessor.cpp b/idd/LGIdd/CSwapChainProcessor.cpp index 84261f06..67dd66a5 100644 --- a/idd/LGIdd/CSwapChainProcessor.cpp +++ b/idd/LGIdd/CSwapChainProcessor.cpp @@ -35,7 +35,7 @@ static const uint64_t PUBLISH_RETRY_NS = 1000000ULL; static const DWORD CANDIDATE_WAIT_MS = 2; static_assert(LGMP_Q_FRAME_LEN == 2, - "IDD damage repair assumes two alternating frame buffers"); + "IDD candidate pipeline assumes two slots"); class CSRWExclusiveLock { @@ -223,8 +223,30 @@ void CSwapChainProcessor::PublisherThread() for (;;) { + const uint64_t now = CFrameScheduler::Nanotime(); + uint64_t target; + CFrameScheduler::Schedule schedule; + bool periodic; + bool republish; + m_devContext->GetPublishTarget( + now, target, schedule, periodic, republish); + if (!HasReadyCandidate()) { + if (republish && m_devContext->HasPublishedFrame()) + { + m_devContext->ProcessFrameQueue(); + if (m_devContext->RepublishFrameBuffer(schedule)) + continue; + + ArmPublishTimer(m_publishTimer.Get(), PUBLISH_RETRY_NS); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + if (m_publishTimer.Get()) CancelWaitableTimer(m_publishTimer.Get()); if (WaitForMultipleObjects( @@ -234,12 +256,6 @@ void CSwapChainProcessor::PublisherThread() continue; } - const uint64_t now = CFrameScheduler::Nanotime(); - uint64_t target; - uint32_t generation; - bool periodic; - m_devContext->GetPublishTarget(now, target, generation, periodic); - if (target > now) { ArmPublishTimer(m_publishTimer.Get(), target - now); @@ -252,9 +268,9 @@ void CSwapChainProcessor::PublisherThread() const uint64_t publishStart = CFrameScheduler::Nanotime(); m_devContext->ProcessFrameQueue(); - if (!m_devContext->FrameBufferAvailable() || + if (!m_devContext->FrameBufferAvailable(schedule) || !PublishNewestCandidate( - generation, periodic, publishStart)) + schedule, periodic, publishStart)) { ArmPublishTimer(m_publishTimer.Get(), PUBLISH_RETRY_NS); if (WaitForMultipleObjects( @@ -907,7 +923,8 @@ void CSwapChainProcessor::SignalCandidateState() } bool CSwapChainProcessor::PublishNewestCandidate( - uint32_t scheduleGeneration, bool periodic, uint64_t publishStart) + const CFrameScheduler::Schedule& schedule, bool periodic, + uint64_t publishStart) { int selectedCandidate = -1; uint64_t newestSequence = 0; @@ -1007,7 +1024,7 @@ bool CSwapChainProcessor::PublishNewestCandidate( RECT copyDirtyRects[LG_MAX_DIRTY_RECTS * 2] = {}; unsigned nbCopyDirtyRects = 0; - bool fullCopy = + bool fullCopy = buffer.fullCopy || candidate.nbDirtyRects == 0 || nbPreviousDirtyRects == 0; if (!fullCopy) @@ -1060,7 +1077,7 @@ bool CSwapChainProcessor::PublishNewestCandidate( // Reserve the LGMP message before submitting the copy. This makes post // failure recoverable without racing a very fast GPU completion callback. if (!m_devContext->PublishFrameBuffer( - buffer.frameIndex, scheduleGeneration)) + buffer.frameIndex, schedule)) { copySlot->Cancel(); m_devContext->AbortFrameBuffer(buffer.frameIndex); @@ -1104,7 +1121,7 @@ bool CSwapChainProcessor::PublishNewestCandidate( ReleaseSRWLockExclusive(&m_damageLock); m_devContext->CommitFrameBuffer( - buffer.frameIndex, scheduleGeneration, periodic); + buffer.frameIndex, schedule, periodic); unsigned superseded = 0; AcquireSRWLockExclusive(&m_candidateLock); diff --git a/idd/LGIdd/CSwapChainProcessor.h b/idd/LGIdd/CSwapChainProcessor.h index e0cc21e0..dd4afdc6 100644 --- a/idd/LGIdd/CSwapChainProcessor.h +++ b/idd/LGIdd/CSwapChainProcessor.h @@ -134,7 +134,8 @@ private: static DWORD CALLBACK _PublisherThread(LPVOID arg); void PublisherThread(); bool PublishNewestCandidate( - uint32_t scheduleGeneration, bool periodic, uint64_t publishStart); + const CFrameScheduler::Schedule& schedule, bool periodic, + uint64_t publishStart); bool HasReadyCandidate(); int AcquireCandidate(); void ReleaseCandidate(unsigned candidateIndex); diff --git a/obs/frame_scheduler.c b/obs/frame_scheduler.c index 4627d641..122ffb11 100644 --- a/obs/frame_scheduler.c +++ b/obs/frame_scheduler.c @@ -35,8 +35,9 @@ void lgFrameSchedulerInit(LGFrameScheduler * scheduler, bool supported, uint32_t clientID) { memset(scheduler, 0, sizeof(*scheduler)); - scheduler->supported = supported; - scheduler->clientID = clientID; + scheduler->supported = supported; + scheduler->immediatePending = supported; + scheduler->clientID = clientID; } void lgFrameSchedulerSetPeriod(LGFrameScheduler * scheduler, @@ -60,43 +61,62 @@ void lgFrameSchedulerSetPeriod(LGFrameScheduler * scheduler, return; ++scheduler->generation; - scheduler->resetPending = true; - scheduler->phaseError = 0; - scheduler->feedbackFrameSerial = 0; - scheduler->feedbackSamples = 0; - scheduler->feedbackDirty = false; + scheduler->resetPending = true; + scheduler->immediatePending = true; + scheduler->phaseError = 0; + scheduler->feedbackFrameSerial = 0; + scheduler->feedbackScheduleEpoch = 0; + scheduler->feedbackSamples = 0; + scheduler->feedbackDirty = false; +} + +void lgFrameSchedulerRequestImmediate(LGFrameScheduler * scheduler) +{ + if (scheduler->supported) + scheduler->immediatePending = true; } void lgFrameSchedulerObserveFrame(LGFrameScheduler * scheduler, - uint32_t frameSerial, uint32_t generation, uint64_t readyTime) + uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, + uint64_t readyTime) { - if (!generation || + if (!generation || !scheduleEpoch || (scheduler->readyFrameSerial == frameSerial && - scheduler->readyGeneration == generation)) + scheduler->readyGeneration == generation && + scheduler->readyScheduleEpoch == scheduleEpoch)) return; - scheduler->readyFrameSerial = frameSerial; - scheduler->readyGeneration = generation; - scheduler->readyTime = readyTime; + scheduler->readyFrameSerial = frameSerial; + scheduler->readyGeneration = generation; + scheduler->readyScheduleEpoch = scheduleEpoch; + scheduler->readyTime = readyTime; } void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, - uint32_t frameSerial, uint32_t generation, uint64_t neededTime) + uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, + uint64_t tickTime) { if (!scheduler->active || generation != scheduler->generation || - frameSerial == scheduler->feedbackFrameSerial) + !scheduleEpoch || + (frameSerial == scheduler->feedbackFrameSerial && + scheduleEpoch == scheduler->feedbackScheduleEpoch) || + scheduler->readyFrameSerial != frameSerial || + scheduler->readyGeneration != generation || + scheduler->readyScheduleEpoch != scheduleEpoch) return; - uint64_t readyTime = neededTime; - if (scheduler->readyFrameSerial == frameSerial && - scheduler->readyGeneration == generation) - readyTime = scheduler->readyTime; + if (scheduler->feedbackScheduleEpoch && + scheduler->feedbackScheduleEpoch != scheduleEpoch) + { + scheduler->phaseError = 0; + scheduler->feedbackSamples = 0; + } - const uint64_t measuredPhase = neededTime > readyTime ? - neededTime - readyTime : 0; - int64_t error = measuredPhase > FRAME_SCHEDULER_TARGET_SLACK_NS ? - (int64_t)(measuredPhase - FRAME_SCHEDULER_TARGET_SLACK_NS) : - -(int64_t)(FRAME_SCHEDULER_TARGET_SLACK_NS - measuredPhase); + const int64_t measuredPhase = tickTime >= scheduler->readyTime ? + (int64_t)(tickTime - scheduler->readyTime) : + -(int64_t)(scheduler->readyTime - tickTime); + int64_t error = measuredPhase - + (int64_t)FRAME_SCHEDULER_TARGET_SLACK_NS; const int64_t period = (int64_t)scheduler->period; const int64_t limit = period / 2; if (error > limit) @@ -111,8 +131,9 @@ void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, if (scheduler->feedbackSamples < 32) ++scheduler->feedbackSamples; - scheduler->feedbackFrameSerial = frameSerial; - scheduler->feedbackDirty = true; + scheduler->feedbackFrameSerial = frameSerial; + scheduler->feedbackScheduleEpoch = scheduleEpoch; + scheduler->feedbackDirty = true; } void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, @@ -124,33 +145,41 @@ void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, const uint64_t interval = scheduler->feedbackDirty ? FRAME_SCHEDULER_FEEDBACK_NS : FRAME_SCHEDULER_RENEW_NS; if (scheduler->active && !scheduler->resetPending && + !scheduler->immediatePending && now - scheduler->lastSend < interval) return; KVMFRFrameScheduleFlags flags = KVMFR_FRAME_SCHEDULE_ACTIVE; if (scheduler->resetPending) flags |= KVMFR_FRAME_SCHEDULE_RESET; + if (scheduler->immediatePending) + flags |= KVMFR_FRAME_SCHEDULE_IMMEDIATE; const uint32_t feedbackFrameSerial = scheduler->feedbackFrameSerial; + const uint32_t feedbackScheduleEpoch = + scheduler->feedbackScheduleEpoch; const KVMFRFrameSchedule message = { - .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, - .clientID = scheduler->clientID, - .generation = scheduler->generation, - .flags = flags, - .period = scheduler->period, - .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, - .phaseError = scheduler->phaseError, - .feedbackFrameSerial = feedbackFrameSerial, - .lease = FRAME_SCHEDULER_LEASE_MS, + .msg.type = KVMFR_MESSAGE_FRAME_SCHEDULE, + .clientID = scheduler->clientID, + .generation = scheduler->generation, + .flags = flags, + .period = scheduler->period, + .targetSlack = FRAME_SCHEDULER_TARGET_SLACK_NS, + .phaseError = scheduler->phaseError, + .feedbackFrameSerial = feedbackFrameSerial, + .feedbackScheduleEpoch = feedbackScheduleEpoch, + .lease = FRAME_SCHEDULER_LEASE_MS, }; if (lgmpClientSendData(queue, &message, sizeof(message), NULL) != LGMP_OK) return; - scheduler->active = true; - scheduler->resetPending = false; - scheduler->lastSend = now; - if (scheduler->feedbackFrameSerial == feedbackFrameSerial) + scheduler->active = true; + scheduler->resetPending = false; + scheduler->immediatePending = false; + scheduler->lastSend = now; + if (scheduler->feedbackFrameSerial == feedbackFrameSerial && + scheduler->feedbackScheduleEpoch == feedbackScheduleEpoch) scheduler->feedbackDirty = false; } diff --git a/obs/frame_scheduler.h b/obs/frame_scheduler.h index 091b71d4..8fa14a36 100644 --- a/obs/frame_scheduler.h +++ b/obs/frame_scheduler.h @@ -31,6 +31,7 @@ typedef struct LGFrameScheduler bool supported; bool active; bool resetPending; + bool immediatePending; bool feedbackDirty; uint32_t clientID; uint32_t generation; @@ -39,10 +40,12 @@ typedef struct LGFrameScheduler int64_t phaseError; uint32_t feedbackFrameSerial; + uint32_t feedbackScheduleEpoch; unsigned feedbackSamples; uint32_t readyFrameSerial; uint32_t readyGeneration; + uint32_t readyScheduleEpoch; uint64_t readyTime; } LGFrameScheduler; @@ -51,10 +54,13 @@ void lgFrameSchedulerInit(LGFrameScheduler * scheduler, bool supported, uint32_t clientID); void lgFrameSchedulerSetPeriod(LGFrameScheduler * scheduler, uint64_t period); +void lgFrameSchedulerRequestImmediate(LGFrameScheduler * scheduler); void lgFrameSchedulerObserveFrame(LGFrameScheduler * scheduler, - uint32_t frameSerial, uint32_t generation, uint64_t readyTime); + uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, + uint64_t readyTime); void lgFrameSchedulerFeedback(LGFrameScheduler * scheduler, - uint32_t frameSerial, uint32_t generation, uint64_t neededTime); + uint32_t frameSerial, uint32_t generation, uint32_t scheduleEpoch, + uint64_t tickTime); void lgFrameSchedulerUpdate(LGFrameScheduler * scheduler, PLGMPClientQueue queue, uint64_t now); diff --git a/obs/lg.c b/obs/lg.c index 45c2110f..c758262a 100644 --- a/obs/lg.c +++ b/obs/lg.c @@ -99,7 +99,9 @@ typedef struct int bpp; struct IVSHMEM shmDev; PLGMPClient lgmp; - PLGMPClientQueue frameQueue, pointerQueue; + PLGMPClientQueue frameQueue; + PLGMPClientQueue frameOwnerQueue[LGMP_Q_FRAME_LEN]; + PLGMPClientQueue pointerQueue; gs_texture_t * texture; gs_texture_t * dstTexture; uint8_t * texData; @@ -114,7 +116,7 @@ typedef struct #if LIBOBS_API_MAJOR_VER >= 27 bool dmabuf; bool dmabufTested; - DMAFrameInfo dmaInfo[LGMP_Q_FRAME_LEN]; + DMAFrameInfo dmaInfo[LGMP_Q_FRAME_BUFFER_LEN]; gs_texture_t * dmaTexture; #endif @@ -124,6 +126,8 @@ typedef struct pthread_t frameThread, pointerThread; os_sem_t * frameSem; + uint32_t frameSerial; + bool frameSerialValid; pthread_mutex_t pointerLock; LGFrameScheduler frameScheduler; @@ -169,6 +173,16 @@ typedef struct } LGPlugin; +typedef struct +{ + LGMPMessage msg; + PLGMPClientQueue queue; + uint32_t generation; + uint32_t scheduleEpoch; + bool owner; +} +LGFrameMessage; + static void * frameThread(void * data); static void * pointerThread(void * data); static void lgUpdate(void * data, obs_data_t * settings); @@ -188,6 +202,95 @@ static const char * lgGetName(void * unused) return obs_module_text("Looking Glass Client"); } +static bool lgFrameSerialNewer(uint32_t lhs, uint32_t rhs) +{ + return (int32_t)(lhs - rhs) > 0; +} + +static LGMP_STATUS lgFrameProcessNewest(LGPlugin * this, + LGFrameMessage * result) +{ + struct + { + PLGMPClientQueue queue; + bool owner; + } + queues[LGMP_Q_FRAME_LEN + 1] = + { + { this->frameQueue, false }, + }; + for (unsigned int i = 0; i < LGMP_Q_FRAME_LEN; ++i) + { + queues[i + 1].queue = this->frameOwnerQueue[i]; + queues[i + 1].owner = true; + } + + memset(result, 0, sizeof(*result)); + for (unsigned int i = 0; i < ARRAY_LENGTH(queues); ++i) + { + if (!queues[i].queue) + continue; + + LGMP_STATUS status = lgmpClientAdvanceToLast(queues[i].queue); + if (status != LGMP_OK && status != LGMP_ERR_QUEUE_EMPTY) + return status; + + LGMPMessage msg; + status = lgmpClientProcess(queues[i].queue, &msg); + if (status == LGMP_ERR_QUEUE_EMPTY) + continue; + if (status != LGMP_OK) + return status; + + const KVMFRFrame * frame = (const KVMFRFrame *)msg.mem; + const bool owner = queues[i].owner || msg.udata != 0; + + if (!result->queue) + { + result->msg = msg; + result->queue = queues[i].queue; + result->owner = owner; + continue; + } + + const KVMFRFrame * selected = (const KVMFRFrame *)result->msg.mem; + const bool replace = lgFrameSerialNewer( + frame->frameSerial, selected->frameSerial) || + (frame->frameSerial == selected->frameSerial && + owner && !result->owner); + PLGMPClientQueue discard = queues[i].queue; + if (replace) + { + discard = result->queue; + result->msg = msg; + result->queue = queues[i].queue; + result->owner = owner; + } + + status = lgmpClientMessageDone(discard); + if (status != LGMP_OK) + return status; + } + + if (result->owner) + { + result->generation = (uint32_t)(result->msg.udata >> 32); + result->scheduleEpoch = (uint32_t)result->msg.udata; + } + + return result->queue ? LGMP_OK : LGMP_ERR_QUEUE_EMPTY; +} + +static void lgFrameUnsubscribeOwnerQueues(LGPlugin * this) +{ + for (unsigned int i = 0; i < LGMP_Q_FRAME_LEN; ++i) + { + if (this->frameOwnerQueue[i]) + lgmpClientUnsubscribe(&this->frameOwnerQueue[i]); + this->frameOwnerQueue[i] = NULL; + } +} + static void * lgCreate(obs_data_t * settings, obs_source_t * context) { LGPlugin * this = bzalloc(sizeof(LGPlugin)); @@ -479,6 +582,12 @@ static obs_properties_t * lgGetProperties(void * data) static void * frameThread(void * data) { LGPlugin * this = (LGPlugin *)data; + for (unsigned int i = 0; i < LGMP_Q_FRAME_LEN; ++i) + this->frameOwnerQueue[i] = NULL; + + this->frameQueue = NULL; + this->frameSerial = 0; + this->frameSerialValid = false; if (lgmpClientSubscribe( this->lgmp, LGMP_Q_FRAME, &this->frameQueue) != LGMP_OK) @@ -487,31 +596,47 @@ static void * frameThread(void * data) return NULL; } + if (this->frameScheduler.supported) + { + for (unsigned int i = 0; i < LGMP_Q_FRAME_LEN; ++i) + { + if (lgmpClientSubscribe( + this->lgmp, LGMP_Q_FRAME_OWNER + i, + &this->frameOwnerQueue[i]) == LGMP_OK) + continue; + + lgFrameUnsubscribeOwnerQueues(this); + lgmpClientUnsubscribe(&this->frameQueue); + this->frameQueue = NULL; + this->state = STATE_STOPPING; + return NULL; + } + } + + lgFrameSchedulerRequestImmediate(&this->frameScheduler); + this->state = STATE_RUNNING; os_sem_post(this->frameSem); while(this->state == STATE_RUNNING) { - LGMP_STATUS status; - os_sem_wait(this->frameSem); - if ((status = lgmpClientAdvanceToLast(this->frameQueue)) != LGMP_OK) + LGFrameMessage frameMessage; + const LGMP_STATUS status = lgFrameProcessNewest(this, &frameMessage); + if (status != LGMP_OK && status != LGMP_ERR_QUEUE_EMPTY) { - if (status != LGMP_ERR_QUEUE_EMPTY) - { - os_sem_post(this->frameSem); - printf("lgmpClientAdvanceToLast: %s\n", lgmpStatusString(status)); - break; - } + os_sem_post(this->frameSem); + printf("lgFrameProcessNewest: %s\n", lgmpStatusString(status)); + break; } uint64_t now = os_gettime_ns(); if (status == LGMP_OK) { - LGMPMessage msg; - if (lgmpClientProcess(this->frameQueue, &msg) == LGMP_OK) + const KVMFRFrame * frame = + (const KVMFRFrame *)frameMessage.msg.mem; + if (frameMessage.owner) { - const KVMFRFrame * frame = (const KVMFRFrame *)msg.mem; const FrameBuffer * fb = (const FrameBuffer *)((const uint8_t *)frame + frame->offset); if (framebuffer_wait( @@ -519,7 +644,8 @@ static void * frameThread(void * data) { now = os_gettime_ns(); lgFrameSchedulerObserveFrame(&this->frameScheduler, - frame->frameSerial, msg.udata, now); + frame->frameSerial, frameMessage.generation, + frameMessage.scheduleEpoch, now); } } } @@ -532,8 +658,10 @@ static void * frameThread(void * data) usleep(1000); } + lgFrameUnsubscribeOwnerQueues(this); lgmpClientUnsubscribe(&this->frameQueue); - this->state = STATE_RESTARTING; + this->frameQueue = NULL; + this->state = STATE_RESTARTING; return NULL; } @@ -584,7 +712,7 @@ static void * pointerThread(void * data) if (msg.udata & CURSOR_FLAG_VISIBLE_VALID) { this->cursorVisible = this->hideMouse ? - 0 : msg.udata & CURSOR_FLAG_VISIBLE; + false : (msg.udata & CURSOR_FLAG_VISIBLE) != 0; if (cursor->sdrWhiteLevel) atomic_store(&this->sdrWhiteLevel, cursor->sdrWhiteLevel); } @@ -966,7 +1094,7 @@ static void lgComputeColorMatrix(LGPlugin * this) } static void lgFormatInit(LGPlugin * this, const KVMFRFrame * frame, - LGMPMessage * msg) + LGMPMessage * msg, PLGMPClientQueue frameQueue) { this->formatVer = frame->formatVer; this->screenWidth = frame->screenWidth; @@ -1061,7 +1189,7 @@ static void lgFormatInit(LGPlugin * this, const KVMFRFrame * frame, default: printf("invalid type %d\n", this->type); - lgmpClientMessageDone(this->frameQueue); + lgmpClientMessageDone(frameQueue); os_sem_post(this->frameSem); obs_leave_graphics(); return; @@ -1101,7 +1229,7 @@ static void lgFormatInit(LGPlugin * this, const KVMFRFrame * frame, if (!this->texture) { printf("create texture failed\n"); - lgmpClientMessageDone(this->frameQueue); + lgmpClientMessageDone(frameQueue); os_sem_post(this->frameSem); obs_leave_graphics(); return; @@ -1143,7 +1271,7 @@ static void lgVideoTick(void * data, float seconds) const uint64_t tickTime = os_gettime_ns(); const uint64_t framePeriod = lgFramePeriod(); LGMP_STATUS status; - LGMPMessage msg; + LGFrameMessage frameMessage; os_sem_wait(this->frameSem); if (this->state != STATE_RUNNING) @@ -1269,17 +1397,7 @@ static void lgVideoTick(void * data, float seconds) os_sem_post(this->cursorSem); } - if ((status = lgmpClientAdvanceToLast(this->frameQueue)) != LGMP_OK) - { - if (status != LGMP_ERR_QUEUE_EMPTY) - { - os_sem_post(this->frameSem); - printf("lgmpClientAdvanceToLast: %s\n", lgmpStatusString(status)); - return; - } - } - - if ((status = lgmpClientProcess(this->frameQueue, &msg)) != LGMP_OK) + if ((status = lgFrameProcessNewest(this, &frameMessage)) != LGMP_OK) { if (status == LGMP_ERR_QUEUE_EMPTY) { @@ -1287,25 +1405,64 @@ static void lgVideoTick(void * data, float seconds) return; } - printf("lgmpClientProcess: %s\n", lgmpStatusString(status)); + printf("lgFrameProcessNewest: %s\n", lgmpStatusString(status)); this->state = STATE_STOPPING; os_sem_post(this->frameSem); return; } - const KVMFRFrame * frame = (KVMFRFrame *)msg.mem; - lgFrameSchedulerFeedback(&this->frameScheduler, - frame->frameSerial, msg.udata, tickTime); + const KVMFRFrame * frame = (const KVMFRFrame *)frameMessage.msg.mem; + const bool equalSerial = this->frameSerialValid && + frame->frameSerial == this->frameSerial; + if (this->frameSerialValid && !equalSerial && + !lgFrameSerialNewer(frame->frameSerial, this->frameSerial)) + { + lgmpClientMessageDone(frameMessage.queue); + os_sem_post(this->frameSem); + return; + } + if (equalSerial && !frameMessage.owner) + { + lgmpClientMessageDone(frameMessage.queue); + os_sem_post(this->frameSem); + return; + } + + const FrameBuffer * fb = + (const FrameBuffer *)((const uint8_t *)frame + frame->offset); + bool frameComplete = false; + if (frameMessage.owner) + { + frameComplete = framebuffer_wait( + fb, (size_t)frame->dataHeight * frame->pitch); + if (!frameComplete) + { + lgmpClientMessageDone(frameMessage.queue); + os_sem_post(this->frameSem); + return; + } + + const uint64_t readyTime = os_gettime_ns(); + lgFrameSchedulerObserveFrame(&this->frameScheduler, + frame->frameSerial, frameMessage.generation, + frameMessage.scheduleEpoch, readyTime); + lgFrameSchedulerFeedback(&this->frameScheduler, + frame->frameSerial, frameMessage.generation, + frameMessage.scheduleEpoch, tickTime); + } + + this->frameSerial = frame->frameSerial; + this->frameSerialValid = true; bool textureValid = (this->dmabufTested && this->dmabuf) || this->texture; if (!textureValid || this->formatVer != frame->formatVer) - lgFormatInit(this, frame, &msg); + lgFormatInit(this, frame, &frameMessage.msg, frameMessage.queue); #if LIBOBS_API_MAJOR_VER >= 27 if (this->dmabuf) { obs_enter_graphics(); - DMAFrameInfo * fi = dmabufOpenDMAFrameInfo(this, &msg, frame, + DMAFrameInfo * fi = dmabufOpenDMAFrameInfo(this, &frameMessage.msg, frame, (size_t)frame->dataHeight * frame->pitch); bool importFailed = !fi; if (fi && !fi->texture) @@ -1315,12 +1472,12 @@ static void lgVideoTick(void * data, float seconds) if (!importFailed) { // wait for the frame to be complete before we try to use it - FrameBuffer * fb = (FrameBuffer *)(((uint8_t*)frame) + frame->offset); - const bool complete = framebuffer_wait( - fb, (size_t)frame->dataHeight * frame->pitch); - lgmpClientMessageDone(this->frameQueue); + if (!frameComplete) + frameComplete = framebuffer_wait( + fb, (size_t)frame->dataHeight * frame->pitch); + lgmpClientMessageDone(frameMessage.queue); - if (!complete) + if (!frameComplete) { os_sem_post(this->frameSem); return; @@ -1333,18 +1490,17 @@ static void lgVideoTick(void * data, float seconds) puts("Failed to create dmabuf texture, falling back to CPU upload"); this->dmabuf = false; - lgFormatInit(this, frame, &msg); + lgFormatInit(this, frame, &frameMessage.msg, frameMessage.queue); } #endif if (!this->texture) { - lgmpClientMessageDone(this->frameQueue); + lgmpClientMessageDone(frameMessage.queue); os_sem_post(this->frameSem); return; } - FrameBuffer * fb = (FrameBuffer *)(((uint8_t*)frame) + frame->offset); framebuffer_read( fb, this->texData , // dst @@ -1355,7 +1511,7 @@ static void lgVideoTick(void * data, float seconds) frame->pitch ); - lgmpClientMessageDone(this->frameQueue); + lgmpClientMessageDone(frameMessage.queue); os_sem_post(this->frameSem); obs_enter_graphics(); diff --git a/repos/LGMP b/repos/LGMP index 37fb5d86..58ea4b8a 160000 --- a/repos/LGMP +++ b/repos/LGMP @@ -1 +1 @@ -Subproject commit 37fb5d8612e2ec8ad705e11fb0adf69c18554144 +Subproject commit 58ea4b8aeaa0c801c8231c36fd5324f1cd2186d3